Runtime C++ compiling and testing on Unix-like targets (#981)

* Work in progress to be able to invoke VS from within code.

* First pass at windows version of refactor of OSProcessSpawner

* Closer to getting VS path lookup working.

* Make OSString assignable/ctor able

* Work out program files directory directly, so don't have to expand %%.

* WIP: Improve handling of process spawning.

* Add support for splitting input by line.

* * Correctly locates visual studio install
* Added functionality to invoke vs via cmd

* Add option to execute the command line.

* Handle in ProcessUtil for windows -> WinHandle.

* Rename files slang-win-visual-studio-util.cpp/.h and slang-process-util.h

* First pass at unix/linux version of ProcessUtil.

* Fix reading Visual Studio path from the registry.

* Get compiling on linux with.

* Fix vcvarsall.bat name

* Use ProcessUtil to execute external code.

* Remove OSProcessSpawner.

* Remove includes for "os.h" where no longer needed.

* Fix tabbing issue in premake5.lua
Remove test code from slang-test-main.cpp

* Fix premake4.lua tabbing issue.

* Small fixes to slang-process-util.h
Init ExecuteResult on Win execute.

* Improve comments.

* Fix bug in StringUtil::calcLines - with oddly terminated source input being able to read past end.
Make slang-generate use StringUtil over it's own impl.

* Fix off by one bug in working out Visual Studio version.

* Fix bug in calculating Visual Studio Version

* Fix compilation on linux with string parameter being passed to messageFormat.

* Remove erroneous use of kOSError codes - use Result.

* First effort to generate standard compiler options.

* Initial efforts in compiling source code in test framework for VisualStudio.

* Testing compiling c code on VisualStudio on Windows.

* Fix warning on linux.

* Fix clang on linux warning (and therefore failing) returning a StringBuilder as String.

* Disable return-std-move on clang.

* CommandLine arguments are now tagged if they are escaped or not. That it is the clients responsibility to escape command lines that cannot be automatically escaped.

* Add checks on unix/linux that command line args are all unescaped.

* WIP getting runtime GCC to work.

* First pass compiler working on unix-like targets.

* Enable c-compile.c test on 'smoke'.

9 files changed, 295 insertions, 41 deletions

diff --git a/source/core/core.vcxproj b/source/core/core.vcxproj
index 684b5592c..a855630c2 100644
--- a/source/core/core.vcxproj
+++ b/source/core/core.vcxproj
@@ -176,6 +176,7 @@
     <ClInclude Include="slang-basic.h" />
     <ClInclude Include="slang-byte-encode-util.h" />
     <ClInclude Include="slang-common.h" />
+    <ClInclude Include="slang-cpp-compiler.h" />
     <ClInclude Include="slang-dictionary.h" />
     <ClInclude Include="slang-exception.h" />
     <ClInclude Include="slang-free-list.h" />
diff --git a/source/core/core.vcxproj.filters b/source/core/core.vcxproj.filters
index eec57c50b..df83072f8 100644
--- a/source/core/core.vcxproj.filters
+++ b/source/core/core.vcxproj.filters
@@ -27,6 +27,9 @@
     <ClInclude Include="slang-common.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="slang-cpp-compiler.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
     <ClInclude Include="slang-dictionary.h">
       <Filter>Header Files</Filter>
     </ClInclude>
diff --git a/source/core/slang-cpp-compiler.h b/source/core/slang-cpp-compiler.h
new file mode 100644
index 000000000..644d3d85b
--- /dev/null
+++ b/source/core/slang-cpp-compiler.h
@@ -0,0 +1,71 @@
+#ifndef SLANG_CPP_COMPILER_H
+#define SLANG_CPP_COMPILER_H
+
+#include "slang-common.h"
+#include "slang-string.h"
+
+namespace Slang
+{
+
+class CPPCompiler
+{
+public:
+    enum class Type
+    {
+        VisualStudio,
+        GCC,
+        Clang,
+    };
+
+    struct Version
+    {
+        Type type;      ///< The compiler type
+        Int major;      ///< The major version number
+        Int minor;      ///< The minor version number
+    };
+};
+
+struct CPPCompileOptions
+{
+    enum class OptimizationLevel
+    {
+        Normal,             ///< Normal optimization
+        Debug,              ///< General has no optimizations
+    };
+
+    enum DebugInfoType
+    {
+        None,               ///< Binary has no debug information
+        Maximum,            ///< Has maximum debug information
+        Normal,             ///< Has normal debug information
+    };
+    enum TargetType
+    {
+        Executable,         ///< Produce an executable
+        SharedLibrary,      ///< Produce a shared library object/dll 
+        Object,             ///< Produce an object file
+    };
+
+    struct Define
+    {
+        String nameWithSig;             ///< If macro takes parameters include in brackets
+        String value;
+    };
+
+    OptimizationLevel optimizationLevel = OptimizationLevel::Debug;
+    DebugInfoType debugInfoType = DebugInfoType::Normal;
+    TargetType targetType = TargetType::Executable;
+
+    String modulePath;      ///< The path/name of the output module. Should not have the extension, as that will be added for each of the target types
+
+    List<Define> defines;
+
+    List<String> sourceFiles;
+
+    List<String> includePaths;
+    List<String> libraryPaths;
+};
+
+}
+
+#endif
diff --git a/source/core/slang-string-util.cpp b/source/core/slang-string-util.cpp
index e571bbc99..60ebc45ba 100644
--- a/source/core/slang-string-util.cpp
+++ b/source/core/slang-string-util.cpp
@@ -243,4 +243,28 @@ ComPtr<ISlangBlob> StringUtil::createStringBlob(const String& string)
     }
 }
 
+/* static */bool StringUtil::areLinesEqual(const UnownedStringSlice& a, const UnownedStringSlice& b)
+{
+    List<UnownedStringSlice> slicesA;
+    List<UnownedStringSlice> slicesB;
+
+    calcLines(a, slicesA);
+    calcLines(b, slicesB);
+
+    const auto linesCount = slicesA.getCount();
+    if (linesCount != slicesB.getCount())
+    {
+        return false;
+    }
+
+    for (Index i = 0; i < linesCount; ++i)
+    {
+        if (slicesA[i] != slicesB[i])
+        {
+            return false;
+        }
+    }
+    return true;
+}
+
 } // namespace Slang
diff --git a/source/core/slang-string-util.h b/source/core/slang-string-util.h
index dc0d7e546..6a3b60240 100644
--- a/source/core/slang-string-util.h
+++ b/source/core/slang-string-util.h
@@ -80,6 +80,9 @@ struct StringUtil
 
         /// Given text, splits into lines stored in outLines. NOTE! That lines is only valid as long as textIn remains valid
     static void calcLines(const UnownedStringSlice& textIn, List<UnownedStringSlice>& lines);
+
+        /// Equal if the lines are equal (in effect a way to ignore differences in line breaks)
+    static bool areLinesEqual(const UnownedStringSlice& a, const UnownedStringSlice& b);
 };
 
 } // namespace Slang
diff --git a/source/core/unix/slang-unix-cpp-compiler-util.cpp b/source/core/unix/slang-unix-cpp-compiler-util.cpp
new file mode 100644
index 000000000..a5459b665
--- /dev/null
+++ b/source/core/unix/slang-unix-cpp-compiler-util.cpp
@@ -0,0 +1,163 @@
+#include "slang-unix-cpp-compiler-util.h"
+
+#include "../slang-common.h"
+#include "../slang-process-util.h"
+#include "../slang-string-util.h"
+
+#include "../slang-shared-library.h"
+
+#include "../slang-io.h"
+
+// The method used to invoke VS was originally inspired by some ideas in
+// https://github.com/RuntimeCompiledCPlusPlus/RuntimeCompiledCPlusPlus/
+
+namespace Slang {
+
+
+/* static */void UnixCPPCompilerUtil::calcArgs(const CPPCompileOptions& options, CommandLine& cmdLine)
+{
+    typedef CPPCompileOptions::OptimizationLevel OptimizationLevel;
+    typedef CPPCompileOptions::TargetType TargetType;
+    typedef CPPCompileOptions::DebugInfoType DebugInfoType;
+
+    cmdLine.addArg("-fvisibility=hidden");
+    // Use shared libraries
+    //cmdLine.addArg("-shared");
+
+    switch (options.optimizationLevel)
+    {
+        case OptimizationLevel::Debug:
+        {
+            // No optimization
+            cmdLine.addArg("-O0");
+            break;
+        }
+        case OptimizationLevel::Normal:
+        {
+            cmdLine.addArg("-Os");
+            break;
+        }
+        default: break;
+    }
+
+    if (options.debugInfoType != DebugInfoType::None)
+    {
+        cmdLine.addArg("-g");
+    }
+
+    switch (options.targetType)
+    {
+        case TargetType::SharedLibrary:
+        {
+            // Position independent
+            cmdLine.addArg("-fPIC");
+
+            String sharedLibraryPath;
+
+            // Work out the shared library name
+            {
+                String moduleDir = Path::getParentDirectory(options.modulePath);
+                String moduleFilename = Path::getFileName(options.modulePath);
+
+                StringBuilder sharedLibraryFilename;
+                SharedLibrary::appendPlatformFileName(moduleFilename.getUnownedSlice(), sharedLibraryFilename);
+
+                if (moduleDir.getLength() > 0)
+                {
+                    sharedLibraryPath = Path::combine(moduleDir, sharedLibraryFilename);
+                }
+                else
+                {
+                    sharedLibraryPath = sharedLibraryFilename;
+                }
+            }
+
+            cmdLine.addArg("-o");
+            cmdLine.addArg(sharedLibraryPath);
+            break;
+        }
+        case TargetType::Executable:
+        {
+            cmdLine.addArg("-o");
+
+            StringBuilder builder;
+            builder << options.modulePath;
+            builder << ProcessUtil::getExecutableSuffix();
+            
+            cmdLine.addArg(options.modulePath);
+            break;
+        }
+        case TargetType::Object:
+        {
+            // Don't link, just produce object file
+            cmdLine.addArg("-c");
+            break;
+        }
+        default: break;
+    }
+
+    // Add defines
+    for (const auto& define : options.defines)
+    {
+        StringBuilder builder;
+        builder << define.nameWithSig;
+        if (define.value.getLength())
+        {
+            builder << "=" << define.value;
+        }
+
+        cmdLine.addArg(builder);
+    }
+
+    // Add includes
+    for (const auto& include : options.includePaths)
+    {
+        cmdLine.addArg("-I");
+        cmdLine.addArg(include);
+    }
+
+    // Link options
+    if (0)
+    {
+        StringBuilder linkOptions;
+        linkOptions << "Wl,";
+        cmdLine.addArg(linkOptions);
+    }
+
+    // Files to compile
+    for (const auto& sourceFile : options.sourceFiles)
+    {
+        cmdLine.addArg(sourceFile);
+    }
+
+    for (const auto& libPath : options.libraryPaths)
+    {
+        // Note that any escaping of the path is handled in the ProcessUtil::
+        cmdLine.addArg("-L");
+        cmdLine.addArg(libPath);
+        cmdLine.addArg("-F");
+        cmdLine.addArg(libPath);
+    }
+}
+
+/* static */SlangResult UnixCPPCompilerUtil::executeCompiler(const CommandLine& commandLine, ExecuteResult& outResult)
+{
+    CommandLine cmdLine;
+    // We'll assume g++ for now
+    cmdLine.setExecutableFilename("g++");
+
+    // Append the command line options
+    cmdLine.addArgs(commandLine.m_args.getBuffer(), commandLine.m_args.getCount());
+
+#if 0
+    // Test
+    {
+        String line = ProcessUtil::getCommandLineString(cmdLine);
+        printf("%s", line.getBuffer());
+    }
+#endif
+
+    return ProcessUtil::execute(cmdLine, outResult);
+}
+
+} // namespace Slang
diff --git a/source/core/unix/slang-unix-cpp-compiler-util.h b/source/core/unix/slang-unix-cpp-compiler-util.h
new file mode 100644
index 000000000..03c1b5e2e
--- /dev/null
+++ b/source/core/unix/slang-unix-cpp-compiler-util.h
@@ -0,0 +1,24 @@
+#ifndef SLANG_UNIX_CPP_COMPILER_UTIL_H
+#define SLANG_UNIX_CPP_COMPILER_UTIL_H
+
+#include "../slang-list.h"
+#include "../slang-string.h"
+
+#include "../slang-process-util.h"
+
+#include "../slang-cpp-compiler.h"
+
+namespace Slang {
+
+struct UnixCPPCompilerUtil
+{
+        /// Run compiler with command line (typically generated by calcArgs). Output placed in outResult.
+    static SlangResult executeCompiler(const CommandLine& commandLine, ExecuteResult& outResult);
+
+        /// Calculate the command line args
+    static void calcArgs(const CPPCompileOptions& options, CommandLine& cmdLine);
+};
+
+} // namespace Slang
+
+#endif 
diff --git a/source/core/unix/slang-unix-process-util.cpp b/source/core/unix/slang-unix-process-util.cpp
index 0447dd499..400f4f773 100644
--- a/source/core/unix/slang-unix-process-util.cpp
+++ b/source/core/unix/slang-unix-process-util.cpp
@@ -20,7 +20,11 @@ namespace Slang {
 
 /* static */UnownedStringSlice ProcessUtil::getExecutableSuffix()
 {
+#if __CYGWIN__
+    return UnownedStringSlice::fromLiteral(".exe");
+#else
     return UnownedStringSlice::fromLiteral("");
+#endif
 }
 
 /* static */void ProcessUtil::appendCommandLineEscaped(const UnownedStringSlice& slice, StringBuilder& out)
diff --git a/source/core/windows/slang-win-visual-studio-util.h b/source/core/windows/slang-win-visual-studio-util.h
index fefb9ad0f..e8afbf418 100644
--- a/source/core/windows/slang-win-visual-studio-util.h
+++ b/source/core/windows/slang-win-visual-studio-util.h
@@ -6,48 +6,9 @@
 
 #include "../slang-process-util.h"
 
-namespace Slang {
-
-struct CPPCompileOptions
-{
-    enum class OptimizationLevel
-    {
-        Normal,             ///< Normal optimization
-        Debug,              ///< General has no optimizations
-    };
-
-    enum DebugInfoType
-    {
-        None,               ///< Binary has no debug information
-        Maximum,            ///< Has maximum debug information
-        Normal,             ///< Has normal debug information
-    };
-    enum TargetType
-    {
-        Executable,         ///< Produce an executable
-        SharedLibrary,      ///< Produce a shared library object/dll 
-        Object,             ///< Produce an object file
-    };
+#include "../slang-cpp-compiler.h"
 
-    struct Define
-    {
-        String nameWithSig;             ///< If macro takes parameters include in brackets
-        String value;
-    };
-
-    OptimizationLevel optimizationLevel = OptimizationLevel::Debug;
-    DebugInfoType debugInfoType = DebugInfoType::Normal;
-    TargetType targetType = TargetType::Executable;
-
-    String modulePath;      ///< The path/name of the output module. Should not have the extension, as that will be added for each of the target types
-
-    List<Define> defines;
-
-    List<String> sourceFiles;
-
-    List<String> includePaths;
-    List<String> libraryPaths;
-};
+namespace Slang {
 
 struct WinVisualStudioUtil
 {