Callable CPU code support (#1014)

* First pass support for compiling to a loaded shared library.

* Improve documentation for cpu target.

* Removed the SLANG_COMPILE_FLAG_LOAD_SHARED_LIBRARY flag.
Use the SLANG_HOST_CALLABLE code target
Document mechanism.

* Fix typo in cpp-resource.slang
In test code if the target is 'callable' we don't need to compile (indeed there is no source file).

* Small refactor using CommandLineCPPCompiler as base class to implement VisualStudioCPPCompiler and GCCCPPCompiler.

* Improvements around CPPCompiler.
Mechanism to know products produced.
Cleaning up products after execution.

* Fix multiple definition of 'SourceType'

10 files changed, 211 insertions, 60 deletions

diff --git a/source/core/slang-cpp-compiler.cpp b/source/core/slang-cpp-compiler.cpp
index 54138ab27..7ac4e96b4 100644
--- a/source/core/slang-cpp-compiler.cpp
+++ b/source/core/slang-cpp-compiler.cpp
@@ -160,9 +160,9 @@ void CPPCompiler::Output::removeByType(OutputMessage::Type type)
     }
 }
 
-/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! GenericCPPCompiler !!!!!!!!!!!!!!!!!!!!!!*/
+/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! CommandLineCPPCompiler !!!!!!!!!!!!!!!!!!!!!!*/
 
-SlangResult GenericCPPCompiler::compile(const CompileOptions& options, Output& outOutput)
+SlangResult CommandLineCPPCompiler::compile(const CompileOptions& options, Output& outOutput)
 {
     outOutput.reset();
 
@@ -170,7 +170,7 @@ SlangResult GenericCPPCompiler::compile(const CompileOptions& options, Output& o
     CommandLine cmdLine(m_cmdLine);
 
     // Append command line args to the end of cmdLine using the target specific function for the specified options
-    m_calcArgsFunc(options, cmdLine);
+    SLANG_RETURN_ON_FAIL(calcArgs(options, cmdLine));
 
     ExecuteResult exeRes;
 
@@ -190,12 +190,7 @@ SlangResult GenericCPPCompiler::compile(const CompileOptions& options, Output& o
     }
 #endif
 
-    return m_parseOutputFunc(exeRes, outOutput);
-}
-
-SlangResult GenericCPPCompiler::calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath)
-{
-    return m_calcModuleFilePathFunc(options, outPath);
+    return parseOutput(exeRes, outOutput);
 }
 
 /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! CPPCompilerUtil !!!!!!!!!!!!!!!!!!!!!!*/
@@ -339,7 +334,8 @@ static void _addGCCFamilyCompiler(const String& exeName, CPPCompilerSet* compile
     CPPCompiler::Desc desc;
     if (SLANG_SUCCEEDED(GCCCompilerUtil::calcVersion(exeName, desc)))
     {
-        RefPtr<CPPCompiler> compiler(new GenericCPPCompiler(desc, exeName, &GCCCompilerUtil::calcArgs, &GCCCompilerUtil::parseOutput, GCCCompilerUtil::calcModuleFilePath));
+        RefPtr<CommandLineCPPCompiler> compiler(new GCCCPPCompiler(desc));
+        compiler->m_cmdLine.setExecutableFilename(exeName);
         compilerSet->addCompiler(compiler);
     }
 }
diff --git a/source/core/slang-cpp-compiler.h b/source/core/slang-cpp-compiler.h
index 851975d72..185e960f3 100644
--- a/source/core/slang-cpp-compiler.h
+++ b/source/core/slang-cpp-compiler.h
@@ -147,6 +147,27 @@ public:
         Int fileLine;                   ///< The line number the error came from
     };
 
+    typedef uint32_t ProductFlags;
+    struct ProductFlag
+    {
+        enum Enum : ProductFlags
+        {
+            Debug               = 0x1,                ///< Used by debugger during execution
+            Execution           = 0x2,                ///< Required for execution
+            Compile             = 0x4,                ///< A product *required* for compilation
+            Miscellaneous       = 0x8,                ///< Anything else
+            All                 = 0xf,                ///< All the flags
+        };
+    };
+
+    enum class Product
+    {
+        DebugRun,
+        Run,
+        CompileTemporary,
+        All,
+    };
+
     struct Output
     {
             /// Reset to an initial empty state
@@ -178,6 +199,10 @@ public:
     virtual SlangResult compile(const CompileOptions& options, Output& outOutput) = 0;
         /// Given the compilation options and the module name, determines the actual file name used for output
     virtual SlangResult calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath) = 0;
+        /// Given options determines the paths to products produced (including the 'moduleFilePath').
+        /// Note that does *not* guarentee all products were or should be produced. Just aims to include all that could
+        /// be produced, such that can be removed on completion.
+    virtual SlangResult calcCompileProducts(const CompileOptions& options, ProductFlags flags, List<String>& outPaths) = 0;
 
         /// Return the compiler type as name
     static UnownedStringSlice getCompilerTypeAsText(CompilerType type);
@@ -191,38 +216,31 @@ protected:
     Desc m_desc;
 };
 
-class GenericCPPCompiler : public CPPCompiler
+class CommandLineCPPCompiler : public CPPCompiler
 {
 public:
     typedef CPPCompiler Super;
 
-    typedef void(*CalcArgsFunc)(const CPPCompiler::CompileOptions& options, CommandLine& cmdLine);
-    typedef SlangResult(*ParseOutputFunc)(const ExecuteResult& exeResult, Output& output);
-    typedef SlangResult(*CalcModuleFilePathFunc)(const CPPCompiler::CompileOptions& options, StringBuilder& outPath);
-
+    // CPPCompiler
     virtual SlangResult compile(const CompileOptions& options, Output& outOutput) SLANG_OVERRIDE;
-    virtual SlangResult calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath) SLANG_OVERRIDE;
+    
+    // Functions to be implemented for a specific CommandLine    
+    virtual SlangResult calcArgs(const CompileOptions& options, CommandLine& cmdLine) = 0;
+    virtual SlangResult parseOutput(const ExecuteResult& exeResult, Output& output) = 0;
 
-    GenericCPPCompiler(const Desc& desc, const String& exeName, CalcArgsFunc calcArgsFunc, ParseOutputFunc parseOutputFunc, CalcModuleFilePathFunc calcModuleFilePathFunc) :
-        Super(desc),
-        m_calcArgsFunc(calcArgsFunc),
-        m_parseOutputFunc(parseOutputFunc),
-        m_calcModuleFilePathFunc(calcModuleFilePathFunc)
+    CommandLineCPPCompiler(const Desc& desc, const String& exeName) :
+        Super(desc)
     {
         m_cmdLine.setExecutableFilename(exeName);
     }
 
-    GenericCPPCompiler(const Desc& desc, const CommandLine& cmdLine, CalcArgsFunc calcArgsFunc, ParseOutputFunc parseOutputFunc, CalcModuleFilePathFunc calcModuleFilePathFunc) :
+    CommandLineCPPCompiler(const Desc& desc, const CommandLine& cmdLine) :
         Super(desc),
-        m_cmdLine(cmdLine),
-        m_calcArgsFunc(calcArgsFunc),
-        m_parseOutputFunc(parseOutputFunc),
-        m_calcModuleFilePathFunc(calcModuleFilePathFunc)
+        m_cmdLine(cmdLine)
     {}
 
-    CalcArgsFunc m_calcArgsFunc;
-    ParseOutputFunc m_parseOutputFunc;
-    CalcModuleFilePathFunc m_calcModuleFilePathFunc;
+    CommandLineCPPCompiler(const Desc& desc):Super(desc) {}
+
     CommandLine m_cmdLine;
 };
 
@@ -261,7 +279,8 @@ protected:
     List<RefPtr<CPPCompiler>> m_compilers;
 };
 
-struct CPPCompilerUtil
+/* Only purpose of having base-class here is to make all the CPPCompiler types available directly in derived Utils */
+struct CPPCompilerBaseUtil
 {
     typedef CPPCompiler::CompileOptions CompileOptions;
     typedef CPPCompiler::OptimizationLevel OptimizationLevel;
@@ -269,6 +288,14 @@ struct CPPCompilerUtil
     typedef CPPCompiler::DebugInfoType DebugInfoType;
     typedef CPPCompiler::SourceType SourceType;
 
+    typedef CPPCompiler::OutputMessage OutputMessage;
+    typedef CPPCompiler::FloatingPointMode FloatingPointMode;
+    typedef CPPCompiler::ProductFlag ProductFlag;
+    typedef CPPCompiler::ProductFlags ProductFlags;
+};
+
+struct CPPCompilerUtil: public CPPCompilerBaseUtil
+{
     enum class MatchType
     {
         MinGreaterEqual,
@@ -289,12 +316,8 @@ struct CPPCompilerUtil
 
         /// Given a set, registers compilers found through standard means and determines a reasonable default compiler if possible
     static SlangResult initializeSet(CPPCompilerSet* set);
-
-    
-    
 };
 
-
 }
 
 #endif
diff --git a/source/core/slang-gcc-compiler-util.cpp b/source/core/slang-gcc-compiler-util.cpp
index 0a53bcac8..d42076039 100644
--- a/source/core/slang-gcc-compiler-util.cpp
+++ b/source/core/slang-gcc-compiler-util.cpp
@@ -354,8 +354,12 @@ static SlangResult _parseGCCFamilyLine(const UnownedStringSlice& line, LineParse
         }
         case TargetType::Object:
         {
+#if __CYGWIN__
+            outPath << options.modulePath << ".obj";
+#else
             // Will be .o for typical gcc targets
             outPath << options.modulePath << ".o";
+#endif
             return SLANG_OK;
         }
     }
@@ -363,7 +367,21 @@ static SlangResult _parseGCCFamilyLine(const UnownedStringSlice& line, LineParse
     return SLANG_FAIL;
 }
 
-/* static */void GCCCompilerUtil::calcArgs(const CompileOptions& options, CommandLine& cmdLine)
+/* static */SlangResult GCCCompilerUtil::calcCompileProducts(const CompileOptions& options, ProductFlags flags, List<String>& outPaths)
+{
+    outPaths.clear();
+
+    if (flags & ProductFlag::Execution)
+    {
+        StringBuilder builder;
+        SLANG_RETURN_ON_FAIL(calcModuleFilePath(options, builder));
+        outPaths.add(builder);
+    }
+
+    return SLANG_OK;
+}
+
+/* static */SlangResult GCCCompilerUtil::calcArgs(const CompileOptions& options, CommandLine& cmdLine)
 {
     cmdLine.addArg("-fvisibility=hidden");
 
@@ -513,6 +531,8 @@ static SlangResult _parseGCCFamilyLine(const UnownedStringSlice& line, LineParse
 	// Make maths lib available
         cmdLine.addArg("-lm");
     }
+
+    return SLANG_OK;
 }
 
 }
diff --git a/source/core/slang-gcc-compiler-util.h b/source/core/slang-gcc-compiler-util.h
index a72080acc..df3cb8a4b 100644
--- a/source/core/slang-gcc-compiler-util.h
+++ b/source/core/slang-gcc-compiler-util.h
@@ -7,15 +7,8 @@ namespace Slang
 {
 
 /* Utility for processing input and output of gcc-like compilers, including clang */
-struct GCCCompilerUtil
+struct GCCCompilerUtil : public CPPCompilerBaseUtil
 {
-    typedef CPPCompiler::CompileOptions CompileOptions;
-    typedef CPPCompiler::OptimizationLevel OptimizationLevel;
-    typedef CPPCompiler::TargetType TargetType;
-    typedef CPPCompiler::DebugInfoType DebugInfoType;
-    typedef CPPCompiler::SourceType SourceType;
-    typedef CPPCompiler::FloatingPointMode FloatingPointMode;
-
         /// Extracts version number into desc from text (assumes gcc/clang -v layout with a line with version)
     static SlangResult parseVersion(const UnownedStringSlice& text, const UnownedStringSlice& prefix, CPPCompiler::Desc& outDesc);
 
@@ -23,7 +16,7 @@ struct GCCCompilerUtil
     static SlangResult calcVersion(const String& exeName, CPPCompiler::Desc& outDesc);
 
         /// Calculate gcc family compilers (including clang) cmdLine arguments from options
-    static void calcArgs(const CompileOptions& options, CommandLine& cmdLine);
+    static SlangResult calcArgs(const CompileOptions& options, CommandLine& cmdLine);
 
         /// Parse ExecuteResult into Output
     static SlangResult parseOutput(const ExecuteResult& exeRes, CPPCompiler::Output& outOutput);
@@ -31,6 +24,23 @@ struct GCCCompilerUtil
         /// Calculate the output module filename 
     static SlangResult calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath);
 
+        /// Given options, calculate paths to products produced for a compilation
+    static SlangResult calcCompileProducts(const CompileOptions& options, ProductFlags flags, List<String>& outPaths);
+};
+
+class GCCCPPCompiler : public CommandLineCPPCompiler
+{
+public:
+    typedef CommandLineCPPCompiler Super;
+    typedef GCCCompilerUtil Util;
+
+    // CommandLineCPPCompiler impl  - just forwards to the Util
+    virtual SlangResult calcArgs(const CompileOptions& options, CommandLine& cmdLine) SLANG_OVERRIDE { return Util::calcArgs(options, cmdLine); }
+    virtual SlangResult parseOutput(const ExecuteResult& exeResult, Output& output) SLANG_OVERRIDE { return Util::parseOutput(exeResult, output); }
+    virtual SlangResult calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath) SLANG_OVERRIDE { return Util::calcModuleFilePath(options, outPath); }
+    virtual SlangResult calcCompileProducts(const CompileOptions& options, ProductFlags flags,  List<String>& outPaths) SLANG_OVERRIDE { return Util::calcCompileProducts(options, flags, outPaths); }
+
+    GCCCPPCompiler(const Desc& desc):Super(desc) {}
 };
 
 }
diff --git a/source/core/slang-io.h b/source/core/slang-io.h
index e2935afe4..758a05edb 100644
--- a/source/core/slang-io.h
+++ b/source/core/slang-io.h
@@ -93,6 +93,14 @@ namespace Slang
     // Helper class to clean up temporary files on dtor
     struct TemporaryFileSet
     {
+        void remove(const String& path)
+        {
+            if (const Index index = m_paths.indexOf(path) >= 0)
+            {
+                m_paths.removeAt(index);
+            }
+        }
+
         void add(const String& path)
         {
             if (m_paths.indexOf(path) < 0)
@@ -100,6 +108,17 @@ namespace Slang
                 m_paths.add(path);
             }
         }
+        void add(const List<String>& paths)
+        {
+            for (const auto& path : paths)
+            {
+                add(path);
+            }
+        }
+        void clear()
+        {
+            m_paths.clear();
+        }
         ~TemporaryFileSet()
         {
             for (const auto& path : m_paths)
diff --git a/source/core/slang-shared-library.cpp b/source/core/slang-shared-library.cpp
index 9932db0b5..fd61ba0a0 100644
--- a/source/core/slang-shared-library.cpp
+++ b/source/core/slang-shared-library.cpp
@@ -56,6 +56,18 @@ SlangResult DefaultSharedLibraryLoader::loadSharedLibrary(const char* path, ISla
 
 /* !!!!!!!!!!!!!!!!!!!!!!!!!! DefaultSharedLibrary !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!*/
 
+TemporarySharedLibrary::~TemporarySharedLibrary()
+{
+    if (m_sharedLibraryHandle)
+    {
+        // We have to unload if we want to be able to remove
+        SharedLibrary::unload(m_sharedLibraryHandle);
+        m_sharedLibraryHandle = nullptr;
+    }
+}
+
+/* !!!!!!!!!!!!!!!!!!!!!!!!!! DefaultSharedLibrary !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!*/
+
 ISlangUnknown* DefaultSharedLibrary::getInterface(const Guid& guid)
 {
     return (guid == IID_ISlangUnknown || guid == IID_ISlangSharedLibrary) ? static_cast<ISlangSharedLibrary*>(this) : nullptr;
@@ -63,7 +75,10 @@ ISlangUnknown* DefaultSharedLibrary::getInterface(const Guid& guid)
 
 DefaultSharedLibrary::~DefaultSharedLibrary()
 {
-    SharedLibrary::unload(m_sharedLibraryHandle);
+    if (m_sharedLibraryHandle)
+    {
+        SharedLibrary::unload(m_sharedLibraryHandle);
+    }
 }
 
 SlangFuncPtr DefaultSharedLibrary::findFuncByName(char const* name)
diff --git a/source/core/slang-shared-library.h b/source/core/slang-shared-library.h
index 5a4eb7229..e48162ebd 100644
--- a/source/core/slang-shared-library.h
+++ b/source/core/slang-shared-library.h
@@ -5,6 +5,7 @@
 #include "../../slang-com-helper.h"
 #include "../../slang-com-ptr.h"
 
+#include "../core/slang-io.h"
 #include "../core/slang-platform.h"
 #include "../core/slang-common.h"
 #include "../core/slang-dictionary.h"
@@ -84,6 +85,30 @@ class DefaultSharedLibrary : public ISlangSharedLibrary, public RefObject
     SharedLibrary::Handle m_sharedLibraryHandle = nullptr;
 };
 
+class TemporarySharedLibrary : public DefaultSharedLibrary
+{
+public:
+    typedef DefaultSharedLibrary Super;
+
+        /// Get the path to the shared library
+    const String& getPath() const { return m_path; }
+
+        /// Ctor
+    TemporarySharedLibrary(const SharedLibrary::Handle sharedLibraryHandle, const String& path):
+        Super(sharedLibraryHandle),
+        m_path(path)
+    {
+    }
+
+    virtual ~TemporarySharedLibrary();
+
+        /// Any files specified in this set will be deleted on exit
+    TemporaryFileSet m_temporaryFileSet;
+
+protected:
+    String m_path;
+};
+
 class ConfigurableSharedLibraryLoader: public ISlangSharedLibraryLoader, public RefObject
 {
 public:
diff --git a/source/core/slang-visual-studio-compiler-util.cpp b/source/core/slang-visual-studio-compiler-util.cpp
index 765edc349..1c79f9004 100644
--- a/source/core/slang-visual-studio-compiler-util.cpp
+++ b/source/core/slang-visual-studio-compiler-util.cpp
@@ -37,7 +37,40 @@ namespace Slang
     return SLANG_FAIL;
 }
 
-/* static */void VisualStudioCompilerUtil::calcArgs(const CompileOptions& options, CommandLine& cmdLine)
+/* static */SlangResult VisualStudioCompilerUtil::calcCompileProducts(const CompileOptions& options, ProductFlags flags, List<String>& outPaths)
+{
+    outPaths.clear();
+
+    if (flags & ProductFlag::Execution)
+    {
+        StringBuilder builder;
+        SLANG_RETURN_ON_FAIL(calcModuleFilePath(options, builder));
+        outPaths.add(builder);
+    }
+    if (flags & ProductFlag::Miscellaneous)
+    {
+        outPaths.add(options.modulePath + ".ilk");
+
+        if (options.targetType == TargetType::SharedLibrary)
+        {
+            outPaths.add(options.modulePath + ".exp");
+            outPaths.add(options.modulePath + ".lib");
+        }
+    }
+    if (flags & ProductFlag::Compile)
+    {
+        outPaths.add(options.modulePath + ".obj");
+    }
+    if (flags & ProductFlag::Debug)
+    {
+        // TODO(JS): Could try and determine based on debug information
+        outPaths.add(options.modulePath + ".pdb");
+    }
+
+    return SLANG_OK;
+}
+
+/* static */SlangResult VisualStudioCompilerUtil::calcArgs(const CompileOptions& options, CommandLine& cmdLine)
 {
     // https://docs.microsoft.com/en-us/cpp/build/reference/compiler-options-listed-alphabetically?view=vs-2019
 
@@ -190,6 +223,8 @@ namespace Slang
         // Note that any escaping of the path is handled in the ProcessUtil::
         cmdLine.addPrefixPathArg("/LIBPATH:", libPath);
     }
+
+    return SLANG_OK;
 }
 
 static SlangResult _parseErrorType(const UnownedStringSlice& in, CPPCompiler::OutputMessage::Type& outType)
diff --git a/source/core/slang-visual-studio-compiler-util.h b/source/core/slang-visual-studio-compiler-util.h
index cc34b49f5..1e2fbb318 100644
--- a/source/core/slang-visual-studio-compiler-util.h
+++ b/source/core/slang-visual-studio-compiler-util.h
@@ -6,25 +6,35 @@
 namespace Slang
 {
 
-struct VisualStudioCompilerUtil
-{
-    typedef CPPCompiler::CompileOptions CompileOptions;
-    typedef CPPCompiler::OptimizationLevel OptimizationLevel;
-    typedef CPPCompiler::TargetType TargetType;
-    typedef CPPCompiler::DebugInfoType DebugInfoType;
-    typedef CPPCompiler::SourceType SourceType;
-    typedef CPPCompiler::OutputMessage OutputMessage;
-    typedef CPPCompiler::FloatingPointMode FloatingPointMode;
 
+struct VisualStudioCompilerUtil : public CPPCompilerBaseUtil
+{
         /// Calculate Visual Studio family compilers cmdLine arguments from options
-    static void calcArgs(const CompileOptions& options, CommandLine& cmdLine);
+    static SlangResult calcArgs(const CompileOptions& options, CommandLine& cmdLine);
         /// Parse Visual Studio exeRes into CPPCompiler::Output
     static SlangResult parseOutput(const ExecuteResult& exeRes, CPPCompiler::Output& outOutput);
 
     static SlangResult calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath);
 
+    static SlangResult calcCompileProducts(const CompileOptions& options, ProductFlags flags, List<String>& outPaths);
+};
+
+class VisualStudioCPPCompiler : public CommandLineCPPCompiler
+{
+public:
+    typedef CommandLineCPPCompiler Super;
+    typedef VisualStudioCompilerUtil Util;
+
+    // CommandLineCPPCompiler impl  - just forwards to the Util
+    virtual SlangResult calcArgs(const CompileOptions& options, CommandLine& cmdLine) SLANG_OVERRIDE { return Util::calcArgs(options, cmdLine); }
+    virtual SlangResult parseOutput(const ExecuteResult& exeResult, Output& output) SLANG_OVERRIDE { return Util::parseOutput(exeResult, output); }
+    virtual SlangResult calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath) SLANG_OVERRIDE { return Util::calcModuleFilePath(options, outPath); }
+    virtual SlangResult calcCompileProducts(const CompileOptions& options, ProductFlags productFlags, List<String>& outPaths) SLANG_OVERRIDE { return Util::calcCompileProducts(options, productFlags, outPaths); }
+
+    VisualStudioCPPCompiler(const Desc& desc):Super(desc) {}
 };
 
+
 }
 
 #endif
diff --git a/source/core/windows/slang-win-visual-studio-util.cpp b/source/core/windows/slang-win-visual-studio-util.cpp
index a406e168f..260f3f56f 100644
--- a/source/core/windows/slang-win-visual-studio-util.cpp
+++ b/source/core/windows/slang-win-visual-studio-util.cpp
@@ -290,10 +290,8 @@ static SlangResult _find(int versionIndex, WinVisualStudioUtil::VersionPath& out
         VersionPath versionPath;
         if (!set->getCompiler(desc) && SLANG_SUCCEEDED(_find(i, versionPath)))
         {
-            CommandLine cmdLine;
-            calcExecuteCompilerArgs(versionPath, cmdLine);
-            
-            RefPtr<GenericCPPCompiler> compiler = new GenericCPPCompiler(desc, cmdLine, &VisualStudioCompilerUtil::calcArgs, &VisualStudioCompilerUtil::parseOutput, &VisualStudioCompilerUtil::calcModuleFilePath);
+            RefPtr<CommandLineCPPCompiler> compiler = new VisualStudioCPPCompiler(desc);
+            calcExecuteCompilerArgs(versionPath, compiler->m_cmdLine);
             set->addCompiler(compiler);
         }
     }