Callable CPU code support (#1014)

* First pass support for compiling to a loaded shared library.

* Improve documentation for cpu target.

* Removed the SLANG_COMPILE_FLAG_LOAD_SHARED_LIBRARY flag.
Use the SLANG_HOST_CALLABLE code target
Document mechanism.

* Fix typo in cpp-resource.slang
In test code if the target is 'callable' we don't need to compile (indeed there is no source file).

* Small refactor using CommandLineCPPCompiler as base class to implement VisualStudioCPPCompiler and GCCCPPCompiler.

* Improvements around CPPCompiler.
Mechanism to know products produced.
Cleaning up products after execution.

* Fix multiple definition of 'SourceType'

16 files changed, 387 insertions, 103 deletions

diff --git a/source/core/slang-cpp-compiler.cpp b/source/core/slang-cpp-compiler.cpp
index 54138ab27..7ac4e96b4 100644
--- a/source/core/slang-cpp-compiler.cpp
+++ b/source/core/slang-cpp-compiler.cpp
@@ -160,9 +160,9 @@ void CPPCompiler::Output::removeByType(OutputMessage::Type type)
     }
 }
 
-/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! GenericCPPCompiler !!!!!!!!!!!!!!!!!!!!!!*/
+/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! CommandLineCPPCompiler !!!!!!!!!!!!!!!!!!!!!!*/
 
-SlangResult GenericCPPCompiler::compile(const CompileOptions& options, Output& outOutput)
+SlangResult CommandLineCPPCompiler::compile(const CompileOptions& options, Output& outOutput)
 {
     outOutput.reset();
 
@@ -170,7 +170,7 @@ SlangResult GenericCPPCompiler::compile(const CompileOptions& options, Output& o
     CommandLine cmdLine(m_cmdLine);
 
     // Append command line args to the end of cmdLine using the target specific function for the specified options
-    m_calcArgsFunc(options, cmdLine);
+    SLANG_RETURN_ON_FAIL(calcArgs(options, cmdLine));
 
     ExecuteResult exeRes;
 
@@ -190,12 +190,7 @@ SlangResult GenericCPPCompiler::compile(const CompileOptions& options, Output& o
     }
 #endif
 
-    return m_parseOutputFunc(exeRes, outOutput);
-}
-
-SlangResult GenericCPPCompiler::calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath)
-{
-    return m_calcModuleFilePathFunc(options, outPath);
+    return parseOutput(exeRes, outOutput);
 }
 
 /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! CPPCompilerUtil !!!!!!!!!!!!!!!!!!!!!!*/
@@ -339,7 +334,8 @@ static void _addGCCFamilyCompiler(const String& exeName, CPPCompilerSet* compile
     CPPCompiler::Desc desc;
     if (SLANG_SUCCEEDED(GCCCompilerUtil::calcVersion(exeName, desc)))
     {
-        RefPtr<CPPCompiler> compiler(new GenericCPPCompiler(desc, exeName, &GCCCompilerUtil::calcArgs, &GCCCompilerUtil::parseOutput, GCCCompilerUtil::calcModuleFilePath));
+        RefPtr<CommandLineCPPCompiler> compiler(new GCCCPPCompiler(desc));
+        compiler->m_cmdLine.setExecutableFilename(exeName);
         compilerSet->addCompiler(compiler);
     }
 }
diff --git a/source/core/slang-cpp-compiler.h b/source/core/slang-cpp-compiler.h
index 851975d72..185e960f3 100644
--- a/source/core/slang-cpp-compiler.h
+++ b/source/core/slang-cpp-compiler.h
@@ -147,6 +147,27 @@ public:
         Int fileLine;                   ///< The line number the error came from
     };
 
+    typedef uint32_t ProductFlags;
+    struct ProductFlag
+    {
+        enum Enum : ProductFlags
+        {
+            Debug               = 0x1,                ///< Used by debugger during execution
+            Execution           = 0x2,                ///< Required for execution
+            Compile             = 0x4,                ///< A product *required* for compilation
+            Miscellaneous       = 0x8,                ///< Anything else
+            All                 = 0xf,                ///< All the flags
+        };
+    };
+
+    enum class Product
+    {
+        DebugRun,
+        Run,
+        CompileTemporary,
+        All,
+    };
+
     struct Output
     {
             /// Reset to an initial empty state
@@ -178,6 +199,10 @@ public:
     virtual SlangResult compile(const CompileOptions& options, Output& outOutput) = 0;
         /// Given the compilation options and the module name, determines the actual file name used for output
     virtual SlangResult calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath) = 0;
+        /// Given options determines the paths to products produced (including the 'moduleFilePath').
+        /// Note that does *not* guarentee all products were or should be produced. Just aims to include all that could
+        /// be produced, such that can be removed on completion.
+    virtual SlangResult calcCompileProducts(const CompileOptions& options, ProductFlags flags, List<String>& outPaths) = 0;
 
         /// Return the compiler type as name
     static UnownedStringSlice getCompilerTypeAsText(CompilerType type);
@@ -191,38 +216,31 @@ protected:
     Desc m_desc;
 };
 
-class GenericCPPCompiler : public CPPCompiler
+class CommandLineCPPCompiler : public CPPCompiler
 {
 public:
     typedef CPPCompiler Super;
 
-    typedef void(*CalcArgsFunc)(const CPPCompiler::CompileOptions& options, CommandLine& cmdLine);
-    typedef SlangResult(*ParseOutputFunc)(const ExecuteResult& exeResult, Output& output);
-    typedef SlangResult(*CalcModuleFilePathFunc)(const CPPCompiler::CompileOptions& options, StringBuilder& outPath);
-
+    // CPPCompiler
     virtual SlangResult compile(const CompileOptions& options, Output& outOutput) SLANG_OVERRIDE;
-    virtual SlangResult calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath) SLANG_OVERRIDE;
+    
+    // Functions to be implemented for a specific CommandLine    
+    virtual SlangResult calcArgs(const CompileOptions& options, CommandLine& cmdLine) = 0;
+    virtual SlangResult parseOutput(const ExecuteResult& exeResult, Output& output) = 0;
 
-    GenericCPPCompiler(const Desc& desc, const String& exeName, CalcArgsFunc calcArgsFunc, ParseOutputFunc parseOutputFunc, CalcModuleFilePathFunc calcModuleFilePathFunc) :
-        Super(desc),
-        m_calcArgsFunc(calcArgsFunc),
-        m_parseOutputFunc(parseOutputFunc),
-        m_calcModuleFilePathFunc(calcModuleFilePathFunc)
+    CommandLineCPPCompiler(const Desc& desc, const String& exeName) :
+        Super(desc)
     {
         m_cmdLine.setExecutableFilename(exeName);
     }
 
-    GenericCPPCompiler(const Desc& desc, const CommandLine& cmdLine, CalcArgsFunc calcArgsFunc, ParseOutputFunc parseOutputFunc, CalcModuleFilePathFunc calcModuleFilePathFunc) :
+    CommandLineCPPCompiler(const Desc& desc, const CommandLine& cmdLine) :
         Super(desc),
-        m_cmdLine(cmdLine),
-        m_calcArgsFunc(calcArgsFunc),
-        m_parseOutputFunc(parseOutputFunc),
-        m_calcModuleFilePathFunc(calcModuleFilePathFunc)
+        m_cmdLine(cmdLine)
     {}
 
-    CalcArgsFunc m_calcArgsFunc;
-    ParseOutputFunc m_parseOutputFunc;
-    CalcModuleFilePathFunc m_calcModuleFilePathFunc;
+    CommandLineCPPCompiler(const Desc& desc):Super(desc) {}
+
     CommandLine m_cmdLine;
 };
 
@@ -261,7 +279,8 @@ protected:
     List<RefPtr<CPPCompiler>> m_compilers;
 };
 
-struct CPPCompilerUtil
+/* Only purpose of having base-class here is to make all the CPPCompiler types available directly in derived Utils */
+struct CPPCompilerBaseUtil
 {
     typedef CPPCompiler::CompileOptions CompileOptions;
     typedef CPPCompiler::OptimizationLevel OptimizationLevel;
@@ -269,6 +288,14 @@ struct CPPCompilerUtil
     typedef CPPCompiler::DebugInfoType DebugInfoType;
     typedef CPPCompiler::SourceType SourceType;
 
+    typedef CPPCompiler::OutputMessage OutputMessage;
+    typedef CPPCompiler::FloatingPointMode FloatingPointMode;
+    typedef CPPCompiler::ProductFlag ProductFlag;
+    typedef CPPCompiler::ProductFlags ProductFlags;
+};
+
+struct CPPCompilerUtil: public CPPCompilerBaseUtil
+{
     enum class MatchType
     {
         MinGreaterEqual,
@@ -289,12 +316,8 @@ struct CPPCompilerUtil
 
         /// Given a set, registers compilers found through standard means and determines a reasonable default compiler if possible
     static SlangResult initializeSet(CPPCompilerSet* set);
-
-    
-    
 };
 
-
 }
 
 #endif
diff --git a/source/core/slang-gcc-compiler-util.cpp b/source/core/slang-gcc-compiler-util.cpp
index 0a53bcac8..d42076039 100644
--- a/source/core/slang-gcc-compiler-util.cpp
+++ b/source/core/slang-gcc-compiler-util.cpp
@@ -354,8 +354,12 @@ static SlangResult _parseGCCFamilyLine(const UnownedStringSlice& line, LineParse
         }
         case TargetType::Object:
         {
+#if __CYGWIN__
+            outPath << options.modulePath << ".obj";
+#else
             // Will be .o for typical gcc targets
             outPath << options.modulePath << ".o";
+#endif
             return SLANG_OK;
         }
     }
@@ -363,7 +367,21 @@ static SlangResult _parseGCCFamilyLine(const UnownedStringSlice& line, LineParse
     return SLANG_FAIL;
 }
 
-/* static */void GCCCompilerUtil::calcArgs(const CompileOptions& options, CommandLine& cmdLine)
+/* static */SlangResult GCCCompilerUtil::calcCompileProducts(const CompileOptions& options, ProductFlags flags, List<String>& outPaths)
+{
+    outPaths.clear();
+
+    if (flags & ProductFlag::Execution)
+    {
+        StringBuilder builder;
+        SLANG_RETURN_ON_FAIL(calcModuleFilePath(options, builder));
+        outPaths.add(builder);
+    }
+
+    return SLANG_OK;
+}
+
+/* static */SlangResult GCCCompilerUtil::calcArgs(const CompileOptions& options, CommandLine& cmdLine)
 {
     cmdLine.addArg("-fvisibility=hidden");
 
@@ -513,6 +531,8 @@ static SlangResult _parseGCCFamilyLine(const UnownedStringSlice& line, LineParse
 	// Make maths lib available
         cmdLine.addArg("-lm");
     }
+
+    return SLANG_OK;
 }
 
 }
diff --git a/source/core/slang-gcc-compiler-util.h b/source/core/slang-gcc-compiler-util.h
index a72080acc..df3cb8a4b 100644
--- a/source/core/slang-gcc-compiler-util.h
+++ b/source/core/slang-gcc-compiler-util.h
@@ -7,15 +7,8 @@ namespace Slang
 {
 
 /* Utility for processing input and output of gcc-like compilers, including clang */
-struct GCCCompilerUtil
+struct GCCCompilerUtil : public CPPCompilerBaseUtil
 {
-    typedef CPPCompiler::CompileOptions CompileOptions;
-    typedef CPPCompiler::OptimizationLevel OptimizationLevel;
-    typedef CPPCompiler::TargetType TargetType;
-    typedef CPPCompiler::DebugInfoType DebugInfoType;
-    typedef CPPCompiler::SourceType SourceType;
-    typedef CPPCompiler::FloatingPointMode FloatingPointMode;
-
         /// Extracts version number into desc from text (assumes gcc/clang -v layout with a line with version)
     static SlangResult parseVersion(const UnownedStringSlice& text, const UnownedStringSlice& prefix, CPPCompiler::Desc& outDesc);
 
@@ -23,7 +16,7 @@ struct GCCCompilerUtil
     static SlangResult calcVersion(const String& exeName, CPPCompiler::Desc& outDesc);
 
         /// Calculate gcc family compilers (including clang) cmdLine arguments from options
-    static void calcArgs(const CompileOptions& options, CommandLine& cmdLine);
+    static SlangResult calcArgs(const CompileOptions& options, CommandLine& cmdLine);
 
         /// Parse ExecuteResult into Output
     static SlangResult parseOutput(const ExecuteResult& exeRes, CPPCompiler::Output& outOutput);
@@ -31,6 +24,23 @@ struct GCCCompilerUtil
         /// Calculate the output module filename 
     static SlangResult calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath);
 
+        /// Given options, calculate paths to products produced for a compilation
+    static SlangResult calcCompileProducts(const CompileOptions& options, ProductFlags flags, List<String>& outPaths);
+};
+
+class GCCCPPCompiler : public CommandLineCPPCompiler
+{
+public:
+    typedef CommandLineCPPCompiler Super;
+    typedef GCCCompilerUtil Util;
+
+    // CommandLineCPPCompiler impl  - just forwards to the Util
+    virtual SlangResult calcArgs(const CompileOptions& options, CommandLine& cmdLine) SLANG_OVERRIDE { return Util::calcArgs(options, cmdLine); }
+    virtual SlangResult parseOutput(const ExecuteResult& exeResult, Output& output) SLANG_OVERRIDE { return Util::parseOutput(exeResult, output); }
+    virtual SlangResult calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath) SLANG_OVERRIDE { return Util::calcModuleFilePath(options, outPath); }
+    virtual SlangResult calcCompileProducts(const CompileOptions& options, ProductFlags flags,  List<String>& outPaths) SLANG_OVERRIDE { return Util::calcCompileProducts(options, flags, outPaths); }
+
+    GCCCPPCompiler(const Desc& desc):Super(desc) {}
 };
 
 }
diff --git a/source/core/slang-io.h b/source/core/slang-io.h
index e2935afe4..758a05edb 100644
--- a/source/core/slang-io.h
+++ b/source/core/slang-io.h
@@ -93,6 +93,14 @@ namespace Slang
     // Helper class to clean up temporary files on dtor
     struct TemporaryFileSet
     {
+        void remove(const String& path)
+        {
+            if (const Index index = m_paths.indexOf(path) >= 0)
+            {
+                m_paths.removeAt(index);
+            }
+        }
+
         void add(const String& path)
         {
             if (m_paths.indexOf(path) < 0)
@@ -100,6 +108,17 @@ namespace Slang
                 m_paths.add(path);
             }
         }
+        void add(const List<String>& paths)
+        {
+            for (const auto& path : paths)
+            {
+                add(path);
+            }
+        }
+        void clear()
+        {
+            m_paths.clear();
+        }
         ~TemporaryFileSet()
         {
             for (const auto& path : m_paths)
diff --git a/source/core/slang-shared-library.cpp b/source/core/slang-shared-library.cpp
index 9932db0b5..fd61ba0a0 100644
--- a/source/core/slang-shared-library.cpp
+++ b/source/core/slang-shared-library.cpp
@@ -56,6 +56,18 @@ SlangResult DefaultSharedLibraryLoader::loadSharedLibrary(const char* path, ISla
 
 /* !!!!!!!!!!!!!!!!!!!!!!!!!! DefaultSharedLibrary !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!*/
 
+TemporarySharedLibrary::~TemporarySharedLibrary()
+{
+    if (m_sharedLibraryHandle)
+    {
+        // We have to unload if we want to be able to remove
+        SharedLibrary::unload(m_sharedLibraryHandle);
+        m_sharedLibraryHandle = nullptr;
+    }
+}
+
+/* !!!!!!!!!!!!!!!!!!!!!!!!!! DefaultSharedLibrary !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!*/
+
 ISlangUnknown* DefaultSharedLibrary::getInterface(const Guid& guid)
 {
     return (guid == IID_ISlangUnknown || guid == IID_ISlangSharedLibrary) ? static_cast<ISlangSharedLibrary*>(this) : nullptr;
@@ -63,7 +75,10 @@ ISlangUnknown* DefaultSharedLibrary::getInterface(const Guid& guid)
 
 DefaultSharedLibrary::~DefaultSharedLibrary()
 {
-    SharedLibrary::unload(m_sharedLibraryHandle);
+    if (m_sharedLibraryHandle)
+    {
+        SharedLibrary::unload(m_sharedLibraryHandle);
+    }
 }
 
 SlangFuncPtr DefaultSharedLibrary::findFuncByName(char const* name)
diff --git a/source/core/slang-shared-library.h b/source/core/slang-shared-library.h
index 5a4eb7229..e48162ebd 100644
--- a/source/core/slang-shared-library.h
+++ b/source/core/slang-shared-library.h
@@ -5,6 +5,7 @@
 #include "../../slang-com-helper.h"
 #include "../../slang-com-ptr.h"
 
+#include "../core/slang-io.h"
 #include "../core/slang-platform.h"
 #include "../core/slang-common.h"
 #include "../core/slang-dictionary.h"
@@ -84,6 +85,30 @@ class DefaultSharedLibrary : public ISlangSharedLibrary, public RefObject
     SharedLibrary::Handle m_sharedLibraryHandle = nullptr;
 };
 
+class TemporarySharedLibrary : public DefaultSharedLibrary
+{
+public:
+    typedef DefaultSharedLibrary Super;
+
+        /// Get the path to the shared library
+    const String& getPath() const { return m_path; }
+
+        /// Ctor
+    TemporarySharedLibrary(const SharedLibrary::Handle sharedLibraryHandle, const String& path):
+        Super(sharedLibraryHandle),
+        m_path(path)
+    {
+    }
+
+    virtual ~TemporarySharedLibrary();
+
+        /// Any files specified in this set will be deleted on exit
+    TemporaryFileSet m_temporaryFileSet;
+
+protected:
+    String m_path;
+};
+
 class ConfigurableSharedLibraryLoader: public ISlangSharedLibraryLoader, public RefObject
 {
 public:
diff --git a/source/core/slang-visual-studio-compiler-util.cpp b/source/core/slang-visual-studio-compiler-util.cpp
index 765edc349..1c79f9004 100644
--- a/source/core/slang-visual-studio-compiler-util.cpp
+++ b/source/core/slang-visual-studio-compiler-util.cpp
@@ -37,7 +37,40 @@ namespace Slang
     return SLANG_FAIL;
 }
 
-/* static */void VisualStudioCompilerUtil::calcArgs(const CompileOptions& options, CommandLine& cmdLine)
+/* static */SlangResult VisualStudioCompilerUtil::calcCompileProducts(const CompileOptions& options, ProductFlags flags, List<String>& outPaths)
+{
+    outPaths.clear();
+
+    if (flags & ProductFlag::Execution)
+    {
+        StringBuilder builder;
+        SLANG_RETURN_ON_FAIL(calcModuleFilePath(options, builder));
+        outPaths.add(builder);
+    }
+    if (flags & ProductFlag::Miscellaneous)
+    {
+        outPaths.add(options.modulePath + ".ilk");
+
+        if (options.targetType == TargetType::SharedLibrary)
+        {
+            outPaths.add(options.modulePath + ".exp");
+            outPaths.add(options.modulePath + ".lib");
+        }
+    }
+    if (flags & ProductFlag::Compile)
+    {
+        outPaths.add(options.modulePath + ".obj");
+    }
+    if (flags & ProductFlag::Debug)
+    {
+        // TODO(JS): Could try and determine based on debug information
+        outPaths.add(options.modulePath + ".pdb");
+    }
+
+    return SLANG_OK;
+}
+
+/* static */SlangResult VisualStudioCompilerUtil::calcArgs(const CompileOptions& options, CommandLine& cmdLine)
 {
     // https://docs.microsoft.com/en-us/cpp/build/reference/compiler-options-listed-alphabetically?view=vs-2019
 
@@ -190,6 +223,8 @@ namespace Slang
         // Note that any escaping of the path is handled in the ProcessUtil::
         cmdLine.addPrefixPathArg("/LIBPATH:", libPath);
     }
+
+    return SLANG_OK;
 }
 
 static SlangResult _parseErrorType(const UnownedStringSlice& in, CPPCompiler::OutputMessage::Type& outType)
diff --git a/source/core/slang-visual-studio-compiler-util.h b/source/core/slang-visual-studio-compiler-util.h
index cc34b49f5..1e2fbb318 100644
--- a/source/core/slang-visual-studio-compiler-util.h
+++ b/source/core/slang-visual-studio-compiler-util.h
@@ -6,25 +6,35 @@
 namespace Slang
 {
 
-struct VisualStudioCompilerUtil
-{
-    typedef CPPCompiler::CompileOptions CompileOptions;
-    typedef CPPCompiler::OptimizationLevel OptimizationLevel;
-    typedef CPPCompiler::TargetType TargetType;
-    typedef CPPCompiler::DebugInfoType DebugInfoType;
-    typedef CPPCompiler::SourceType SourceType;
-    typedef CPPCompiler::OutputMessage OutputMessage;
-    typedef CPPCompiler::FloatingPointMode FloatingPointMode;
 
+struct VisualStudioCompilerUtil : public CPPCompilerBaseUtil
+{
         /// Calculate Visual Studio family compilers cmdLine arguments from options
-    static void calcArgs(const CompileOptions& options, CommandLine& cmdLine);
+    static SlangResult calcArgs(const CompileOptions& options, CommandLine& cmdLine);
         /// Parse Visual Studio exeRes into CPPCompiler::Output
     static SlangResult parseOutput(const ExecuteResult& exeRes, CPPCompiler::Output& outOutput);
 
     static SlangResult calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath);
 
+    static SlangResult calcCompileProducts(const CompileOptions& options, ProductFlags flags, List<String>& outPaths);
+};
+
+class VisualStudioCPPCompiler : public CommandLineCPPCompiler
+{
+public:
+    typedef CommandLineCPPCompiler Super;
+    typedef VisualStudioCompilerUtil Util;
+
+    // CommandLineCPPCompiler impl  - just forwards to the Util
+    virtual SlangResult calcArgs(const CompileOptions& options, CommandLine& cmdLine) SLANG_OVERRIDE { return Util::calcArgs(options, cmdLine); }
+    virtual SlangResult parseOutput(const ExecuteResult& exeResult, Output& output) SLANG_OVERRIDE { return Util::parseOutput(exeResult, output); }
+    virtual SlangResult calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath) SLANG_OVERRIDE { return Util::calcModuleFilePath(options, outPath); }
+    virtual SlangResult calcCompileProducts(const CompileOptions& options, ProductFlags productFlags, List<String>& outPaths) SLANG_OVERRIDE { return Util::calcCompileProducts(options, productFlags, outPaths); }
+
+    VisualStudioCPPCompiler(const Desc& desc):Super(desc) {}
 };
 
+
 }
 
 #endif
diff --git a/source/core/windows/slang-win-visual-studio-util.cpp b/source/core/windows/slang-win-visual-studio-util.cpp
index a406e168f..260f3f56f 100644
--- a/source/core/windows/slang-win-visual-studio-util.cpp
+++ b/source/core/windows/slang-win-visual-studio-util.cpp
@@ -290,10 +290,8 @@ static SlangResult _find(int versionIndex, WinVisualStudioUtil::VersionPath& out
         VersionPath versionPath;
         if (!set->getCompiler(desc) && SLANG_SUCCEEDED(_find(i, versionPath)))
         {
-            CommandLine cmdLine;
-            calcExecuteCompilerArgs(versionPath, cmdLine);
-            
-            RefPtr<GenericCPPCompiler> compiler = new GenericCPPCompiler(desc, cmdLine, &VisualStudioCompilerUtil::calcArgs, &VisualStudioCompilerUtil::parseOutput, &VisualStudioCompilerUtil::calcModuleFilePath);
+            RefPtr<CommandLineCPPCompiler> compiler = new VisualStudioCPPCompiler(desc);
+            calcExecuteCompilerArgs(versionPath, compiler->m_cmdLine);
             set->addCompiler(compiler);
         }
     }
diff --git a/source/slang/slang-compiler.cpp b/source/slang/slang-compiler.cpp
index 1c0bed065..8ead63784 100644
--- a/source/slang/slang-compiler.cpp
+++ b/source/slang/slang-compiler.cpp
@@ -87,7 +87,8 @@ namespace Slang
     x("c", CSource) \
     x("cpp", CPPSource) \
     x("exe,executable", Executable) \
-    x("sharedlib,sharedlibrary,dll", SharedLibrary) 
+    x("sharedlib,sharedlibrary,dll", SharedLibrary) \
+    x("callable,host-callable", HostCallable)
 
 #define SLANG_CODE_GEN_INFO(names, e) \
     { CodeGenTarget::e, UnownedStringSlice::fromLiteral(names) },
@@ -149,28 +150,64 @@ namespace Slang
         {
             outputBinary.addRange(result.outputBinary.getBuffer(), result.outputBinary.getCount());
         }
+        else if (appendTo == ResultFormat::SharedLibrary)
+        {
+            SLANG_ASSERT(!"Trying to append to a shared library");
+        }
+    }
+
+    SlangResult CompileResult::getSharedLibrary(ComPtr<ISlangSharedLibrary>& outSharedLibrary)
+    {
+        if (format == ResultFormat::SharedLibrary && sharedLibrary)
+        {
+            outSharedLibrary = sharedLibrary;
+            return SLANG_OK;
+        }
+        return SLANG_FAIL;
     }
 
-    ComPtr<ISlangBlob> CompileResult::getBlob()
+    SlangResult CompileResult::getBlob(ComPtr<ISlangBlob>& outBlob)
     {
         if(!blob)
         {
             switch(format)
             {
-            case ResultFormat::None:
-            default:
-                break;
+                case ResultFormat::None:
+                default:
+                    break;
 
-            case ResultFormat::Text:
-                blob = StringUtil::createStringBlob(outputString);
-                break;
+                case ResultFormat::Text:
+                    blob = StringUtil::createStringBlob(outputString);
+                    break;
 
-            case ResultFormat::Binary:
-                blob = createRawBlob(outputBinary.getBuffer(), outputBinary.getCount());
-                break;
+                case ResultFormat::Binary:
+                    blob = createRawBlob(outputBinary.getBuffer(), outputBinary.getCount());
+                    break;
+                case ResultFormat::SharedLibrary:
+                {
+                    // TODO(JS): Could do something more sophisticated to check this cast is safe (like have an interface to get the path), but this
+                    // is simple and works with current impl
+                    TemporarySharedLibrary* tempLibrary = static_cast<TemporarySharedLibrary*>(sharedLibrary.get());
+
+                    SLANG_ASSERT(sharedLibrary);
+                    List<uint8_t> contents;
+                    try
+                    {
+                        // We can read it from the shared library...
+                        contents = File::readAllBytes(tempLibrary->getPath());
+                    }
+                    catch (const Slang::IOException&)
+                    {
+                        return SLANG_E_CANNOT_OPEN; 
+                    }
+                    blob = createRawBlob(contents.getBuffer(), contents.getCount());
+                    break;
+                }
             }
         }
-        return blob;
+
+        outBlob = blob;
+        return SLANG_OK;
     }
 
     //
@@ -478,6 +515,7 @@ namespace Slang
                 // Don't need an external compiler to output C and C++ code
                 return PassThroughMode::None;
             }
+            case CodeGenTarget::HostCallable:
             case CodeGenTarget::SharedLibrary:
             case CodeGenTarget::Executable:
             {
@@ -1103,13 +1141,15 @@ SlangResult dissassembleDXILUsingDXC(
         Int                     entryPointIndex,
         TargetRequest*          targetReq,
         EndToEndCompileRequest* endToEndReq,
-        List<uint8_t>&          binOut)
+        ComPtr<ISlangSharedLibrary>& outSharedLib,
+        List<uint8_t>&          outBin)
     {
         auto sink = slangRequest->getSink();
 
         const String originalSourcePath = calcSourcePathForEntryPoint(endToEndReq, entryPointIndex);
 
-        binOut.clear();
+        outBin.clear();
+        outSharedLib.setNull();
       
         CPPCompilerSet* compilerSet = slangRequest->getSession()->requireCPPCompilerSet();
 
@@ -1311,6 +1351,7 @@ SlangResult dissassembleDXILUsingDXC(
         // Set what kind of target we should build
         switch (targetReq->target)
         {
+            case CodeGenTarget::HostCallable:
             case CodeGenTarget::SharedLibrary:
             {
                 options.targetType = CPPCompiler::TargetType::SharedLibrary;
@@ -1332,8 +1373,13 @@ SlangResult dissassembleDXILUsingDXC(
             moduleFilePath = builder.ProduceString();
         }
 
-        // Add so it's deleted at the end
-        temporaryFileSet.add(moduleFilePath);
+        // Find all the files that will be produced
+        TemporaryFileSet productFileSet;
+        {
+            List<String> paths;
+            SLANG_RETURN_ON_FAIL(compiler->calcCompileProducts(options, CPPCompiler::ProductFlag::All, paths));
+            productFileSet.add(paths);
+        }
 
         // Need to configure for the compilation
 
@@ -1457,16 +1503,48 @@ SlangResult dissassembleDXILUsingDXC(
             return SLANG_FAIL;
         }
 
-        // Read the binary
-        try
+        // If callable we need to load the shared library
+        if (targetReq->target == CodeGenTarget::HostCallable)
         {
-            // Read the contents of the binary
-            binOut = File::readAllBytes(moduleFilePath);
+            // Try loading the shared library
+            SharedLibrary::Handle handle;
+            if (SLANG_FAILED(SharedLibrary::loadWithPlatformPath(moduleFilePath.getBuffer(), handle)))
+            {
+                sink->diagnose(SourceLoc(), Diagnostics::unableToReadFile, moduleFilePath);
+                return SLANG_FAIL;
+            }
+            RefPtr<TemporarySharedLibrary> sharedLib(new TemporarySharedLibrary(handle, moduleFilePath));
+            sharedLib->m_temporaryFileSet = productFileSet;
+            productFileSet.clear();
         }
-        catch (const Slang::IOException&)
+        else
         {
-            sink->diagnose(SourceLoc(), Diagnostics::unableToReadFile, moduleFilePath);
-            return SLANG_FAIL;
+            // Read the binary
+            try
+            {
+                // TODO(JS): We have a problem here.. productFileSet will clear up all temporaries
+                // and although we return the binary here (through outBin), we don't return debug info
+                // which is separate (say with a pdb). To work around this we reevaluate productFileSet,
+                // so we don't include debug info. The executable will presumably be reconstructed from
+                // outBin
+                // The problem is that these files have no specific lifetime (unlike with HostCallable).
+
+                CPPCompiler::ProductFlags flags = CPPCompiler::ProductFlag::All;
+                flags &= ~CPPCompiler::ProductFlag::Debug;
+
+                List<String> paths;
+                SLANG_RETURN_ON_FAIL(compiler->calcCompileProducts(options, flags, paths));
+                productFileSet.clear();
+                productFileSet.add(paths);
+
+                // Read the contents of the binary
+                outBin = File::readAllBytes(moduleFilePath);
+            }
+            catch (const Slang::IOException&)
+            {
+                sink->diagnose(SourceLoc(), Diagnostics::unableToReadFile, moduleFilePath);
+                return SLANG_FAIL;
+            }
         }
 
         return SLANG_OK;
@@ -1550,9 +1628,12 @@ SlangResult dissassembleDXILUsingDXC(
 
         switch (target)
         {
+        case CodeGenTarget::HostCallable:
         case CodeGenTarget::SharedLibrary:
         case CodeGenTarget::Executable:
             {
+                ComPtr<ISlangSharedLibrary> sharedLibrary;
+
                 List<uint8_t> code;
                 if (SLANG_SUCCEEDED(emitCPUBinaryForEntryPoint(
                     compileRequest,
@@ -1560,10 +1641,18 @@ SlangResult dissassembleDXILUsingDXC(
                     entryPointIndex,
                     targetReq,
                     endToEndReq,
+                    sharedLibrary,
                     code)))
                 {
-                    maybeDumpIntermediate(compileRequest, code.getBuffer(), code.getCount(), target);
-                    result = CompileResult(code);
+                    if (target == CodeGenTarget::HostCallable)
+                    {
+                        result = CompileResult(sharedLibrary);
+                    }
+                    else
+                    {
+                        maybeDumpIntermediate(compileRequest, code.getBuffer(), code.getCount(), target);
+                        result = CompileResult(code);
+                    }
                 }
             }
             break;
@@ -1860,6 +1949,9 @@ SlangResult dissassembleDXILUsingDXC(
             writeOutputToConsole(writer, result.outputString);
             break;
 
+        case ResultFormat::SharedLibrary:
+            break;
+
         case ResultFormat::Binary:
             {
                 auto& data = result.outputBinary;
@@ -2232,6 +2324,7 @@ SlangResult dissassembleDXILUsingDXC(
             // for now
             dumpIntermediateBinary(compileRequest, data, size, ".exe");
             break;
+        case CodeGenTarget::HostCallable:
         case CodeGenTarget::SharedLibrary:
             dumpIntermediateBinary(compileRequest, data, size, ".shared-lib");
             break;
diff --git a/source/slang/slang-compiler.h b/source/slang/slang-compiler.h
index 8f9c20044..bbc6505b0 100644
--- a/source/slang/slang-compiler.h
+++ b/source/slang/slang-compiler.h
@@ -61,6 +61,7 @@ namespace Slang
         CPPSource           = SLANG_CPP_SOURCE,
         Executable          = SLANG_EXECUTABLE,
         SharedLibrary       = SLANG_SHARED_LIBRARY,
+        HostCallable        = SLANG_HOST_CALLABLE,
     };
 
     CodeGenTarget calcCodeGenTargetFromName(const UnownedStringSlice& name);
@@ -84,7 +85,8 @@ namespace Slang
     {
         None,
         Text,
-        Binary
+        Binary,
+        SharedLibrary,
     };
 
     // When storing the layout for a matrix-type
@@ -122,22 +124,25 @@ namespace Slang
     class TranslationUnitRequest;
 
     // Result of compiling an entry point.
-    // Should only ever be string OR binary.
+    // Should only ever be string, binary or shared library
     class CompileResult
     {
     public:
         CompileResult() = default;
         CompileResult(String const& str) : format(ResultFormat::Text), outputString(str) {}
         CompileResult(List<uint8_t> const& buffer) : format(ResultFormat::Binary), outputBinary(buffer) {}
+        CompileResult(ISlangSharedLibrary* inSharedLibrary) : format(ResultFormat::SharedLibrary), sharedLibrary(inSharedLibrary) {}
 
         void append(CompileResult const& result);
 
-        ComPtr<ISlangBlob> getBlob();
+        SlangResult getBlob(ComPtr<ISlangBlob>& outBlob);
+        SlangResult getSharedLibrary(ComPtr<ISlangSharedLibrary>& outSharedLibrary);
 
         ResultFormat format = ResultFormat::None;
         String outputString;
         List<uint8_t> outputBinary;
 
+        ComPtr<ISlangSharedLibrary> sharedLibrary;
         ComPtr<ISlangBlob> blob;
     };
 
diff --git a/source/slang/slang-ir-entry-point-uniforms.cpp b/source/slang/slang-ir-entry-point-uniforms.cpp
index da036d798..822174620 100644
--- a/source/slang/slang-ir-entry-point-uniforms.cpp
+++ b/source/slang/slang-ir-entry-point-uniforms.cpp
@@ -441,6 +441,7 @@ void moveEntryPointUniformParamsToGlobalScope(
         case CodeGenTarget::CSource:
         case CodeGenTarget::Executable:
         case CodeGenTarget::SharedLibrary:
+        case CodeGenTarget::HostCallable:
         {
             context.targetNeedsConstantBuffer = false;
             break;
diff --git a/source/slang/slang-parameter-binding.cpp b/source/slang/slang-parameter-binding.cpp
index f9657e776..6e5838da5 100644
--- a/source/slang/slang-parameter-binding.cpp
+++ b/source/slang/slang-parameter-binding.cpp
@@ -2640,6 +2640,7 @@ static bool _isCPUTarget(CodeGenTarget target)
         case CodeGenTarget::CSource:
         case CodeGenTarget::Executable:
         case CodeGenTarget::SharedLibrary:
+        case CodeGenTarget::HostCallable:
         {
             return true;
         }
diff --git a/source/slang/slang-type-layout.cpp b/source/slang/slang-type-layout.cpp
index 3b14b74c2..9551cfe4e 100644
--- a/source/slang/slang-type-layout.cpp
+++ b/source/slang/slang-type-layout.cpp
@@ -982,7 +982,7 @@ LayoutRulesFamilyImpl* getDefaultLayoutRulesFamilyForTarget(TargetRequest* targe
     case CodeGenTarget::SPIRVAssembly:
         return &kGLSLLayoutRulesFamilyImpl;
 
-
+    case CodeGenTarget::HostCallable:
     case CodeGenTarget::Executable:
     case CodeGenTarget::SharedLibrary:
     case CodeGenTarget::CPPSource:
diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp
index cae9855e0..bb7e705e6 100644
--- a/source/slang/slang.cpp
+++ b/source/slang/slang.cpp
@@ -1716,7 +1716,9 @@ SLANG_NO_THROW SlangResult SLANG_MCALL ComponentType::getEntryPointCode(
     if(entryPointResult.format == ResultFormat::None )
         return SLANG_FAIL;
 
-    *outCode = entryPointResult.getBlob().detach();
+    ComPtr<ISlangBlob> blob;
+    SLANG_RETURN_ON_FAIL(entryPointResult.getBlob(blob));
+    *outCode = blob.detach();
     return SLANG_OK;
 }
 
@@ -3140,29 +3142,28 @@ SLANG_API void const* spGetEntryPointCode(
     return data;
 }
 
-SLANG_API SlangResult spGetEntryPointCodeBlob(
-        SlangCompileRequest*    request,
-        int                     entryPointIndex,
-        int                     targetIndex,
-        ISlangBlob**            outBlob)
+static SlangResult _getEntryPointResult(
+    SlangCompileRequest*    request,
+    int                     entryPointIndex,
+    int                     targetIndex,
+    Slang::CompileResult**  outCompileResult)
 {
     using namespace Slang;
-    if(!request) return SLANG_ERROR_INVALID_PARAMETER;
-    if(!outBlob) return SLANG_ERROR_INVALID_PARAMETER;
-
+    if (!request) return SLANG_ERROR_INVALID_PARAMETER;
+    
     auto req = Slang::asInternal(request);
     auto linkage = req->getLinkage();
     auto program = req->getSpecializedGlobalAndEntryPointsComponentType();
 
     Index targetCount = linkage->targets.getCount();
-    if((targetIndex < 0) || (targetIndex >= targetCount))
+    if ((targetIndex < 0) || (targetIndex >= targetCount))
     {
         return SLANG_ERROR_INVALID_PARAMETER;
     }
     auto targetReq = linkage->targets[targetIndex];
 
     Index entryPointCount = req->entryPoints.getCount();
-    if((entryPointIndex < 0) || (entryPointIndex >= entryPointCount))
+    if ((entryPointIndex < 0) || (entryPointIndex >= entryPointCount))
     {
         return SLANG_ERROR_INVALID_PARAMETER;
     }
@@ -3170,15 +3171,47 @@ SLANG_API SlangResult spGetEntryPointCodeBlob(
 
 
     auto targetProgram = program->getTargetProgram(targetReq);
-    if(!targetProgram)
+    if (!targetProgram)
         return SLANG_FAIL;
-    Slang::CompileResult& result = targetProgram->getExistingEntryPointResult(entryPointIndex);
+    *outCompileResult = &targetProgram->getExistingEntryPointResult(entryPointIndex);
+    return SLANG_OK;
+}
+
+SLANG_API SlangResult spGetEntryPointCodeBlob(
+        SlangCompileRequest*    request,
+        int                     entryPointIndex,
+        int                     targetIndex,
+        ISlangBlob**            outBlob)
+{
+    using namespace Slang;
+    if(!outBlob) return SLANG_ERROR_INVALID_PARAMETER;
+    Slang::CompileResult* compileResult = nullptr;
+    SLANG_RETURN_ON_FAIL(_getEntryPointResult(request, entryPointIndex, targetIndex, &compileResult));
 
-    auto blob = result.getBlob();
+    ComPtr<ISlangBlob> blob;
+    SLANG_RETURN_ON_FAIL(compileResult->getBlob(blob));
     *outBlob = blob.detach();
     return SLANG_OK;
 }
 
+SLANG_API SlangResult spGetEntryPointHostCallable(
+    SlangCompileRequest*    request,
+    int                     entryPointIndex,
+    int                     targetIndex,
+    ISlangSharedLibrary**   outSharedLibrary)
+{
+    using namespace Slang;
+    if (!outSharedLibrary) return SLANG_ERROR_INVALID_PARAMETER;
+
+    Slang::CompileResult* compileResult = nullptr;
+    SLANG_RETURN_ON_FAIL(_getEntryPointResult(request, entryPointIndex, targetIndex, &compileResult));
+
+    ComPtr<ISlangSharedLibrary> sharedLibrary;
+    SLANG_RETURN_ON_FAIL(compileResult->getSharedLibrary(sharedLibrary));
+    *outSharedLibrary = sharedLibrary.detach();
+    return SLANG_OK;
+}
+
 SLANG_API char const* spGetEntryPointSource(
     SlangCompileRequest*    request,
     int                     entryPointIndex)