Callable CPU code support (#1014)

* First pass support for compiling to a loaded shared library.

* Improve documentation for cpu target.

* Removed the SLANG_COMPILE_FLAG_LOAD_SHARED_LIBRARY flag.
Use the SLANG_HOST_CALLABLE code target
Document mechanism.

* Fix typo in cpp-resource.slang
In test code if the target is 'callable' we don't need to compile (indeed there is no source file).

* Small refactor using CommandLineCPPCompiler as base class to implement VisualStudioCPPCompiler and GCCCPPCompiler.

* Improvements around CPPCompiler.
Mechanism to know products produced.
Cleaning up products after execution.

* Fix multiple definition of 'SourceType'

6 files changed, 176 insertions, 43 deletions

diff --git a/source/slang/slang-compiler.cpp b/source/slang/slang-compiler.cpp
index 1c0bed065..8ead63784 100644
--- a/source/slang/slang-compiler.cpp
+++ b/source/slang/slang-compiler.cpp
@@ -87,7 +87,8 @@ namespace Slang
     x("c", CSource) \
     x("cpp", CPPSource) \
     x("exe,executable", Executable) \
-    x("sharedlib,sharedlibrary,dll", SharedLibrary) 
+    x("sharedlib,sharedlibrary,dll", SharedLibrary) \
+    x("callable,host-callable", HostCallable)
 
 #define SLANG_CODE_GEN_INFO(names, e) \
     { CodeGenTarget::e, UnownedStringSlice::fromLiteral(names) },
@@ -149,28 +150,64 @@ namespace Slang
         {
             outputBinary.addRange(result.outputBinary.getBuffer(), result.outputBinary.getCount());
         }
+        else if (appendTo == ResultFormat::SharedLibrary)
+        {
+            SLANG_ASSERT(!"Trying to append to a shared library");
+        }
+    }
+
+    SlangResult CompileResult::getSharedLibrary(ComPtr<ISlangSharedLibrary>& outSharedLibrary)
+    {
+        if (format == ResultFormat::SharedLibrary && sharedLibrary)
+        {
+            outSharedLibrary = sharedLibrary;
+            return SLANG_OK;
+        }
+        return SLANG_FAIL;
     }
 
-    ComPtr<ISlangBlob> CompileResult::getBlob()
+    SlangResult CompileResult::getBlob(ComPtr<ISlangBlob>& outBlob)
     {
         if(!blob)
         {
             switch(format)
             {
-            case ResultFormat::None:
-            default:
-                break;
+                case ResultFormat::None:
+                default:
+                    break;
 
-            case ResultFormat::Text:
-                blob = StringUtil::createStringBlob(outputString);
-                break;
+                case ResultFormat::Text:
+                    blob = StringUtil::createStringBlob(outputString);
+                    break;
 
-            case ResultFormat::Binary:
-                blob = createRawBlob(outputBinary.getBuffer(), outputBinary.getCount());
-                break;
+                case ResultFormat::Binary:
+                    blob = createRawBlob(outputBinary.getBuffer(), outputBinary.getCount());
+                    break;
+                case ResultFormat::SharedLibrary:
+                {
+                    // TODO(JS): Could do something more sophisticated to check this cast is safe (like have an interface to get the path), but this
+                    // is simple and works with current impl
+                    TemporarySharedLibrary* tempLibrary = static_cast<TemporarySharedLibrary*>(sharedLibrary.get());
+
+                    SLANG_ASSERT(sharedLibrary);
+                    List<uint8_t> contents;
+                    try
+                    {
+                        // We can read it from the shared library...
+                        contents = File::readAllBytes(tempLibrary->getPath());
+                    }
+                    catch (const Slang::IOException&)
+                    {
+                        return SLANG_E_CANNOT_OPEN; 
+                    }
+                    blob = createRawBlob(contents.getBuffer(), contents.getCount());
+                    break;
+                }
             }
         }
-        return blob;
+
+        outBlob = blob;
+        return SLANG_OK;
     }
 
     //
@@ -478,6 +515,7 @@ namespace Slang
                 // Don't need an external compiler to output C and C++ code
                 return PassThroughMode::None;
             }
+            case CodeGenTarget::HostCallable:
             case CodeGenTarget::SharedLibrary:
             case CodeGenTarget::Executable:
             {
@@ -1103,13 +1141,15 @@ SlangResult dissassembleDXILUsingDXC(
         Int                     entryPointIndex,
         TargetRequest*          targetReq,
         EndToEndCompileRequest* endToEndReq,
-        List<uint8_t>&          binOut)
+        ComPtr<ISlangSharedLibrary>& outSharedLib,
+        List<uint8_t>&          outBin)
     {
         auto sink = slangRequest->getSink();
 
         const String originalSourcePath = calcSourcePathForEntryPoint(endToEndReq, entryPointIndex);
 
-        binOut.clear();
+        outBin.clear();
+        outSharedLib.setNull();
       
         CPPCompilerSet* compilerSet = slangRequest->getSession()->requireCPPCompilerSet();
 
@@ -1311,6 +1351,7 @@ SlangResult dissassembleDXILUsingDXC(
         // Set what kind of target we should build
         switch (targetReq->target)
         {
+            case CodeGenTarget::HostCallable:
             case CodeGenTarget::SharedLibrary:
             {
                 options.targetType = CPPCompiler::TargetType::SharedLibrary;
@@ -1332,8 +1373,13 @@ SlangResult dissassembleDXILUsingDXC(
             moduleFilePath = builder.ProduceString();
         }
 
-        // Add so it's deleted at the end
-        temporaryFileSet.add(moduleFilePath);
+        // Find all the files that will be produced
+        TemporaryFileSet productFileSet;
+        {
+            List<String> paths;
+            SLANG_RETURN_ON_FAIL(compiler->calcCompileProducts(options, CPPCompiler::ProductFlag::All, paths));
+            productFileSet.add(paths);
+        }
 
         // Need to configure for the compilation
 
@@ -1457,16 +1503,48 @@ SlangResult dissassembleDXILUsingDXC(
             return SLANG_FAIL;
         }
 
-        // Read the binary
-        try
+        // If callable we need to load the shared library
+        if (targetReq->target == CodeGenTarget::HostCallable)
         {
-            // Read the contents of the binary
-            binOut = File::readAllBytes(moduleFilePath);
+            // Try loading the shared library
+            SharedLibrary::Handle handle;
+            if (SLANG_FAILED(SharedLibrary::loadWithPlatformPath(moduleFilePath.getBuffer(), handle)))
+            {
+                sink->diagnose(SourceLoc(), Diagnostics::unableToReadFile, moduleFilePath);
+                return SLANG_FAIL;
+            }
+            RefPtr<TemporarySharedLibrary> sharedLib(new TemporarySharedLibrary(handle, moduleFilePath));
+            sharedLib->m_temporaryFileSet = productFileSet;
+            productFileSet.clear();
         }
-        catch (const Slang::IOException&)
+        else
         {
-            sink->diagnose(SourceLoc(), Diagnostics::unableToReadFile, moduleFilePath);
-            return SLANG_FAIL;
+            // Read the binary
+            try
+            {
+                // TODO(JS): We have a problem here.. productFileSet will clear up all temporaries
+                // and although we return the binary here (through outBin), we don't return debug info
+                // which is separate (say with a pdb). To work around this we reevaluate productFileSet,
+                // so we don't include debug info. The executable will presumably be reconstructed from
+                // outBin
+                // The problem is that these files have no specific lifetime (unlike with HostCallable).
+
+                CPPCompiler::ProductFlags flags = CPPCompiler::ProductFlag::All;
+                flags &= ~CPPCompiler::ProductFlag::Debug;
+
+                List<String> paths;
+                SLANG_RETURN_ON_FAIL(compiler->calcCompileProducts(options, flags, paths));
+                productFileSet.clear();
+                productFileSet.add(paths);
+
+                // Read the contents of the binary
+                outBin = File::readAllBytes(moduleFilePath);
+            }
+            catch (const Slang::IOException&)
+            {
+                sink->diagnose(SourceLoc(), Diagnostics::unableToReadFile, moduleFilePath);
+                return SLANG_FAIL;
+            }
         }
 
         return SLANG_OK;
@@ -1550,9 +1628,12 @@ SlangResult dissassembleDXILUsingDXC(
 
         switch (target)
         {
+        case CodeGenTarget::HostCallable:
         case CodeGenTarget::SharedLibrary:
         case CodeGenTarget::Executable:
             {
+                ComPtr<ISlangSharedLibrary> sharedLibrary;
+
                 List<uint8_t> code;
                 if (SLANG_SUCCEEDED(emitCPUBinaryForEntryPoint(
                     compileRequest,
@@ -1560,10 +1641,18 @@ SlangResult dissassembleDXILUsingDXC(
                     entryPointIndex,
                     targetReq,
                     endToEndReq,
+                    sharedLibrary,
                     code)))
                 {
-                    maybeDumpIntermediate(compileRequest, code.getBuffer(), code.getCount(), target);
-                    result = CompileResult(code);
+                    if (target == CodeGenTarget::HostCallable)
+                    {
+                        result = CompileResult(sharedLibrary);
+                    }
+                    else
+                    {
+                        maybeDumpIntermediate(compileRequest, code.getBuffer(), code.getCount(), target);
+                        result = CompileResult(code);
+                    }
                 }
             }
             break;
@@ -1860,6 +1949,9 @@ SlangResult dissassembleDXILUsingDXC(
             writeOutputToConsole(writer, result.outputString);
             break;
 
+        case ResultFormat::SharedLibrary:
+            break;
+
         case ResultFormat::Binary:
             {
                 auto& data = result.outputBinary;
@@ -2232,6 +2324,7 @@ SlangResult dissassembleDXILUsingDXC(
             // for now
             dumpIntermediateBinary(compileRequest, data, size, ".exe");
             break;
+        case CodeGenTarget::HostCallable:
         case CodeGenTarget::SharedLibrary:
             dumpIntermediateBinary(compileRequest, data, size, ".shared-lib");
             break;
diff --git a/source/slang/slang-compiler.h b/source/slang/slang-compiler.h
index 8f9c20044..bbc6505b0 100644
--- a/source/slang/slang-compiler.h
+++ b/source/slang/slang-compiler.h
@@ -61,6 +61,7 @@ namespace Slang
         CPPSource           = SLANG_CPP_SOURCE,
         Executable          = SLANG_EXECUTABLE,
         SharedLibrary       = SLANG_SHARED_LIBRARY,
+        HostCallable        = SLANG_HOST_CALLABLE,
     };
 
     CodeGenTarget calcCodeGenTargetFromName(const UnownedStringSlice& name);
@@ -84,7 +85,8 @@ namespace Slang
     {
         None,
         Text,
-        Binary
+        Binary,
+        SharedLibrary,
     };
 
     // When storing the layout for a matrix-type
@@ -122,22 +124,25 @@ namespace Slang
     class TranslationUnitRequest;
 
     // Result of compiling an entry point.
-    // Should only ever be string OR binary.
+    // Should only ever be string, binary or shared library
     class CompileResult
     {
     public:
         CompileResult() = default;
         CompileResult(String const& str) : format(ResultFormat::Text), outputString(str) {}
         CompileResult(List<uint8_t> const& buffer) : format(ResultFormat::Binary), outputBinary(buffer) {}
+        CompileResult(ISlangSharedLibrary* inSharedLibrary) : format(ResultFormat::SharedLibrary), sharedLibrary(inSharedLibrary) {}
 
         void append(CompileResult const& result);
 
-        ComPtr<ISlangBlob> getBlob();
+        SlangResult getBlob(ComPtr<ISlangBlob>& outBlob);
+        SlangResult getSharedLibrary(ComPtr<ISlangSharedLibrary>& outSharedLibrary);
 
         ResultFormat format = ResultFormat::None;
         String outputString;
         List<uint8_t> outputBinary;
 
+        ComPtr<ISlangSharedLibrary> sharedLibrary;
         ComPtr<ISlangBlob> blob;
     };
 
diff --git a/source/slang/slang-ir-entry-point-uniforms.cpp b/source/slang/slang-ir-entry-point-uniforms.cpp
index da036d798..822174620 100644
--- a/source/slang/slang-ir-entry-point-uniforms.cpp
+++ b/source/slang/slang-ir-entry-point-uniforms.cpp
@@ -441,6 +441,7 @@ void moveEntryPointUniformParamsToGlobalScope(
         case CodeGenTarget::CSource:
         case CodeGenTarget::Executable:
         case CodeGenTarget::SharedLibrary:
+        case CodeGenTarget::HostCallable:
         {
             context.targetNeedsConstantBuffer = false;
             break;
diff --git a/source/slang/slang-parameter-binding.cpp b/source/slang/slang-parameter-binding.cpp
index f9657e776..6e5838da5 100644
--- a/source/slang/slang-parameter-binding.cpp
+++ b/source/slang/slang-parameter-binding.cpp
@@ -2640,6 +2640,7 @@ static bool _isCPUTarget(CodeGenTarget target)
         case CodeGenTarget::CSource:
         case CodeGenTarget::Executable:
         case CodeGenTarget::SharedLibrary:
+        case CodeGenTarget::HostCallable:
         {
             return true;
         }
diff --git a/source/slang/slang-type-layout.cpp b/source/slang/slang-type-layout.cpp
index 3b14b74c2..9551cfe4e 100644
--- a/source/slang/slang-type-layout.cpp
+++ b/source/slang/slang-type-layout.cpp
@@ -982,7 +982,7 @@ LayoutRulesFamilyImpl* getDefaultLayoutRulesFamilyForTarget(TargetRequest* targe
     case CodeGenTarget::SPIRVAssembly:
         return &kGLSLLayoutRulesFamilyImpl;
 
-
+    case CodeGenTarget::HostCallable:
     case CodeGenTarget::Executable:
     case CodeGenTarget::SharedLibrary:
     case CodeGenTarget::CPPSource:
diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp
index cae9855e0..bb7e705e6 100644
--- a/source/slang/slang.cpp
+++ b/source/slang/slang.cpp
@@ -1716,7 +1716,9 @@ SLANG_NO_THROW SlangResult SLANG_MCALL ComponentType::getEntryPointCode(
     if(entryPointResult.format == ResultFormat::None )
         return SLANG_FAIL;
 
-    *outCode = entryPointResult.getBlob().detach();
+    ComPtr<ISlangBlob> blob;
+    SLANG_RETURN_ON_FAIL(entryPointResult.getBlob(blob));
+    *outCode = blob.detach();
     return SLANG_OK;
 }
 
@@ -3140,29 +3142,28 @@ SLANG_API void const* spGetEntryPointCode(
     return data;
 }
 
-SLANG_API SlangResult spGetEntryPointCodeBlob(
-        SlangCompileRequest*    request,
-        int                     entryPointIndex,
-        int                     targetIndex,
-        ISlangBlob**            outBlob)
+static SlangResult _getEntryPointResult(
+    SlangCompileRequest*    request,
+    int                     entryPointIndex,
+    int                     targetIndex,
+    Slang::CompileResult**  outCompileResult)
 {
     using namespace Slang;
-    if(!request) return SLANG_ERROR_INVALID_PARAMETER;
-    if(!outBlob) return SLANG_ERROR_INVALID_PARAMETER;
-
+    if (!request) return SLANG_ERROR_INVALID_PARAMETER;
+    
     auto req = Slang::asInternal(request);
     auto linkage = req->getLinkage();
     auto program = req->getSpecializedGlobalAndEntryPointsComponentType();
 
     Index targetCount = linkage->targets.getCount();
-    if((targetIndex < 0) || (targetIndex >= targetCount))
+    if ((targetIndex < 0) || (targetIndex >= targetCount))
     {
         return SLANG_ERROR_INVALID_PARAMETER;
     }
     auto targetReq = linkage->targets[targetIndex];
 
     Index entryPointCount = req->entryPoints.getCount();
-    if((entryPointIndex < 0) || (entryPointIndex >= entryPointCount))
+    if ((entryPointIndex < 0) || (entryPointIndex >= entryPointCount))
     {
         return SLANG_ERROR_INVALID_PARAMETER;
     }
@@ -3170,15 +3171,47 @@ SLANG_API SlangResult spGetEntryPointCodeBlob(
 
 
     auto targetProgram = program->getTargetProgram(targetReq);
-    if(!targetProgram)
+    if (!targetProgram)
         return SLANG_FAIL;
-    Slang::CompileResult& result = targetProgram->getExistingEntryPointResult(entryPointIndex);
+    *outCompileResult = &targetProgram->getExistingEntryPointResult(entryPointIndex);
+    return SLANG_OK;
+}
+
+SLANG_API SlangResult spGetEntryPointCodeBlob(
+        SlangCompileRequest*    request,
+        int                     entryPointIndex,
+        int                     targetIndex,
+        ISlangBlob**            outBlob)
+{
+    using namespace Slang;
+    if(!outBlob) return SLANG_ERROR_INVALID_PARAMETER;
+    Slang::CompileResult* compileResult = nullptr;
+    SLANG_RETURN_ON_FAIL(_getEntryPointResult(request, entryPointIndex, targetIndex, &compileResult));
 
-    auto blob = result.getBlob();
+    ComPtr<ISlangBlob> blob;
+    SLANG_RETURN_ON_FAIL(compileResult->getBlob(blob));
     *outBlob = blob.detach();
     return SLANG_OK;
 }
 
+SLANG_API SlangResult spGetEntryPointHostCallable(
+    SlangCompileRequest*    request,
+    int                     entryPointIndex,
+    int                     targetIndex,
+    ISlangSharedLibrary**   outSharedLibrary)
+{
+    using namespace Slang;
+    if (!outSharedLibrary) return SLANG_ERROR_INVALID_PARAMETER;
+
+    Slang::CompileResult* compileResult = nullptr;
+    SLANG_RETURN_ON_FAIL(_getEntryPointResult(request, entryPointIndex, targetIndex, &compileResult));
+
+    ComPtr<ISlangSharedLibrary> sharedLibrary;
+    SLANG_RETURN_ON_FAIL(compileResult->getSharedLibrary(sharedLibrary));
+    *outSharedLibrary = sharedLibrary.detach();
+    return SLANG_OK;
+}
+
 SLANG_API char const* spGetEntryPointSource(
     SlangCompileRequest*    request,
     int                     entryPointIndex)