Callable CPU code support (#1014)

* First pass support for compiling to a loaded shared library.

* Improve documentation for cpu target.

* Removed the SLANG_COMPILE_FLAG_LOAD_SHARED_LIBRARY flag.
Use the SLANG_HOST_CALLABLE code target
Document mechanism.

* Fix typo in cpp-resource.slang
In test code if the target is 'callable' we don't need to compile (indeed there is no source file).

* Small refactor using CommandLineCPPCompiler as base class to implement VisualStudioCPPCompiler and GCCCPPCompiler.

* Improvements around CPPCompiler.
Mechanism to know products produced.
Cleaning up products after execution.

* Fix multiple definition of 'SourceType'

1 files changed, 118 insertions, 25 deletions

diff --git a/source/slang/slang-compiler.cpp b/source/slang/slang-compiler.cpp
index 1c0bed065..8ead63784 100644
--- a/source/slang/slang-compiler.cpp
+++ b/source/slang/slang-compiler.cpp
@@ -87,7 +87,8 @@ namespace Slang
     x("c", CSource) \
     x("cpp", CPPSource) \
     x("exe,executable", Executable) \
-    x("sharedlib,sharedlibrary,dll", SharedLibrary) 
+    x("sharedlib,sharedlibrary,dll", SharedLibrary) \
+    x("callable,host-callable", HostCallable)
 
 #define SLANG_CODE_GEN_INFO(names, e) \
     { CodeGenTarget::e, UnownedStringSlice::fromLiteral(names) },
@@ -149,28 +150,64 @@ namespace Slang
         {
             outputBinary.addRange(result.outputBinary.getBuffer(), result.outputBinary.getCount());
         }
+        else if (appendTo == ResultFormat::SharedLibrary)
+        {
+            SLANG_ASSERT(!"Trying to append to a shared library");
+        }
+    }
+
+    SlangResult CompileResult::getSharedLibrary(ComPtr<ISlangSharedLibrary>& outSharedLibrary)
+    {
+        if (format == ResultFormat::SharedLibrary && sharedLibrary)
+        {
+            outSharedLibrary = sharedLibrary;
+            return SLANG_OK;
+        }
+        return SLANG_FAIL;
     }
 
-    ComPtr<ISlangBlob> CompileResult::getBlob()
+    SlangResult CompileResult::getBlob(ComPtr<ISlangBlob>& outBlob)
     {
         if(!blob)
         {
             switch(format)
             {
-            case ResultFormat::None:
-            default:
-                break;
+                case ResultFormat::None:
+                default:
+                    break;
 
-            case ResultFormat::Text:
-                blob = StringUtil::createStringBlob(outputString);
-                break;
+                case ResultFormat::Text:
+                    blob = StringUtil::createStringBlob(outputString);
+                    break;
 
-            case ResultFormat::Binary:
-                blob = createRawBlob(outputBinary.getBuffer(), outputBinary.getCount());
-                break;
+                case ResultFormat::Binary:
+                    blob = createRawBlob(outputBinary.getBuffer(), outputBinary.getCount());
+                    break;
+                case ResultFormat::SharedLibrary:
+                {
+                    // TODO(JS): Could do something more sophisticated to check this cast is safe (like have an interface to get the path), but this
+                    // is simple and works with current impl
+                    TemporarySharedLibrary* tempLibrary = static_cast<TemporarySharedLibrary*>(sharedLibrary.get());
+
+                    SLANG_ASSERT(sharedLibrary);
+                    List<uint8_t> contents;
+                    try
+                    {
+                        // We can read it from the shared library...
+                        contents = File::readAllBytes(tempLibrary->getPath());
+                    }
+                    catch (const Slang::IOException&)
+                    {
+                        return SLANG_E_CANNOT_OPEN; 
+                    }
+                    blob = createRawBlob(contents.getBuffer(), contents.getCount());
+                    break;
+                }
             }
         }
-        return blob;
+
+        outBlob = blob;
+        return SLANG_OK;
     }
 
     //
@@ -478,6 +515,7 @@ namespace Slang
                 // Don't need an external compiler to output C and C++ code
                 return PassThroughMode::None;
             }
+            case CodeGenTarget::HostCallable:
             case CodeGenTarget::SharedLibrary:
             case CodeGenTarget::Executable:
             {
@@ -1103,13 +1141,15 @@ SlangResult dissassembleDXILUsingDXC(
         Int                     entryPointIndex,
         TargetRequest*          targetReq,
         EndToEndCompileRequest* endToEndReq,
-        List<uint8_t>&          binOut)
+        ComPtr<ISlangSharedLibrary>& outSharedLib,
+        List<uint8_t>&          outBin)
     {
         auto sink = slangRequest->getSink();
 
         const String originalSourcePath = calcSourcePathForEntryPoint(endToEndReq, entryPointIndex);
 
-        binOut.clear();
+        outBin.clear();
+        outSharedLib.setNull();
       
         CPPCompilerSet* compilerSet = slangRequest->getSession()->requireCPPCompilerSet();
 
@@ -1311,6 +1351,7 @@ SlangResult dissassembleDXILUsingDXC(
         // Set what kind of target we should build
         switch (targetReq->target)
         {
+            case CodeGenTarget::HostCallable:
             case CodeGenTarget::SharedLibrary:
             {
                 options.targetType = CPPCompiler::TargetType::SharedLibrary;
@@ -1332,8 +1373,13 @@ SlangResult dissassembleDXILUsingDXC(
             moduleFilePath = builder.ProduceString();
         }
 
-        // Add so it's deleted at the end
-        temporaryFileSet.add(moduleFilePath);
+        // Find all the files that will be produced
+        TemporaryFileSet productFileSet;
+        {
+            List<String> paths;
+            SLANG_RETURN_ON_FAIL(compiler->calcCompileProducts(options, CPPCompiler::ProductFlag::All, paths));
+            productFileSet.add(paths);
+        }
 
         // Need to configure for the compilation
 
@@ -1457,16 +1503,48 @@ SlangResult dissassembleDXILUsingDXC(
             return SLANG_FAIL;
         }
 
-        // Read the binary
-        try
+        // If callable we need to load the shared library
+        if (targetReq->target == CodeGenTarget::HostCallable)
         {
-            // Read the contents of the binary
-            binOut = File::readAllBytes(moduleFilePath);
+            // Try loading the shared library
+            SharedLibrary::Handle handle;
+            if (SLANG_FAILED(SharedLibrary::loadWithPlatformPath(moduleFilePath.getBuffer(), handle)))
+            {
+                sink->diagnose(SourceLoc(), Diagnostics::unableToReadFile, moduleFilePath);
+                return SLANG_FAIL;
+            }
+            RefPtr<TemporarySharedLibrary> sharedLib(new TemporarySharedLibrary(handle, moduleFilePath));
+            sharedLib->m_temporaryFileSet = productFileSet;
+            productFileSet.clear();
         }
-        catch (const Slang::IOException&)
+        else
         {
-            sink->diagnose(SourceLoc(), Diagnostics::unableToReadFile, moduleFilePath);
-            return SLANG_FAIL;
+            // Read the binary
+            try
+            {
+                // TODO(JS): We have a problem here.. productFileSet will clear up all temporaries
+                // and although we return the binary here (through outBin), we don't return debug info
+                // which is separate (say with a pdb). To work around this we reevaluate productFileSet,
+                // so we don't include debug info. The executable will presumably be reconstructed from
+                // outBin
+                // The problem is that these files have no specific lifetime (unlike with HostCallable).
+
+                CPPCompiler::ProductFlags flags = CPPCompiler::ProductFlag::All;
+                flags &= ~CPPCompiler::ProductFlag::Debug;
+
+                List<String> paths;
+                SLANG_RETURN_ON_FAIL(compiler->calcCompileProducts(options, flags, paths));
+                productFileSet.clear();
+                productFileSet.add(paths);
+
+                // Read the contents of the binary
+                outBin = File::readAllBytes(moduleFilePath);
+            }
+            catch (const Slang::IOException&)
+            {
+                sink->diagnose(SourceLoc(), Diagnostics::unableToReadFile, moduleFilePath);
+                return SLANG_FAIL;
+            }
         }
 
         return SLANG_OK;
@@ -1550,9 +1628,12 @@ SlangResult dissassembleDXILUsingDXC(
 
         switch (target)
         {
+        case CodeGenTarget::HostCallable:
         case CodeGenTarget::SharedLibrary:
         case CodeGenTarget::Executable:
             {
+                ComPtr<ISlangSharedLibrary> sharedLibrary;
+
                 List<uint8_t> code;
                 if (SLANG_SUCCEEDED(emitCPUBinaryForEntryPoint(
                     compileRequest,
@@ -1560,10 +1641,18 @@ SlangResult dissassembleDXILUsingDXC(
                     entryPointIndex,
                     targetReq,
                     endToEndReq,
+                    sharedLibrary,
                     code)))
                 {
-                    maybeDumpIntermediate(compileRequest, code.getBuffer(), code.getCount(), target);
-                    result = CompileResult(code);
+                    if (target == CodeGenTarget::HostCallable)
+                    {
+                        result = CompileResult(sharedLibrary);
+                    }
+                    else
+                    {
+                        maybeDumpIntermediate(compileRequest, code.getBuffer(), code.getCount(), target);
+                        result = CompileResult(code);
+                    }
                 }
             }
             break;
@@ -1860,6 +1949,9 @@ SlangResult dissassembleDXILUsingDXC(
             writeOutputToConsole(writer, result.outputString);
             break;
 
+        case ResultFormat::SharedLibrary:
+            break;
+
         case ResultFormat::Binary:
             {
                 auto& data = result.outputBinary;
@@ -2232,6 +2324,7 @@ SlangResult dissassembleDXILUsingDXC(
             // for now
             dumpIntermediateBinary(compileRequest, data, size, ".exe");
             break;
+        case CodeGenTarget::HostCallable:
         case CodeGenTarget::SharedLibrary:
             dumpIntermediateBinary(compileRequest, data, size, ".shared-lib");
             break;