User defined downstream compiler prelude (#1028)

* Added setDownstreamCompilerPrelude
Renamed setPassThroughPath to setDownstreamCompilerPath.
Fixed tests.
Added prelude directory & code to TestToolUtil to setup default preludes for testing/command line apis.

* Fix merge problem

* Remove hacks to make prelude work by adding a search path as no longer needed with 'user prelude'.

* Split up prelude into scalar intrinsics, and types.
Use slang.h for main header.
slang-cpp-prelude.h can now just include what it needs (relative to prelude directory) and define the few remaining things/work arounds.

* Fix typo.

7 files changed, 76 insertions, 83 deletions

diff --git a/source/slang/slang-check.cpp b/source/slang/slang-check.cpp
index 464dafcd9..33fc10e45 100644
--- a/source/slang/slang-check.cpp
+++ b/source/slang/slang-check.cpp
@@ -364,9 +364,9 @@ namespace Slang
 
             StringBuilder builder;
             PassThroughMode passThrough = _toPassThroughMode(type);
-            if (passThrough != PassThroughMode::None && m_passThroughPaths[int(passThrough)].getLength() > 0)
+            if (passThrough != PassThroughMode::None && m_downstreamCompilerPaths[int(passThrough)].getLength() > 0)
             {
-                Path::combineIntoBuilder(m_passThroughPaths[int(passThrough)].getUnownedSlice(), UnownedStringSlice(libName), builder);
+                Path::combineIntoBuilder(m_downstreamCompilerPaths[int(passThrough)].getUnownedSlice(), UnownedStringSlice(libName), builder);
                 libName = builder.getBuffer();
             }
 
@@ -424,9 +424,9 @@ namespace Slang
             typedef CPPCompiler::CompilerType CompilerType;
             CPPCompilerUtil::InitializeSetDesc desc;
        
-            desc.paths[int(CompilerType::GCC)] = m_passThroughPaths[int(PassThroughMode::Gcc)];
-            desc.paths[int(CompilerType::Clang)] = m_passThroughPaths[int(PassThroughMode::Clang)];
-            desc.paths[int(CompilerType::VisualStudio)] = m_passThroughPaths[int(PassThroughMode::VisualStudio)];
+            desc.paths[int(CompilerType::GCC)] = m_downstreamCompilerPaths[int(PassThroughMode::Gcc)];
+            desc.paths[int(CompilerType::Clang)] = m_downstreamCompilerPaths[int(PassThroughMode::Clang)];
+            desc.paths[int(CompilerType::VisualStudio)] = m_downstreamCompilerPaths[int(PassThroughMode::VisualStudio)];
 
             CPPCompilerUtil::initializeSet(desc, cppCompilerSet);
         }
diff --git a/source/slang/slang-compiler.cpp b/source/slang/slang-compiler.cpp
index 75cd61bf9..07e2b66fe 100644
--- a/source/slang/slang-compiler.cpp
+++ b/source/slang/slang-compiler.cpp
@@ -474,7 +474,7 @@ namespace Slang
         return SLANG_E_NOT_IMPLEMENTED;
     }
 
-    static PassThroughMode _getExternalCompilerRequiredForTarget(CodeGenTarget target)
+    PassThroughMode getDownstreamCompilerRequiredForTarget(CodeGenTarget target)
     {
         switch (target)
         {
@@ -530,9 +530,22 @@ namespace Slang
         return PassThroughMode::None;
     }
 
+    PassThroughMode getPassThroughModeForCPPCompiler(CPPCompiler::CompilerType type)
+    {
+        typedef CPPCompiler::CompilerType CompilerType;
+
+        switch (type)
+        {
+            case CompilerType::VisualStudio:        return PassThroughMode::VisualStudio;
+            case CompilerType::GCC:                 return PassThroughMode::Gcc;
+            case CompilerType::Clang:               return PassThroughMode::Clang;
+            default:                                return PassThroughMode::None;
+        }
+    }
+
     SlangResult checkCompileTargetSupport(Session* session, CodeGenTarget target)
     {
-        const PassThroughMode mode = _getExternalCompilerRequiredForTarget(target);
+        const PassThroughMode mode = getDownstreamCompilerRequiredForTarget(target);
         return (mode != PassThroughMode::None) ?
             checkExternalCompilerSupport(session, mode) :
             SLANG_OK;
@@ -561,21 +574,6 @@ namespace Slang
         return translationUnit;
     }
 
-    static TranslationUnitRequest* _getTranslationUnit(
-        EndToEndCompileRequest* endToEndReq,
-        Int                     entryPointIndex)
-    {
-        // If there isn't an end-to-end compile going on,
-        // there can be no pass-through.
-        //
-        if (!endToEndReq) return nullptr;
-
-        auto frontEndReq = endToEndReq->getFrontEndReq();
-        auto entryPointReq = frontEndReq->getEntryPointReq(entryPointIndex);
-        auto translationUnit = entryPointReq->getTranslationUnit();
-        return translationUnit;
-    }
-
     static void _appendEscapedPath(const UnownedStringSlice& path, StringBuilder& outBuilder)
     {
         for (auto c : path)
@@ -1316,28 +1314,6 @@ SlangResult dissassembleDXILUsingDXC(
         }
         else
         {
-            // TODO(JS): This is a hack for two reasons
-            // * That we just inject the source path for C/C++ include paths if we find the file
-            // * We should access the files through the ISlangFileSystem
-
-            translationUnit = _getTranslationUnit(endToEndReq, entryPointIndex);
-
-            const auto& sourceFiles = translationUnit->getSourceFiles();
-            if (sourceFiles.getCount() == 1)
-            {
-                const SourceFile* sourceFile = sourceFiles[0];
-                const PathInfo& pathInfo = sourceFile->getPathInfo();
-                if (pathInfo.type == PathInfo::Type::FoundPath || pathInfo.type == PathInfo::Type::Normal || pathInfo.type == PathInfo::Type::FromString)
-                {
-                    String canonicalPath;
-                    if (File::exists(pathInfo.foundPath) && SLANG_SUCCEEDED(Path::getCanonical(pathInfo.foundPath, canonicalPath)))
-                    {
-                        String sourceDir = Path::getParentDirectory(canonicalPath);
-                        includePaths.add(sourceDir);                        
-                    }
-                }
-            }
-
             rawSource = emitCPPForEntryPoint(
                 slangRequest,
                 entryPoint,
@@ -1477,37 +1453,6 @@ SlangResult dissassembleDXILUsingDXC(
             }
         }
 
-        // TODO(JS): HACK! We need to include the prelude from somewhere, but where? The generated output
-        // is sitting in some temp directory.
-        // So here, we search all the 'sourceFiles', and try their paths for plausibility, and take the first
-        {
-            auto frontEndReq = endToEndReq->getFrontEndReq();
-            auto entryPointReq = frontEndReq->getEntryPointReq(entryPointIndex);
-            auto translationUnit = entryPointReq->getTranslationUnit();
-
-            for (SourceFile* sourceFile : translationUnit->m_sourceFiles)
-            {
-                const auto& pathInfo = sourceFile->getPathInfo();
-
-                if (pathInfo.type == PathInfo::Type::FoundPath ||
-                    pathInfo.type == PathInfo::Type::Normal)
-                {
-                    String originalSourceDirectory = Path::getParentDirectory(pathInfo.foundPath);
-
-                    if (originalSourceDirectory.getLength() && File::exists(originalSourceDirectory))
-                    {
-                        // We can't use this path directly, so make canonical so it is absolute
-                        StringBuilder canonicalPath;
-                        if (SLANG_SUCCEEDED(Path::getCanonical(originalSourceDirectory, canonicalPath)))
-                        {
-                            options.includePaths.add(canonicalPath.ProduceString());
-                            break;
-                        }
-                    }
-                }
-            }
-        }
-
         // Compile
         CPPCompiler::Output output;
         SLANG_RETURN_ON_FAIL(compiler->compile(options, output));
diff --git a/source/slang/slang-compiler.h b/source/slang/slang-compiler.h
index ce01e5ea3..a6be59c76 100644
--- a/source/slang/slang-compiler.h
+++ b/source/slang/slang-compiler.h
@@ -62,6 +62,7 @@ namespace Slang
         Executable          = SLANG_EXECUTABLE,
         SharedLibrary       = SLANG_SHARED_LIBRARY,
         HostCallable        = SLANG_HOST_CALLABLE,
+        CountOf             = SLANG_TARGET_COUNT_OF,
     };
 
     CodeGenTarget calcCodeGenTargetFromName(const UnownedStringSlice& name);
@@ -1057,6 +1058,13 @@ namespace Slang
     ///
     ComPtr<ISlangBlob> createRawBlob(void const* data, size_t size);
 
+
+        /// Given a target returns the required downstream compiler
+    PassThroughMode getDownstreamCompilerRequiredForTarget(CodeGenTarget target);
+
+    PassThroughMode getPassThroughModeForCPPCompiler(CPPCompiler::CompilerType type);
+
+
         /// A context for loading and re-using code modules.
     class Linkage : public RefObject, public slang::ISession
     {
@@ -1791,10 +1799,13 @@ namespace Slang
         SLANG_NO_THROW SlangProfileID SLANG_MCALL findProfile(
             char const*     name) override;
 
-        SLANG_NO_THROW void SLANG_MCALL setPassThroughPath(
+        SLANG_NO_THROW void SLANG_MCALL setDownstreamCompilerPath(
             SlangPassThrough passThrough,
             char const* path) override;
 
+        SLANG_NO_THROW void SLANG_MCALL setDownstreamCompilerPrelude(
+            SlangPassThrough inPassThrough,
+            char const* prelude) override;
 
         enum class SharedLibraryFuncType
         {
@@ -1928,6 +1939,9 @@ namespace Slang
 
         SlangFuncPtr getSharedLibraryFunc(SharedLibraryFuncType type, DiagnosticSink* sink);
 
+            /// Get the downstream compiler prelude
+        const String& getDownstreamCompilerPrelude(PassThroughMode mode) { return m_downstreamCompilerPreludes[int(mode)]; }
+
             /// Finds out what compilers are present and caches the result
         CPPCompilerSet* requireCPPCompilerSet();
 
@@ -1943,7 +1957,8 @@ namespace Slang
             /// Linkage used for all built-in (stdlib) code.
         RefPtr<Linkage> m_builtinLinkage;
 
-        String m_passThroughPaths[int(PassThroughMode::CountOf)];               ///< Paths for each pass through
+        String m_downstreamCompilerPaths[int(PassThroughMode::CountOf)];              ///< Paths for each pass through
+        String m_downstreamCompilerPreludes[int(PassThroughMode::CountOf)];             ///< Prelude for each type of target
     };
 
 
diff --git a/source/slang/slang-emit-cpp.cpp b/source/slang/slang-emit-cpp.cpp
index 0228955dc..dc9ab23cc 100644
--- a/source/slang/slang-emit-cpp.cpp
+++ b/source/slang/slang-emit-cpp.cpp
@@ -2213,8 +2213,6 @@ void CPPSourceEmitter::emitPreprocessorDirectivesImpl()
 
     writer->emit("\n");
 
-    writer->emit("#include <slang-cpp-prelude.h>\n\n");
-
     // Emit the type definitions
     for (const auto& keyValue : m_typeNameMap)
     {
diff --git a/source/slang/slang-emit.cpp b/source/slang/slang-emit.cpp
index 205f8ee0e..5a4e8300a 100644
--- a/source/slang/slang-emit.cpp
+++ b/source/slang/slang-emit.cpp
@@ -540,6 +540,31 @@ String emitEntryPoint(
     // Now that we've emitted the code for all the declarations in the file,
     // it is time to stitch together the final output.
 
+    {
+        Session* session = compileRequest->getSession();
+
+        // Get the downstream compiler needed for final target
+        PassThroughMode passThru = getDownstreamCompilerRequiredForTarget(targetRequest->target);
+
+        // If generic CPP work out what compiler will actually be used
+        if (passThru == PassThroughMode::GenericCCpp)
+        {
+            CPPCompilerSet* compilerSet = session->requireCPPCompilerSet();
+            CPPCompiler* compiler = compilerSet->getDefaultCompiler();
+            if (compiler)
+            {
+                passThru = getPassThroughModeForCPPCompiler(compiler->getDesc().type);
+            }
+        }
+
+        // If there is a prelude emit it
+        const auto& prelude = compileRequest->getSession()->getDownstreamCompilerPrelude(passThru);
+        if (prelude.getLength() > 0)
+        {
+            sourceWriter.emit(prelude.getUnownedSlice());
+        }
+    }
+
     // There may be global-scope modifiers that we should emit now
     sourceEmitter->emitPreprocessorDirectives();
 
diff --git a/source/slang/slang-options.cpp b/source/slang/slang-options.cpp
index c07ace929..a55322d31 100644
--- a/source/slang/slang-options.cpp
+++ b/source/slang/slang-options.cpp
@@ -824,7 +824,7 @@ struct OptionsParser
                             SlangPassThrough passThrough = SLANG_PASS_THROUGH_NONE;
                             if (SLANG_SUCCEEDED(_parsePassThrough(slice.getUnownedSlice(), passThrough)))
                             {
-                                session->setPassThroughPath(passThrough, name.getBuffer());
+                                session->setDownstreamCompilerPath(passThrough, name.getBuffer());
                                 continue;
                             }
                         }
diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp
index eef0dff6f..601312def 100644
--- a/source/slang/slang.cpp
+++ b/source/slang/slang.cpp
@@ -146,14 +146,14 @@ SLANG_NO_THROW SlangProfileID SLANG_MCALL Session::findProfile(
     return Slang::Profile::LookUp(name).raw;
 }
 
-SLANG_NO_THROW void SLANG_MCALL Session::setPassThroughPath(
+SLANG_NO_THROW void SLANG_MCALL Session::setDownstreamCompilerPath(
     SlangPassThrough inPassThrough,
     char const* path)
 {
     PassThroughMode passThrough = PassThroughMode(inPassThrough);
     SLANG_ASSERT(int(passThrough) > int(PassThroughMode::None) && int(passThrough) < int(PassThroughMode::CountOf));
     
-    if (m_passThroughPaths[int(passThrough)] != path)
+    if (m_downstreamCompilerPaths[int(passThrough)] != path)
     {
         // If it's changed we should unload any shared libraries that use it
         switch (passThrough)
@@ -187,10 +187,20 @@ SLANG_NO_THROW void SLANG_MCALL Session::setPassThroughPath(
         }
 
         // Set the path
-        m_passThroughPaths[int(passThrough)] = path;
+        m_downstreamCompilerPaths[int(passThrough)] = path;
     }
 }
 
+SLANG_NO_THROW void SLANG_MCALL Session::setDownstreamCompilerPrelude(
+    SlangPassThrough inPassThrough,
+    char const* prelude)
+{
+    PassThroughMode passThrough = PassThroughMode(inPassThrough);
+    SLANG_ASSERT(int(passThrough) > int(PassThroughMode::None) && int(passThrough) < int(PassThroughMode::CountOf));
+
+    m_downstreamCompilerPreludes[int(passThrough)] = prelude;
+}
+
 struct IncludeHandlerImpl : IncludeHandler
 {
     Linkage*    linkage;