Feature/source downstream compiler (#1153)

* CPPCompiler -> DownstreamCompiler

* Added DownstreamCompileResult to start abstraction such that we don't need files.

* * Split out slang-blob.cpp
* Made CompileResult hold a DownstreamCompileResult - for access to binary or ISlangSharedLibrary

* Keep temporary files in scope.

* Add a hash to the hex dump stream.

* Move all file tracking into DownstreamCompiler.

* WIP support for nvrtc.

* WIP: Adding support for nvrtc compiler.
Adding enum types, wiring up the nvrtc into slang.

* Fix remaining CPPCompiler references.

* Fix order issue on target string matching.

* Use ISlangSharedLibrary for nvrtc.

* Use DownstreamCompiler for nvrtc.

* WIP first pass at compilation win nvrtc.

* Added testing if file is on file system into CommandLineDownstreamCompiler.
Added sourceContentsPath.

* Make test cuda-compile.cu work by just compiling not comparing output.

* Genearlize DownstreamCompiler usage.

* Fix warning on clang.

* Remove CompilerType from DownstreamCompiler.

* Use DownstreamCompiler interface for all compilers.

NOTE for FXC, DXC and GLSLANG this doesn't mean using 'compile' - it's still extracting functions from shared library.

* Replace DownstreamCompiler::SourceType -> SlangSourceLanguage

* Replace _canCompile with something data driven.

* Fix compiling on gcc/clang for DownstreamCompiler.

* Moved some text conversions into DownstreamCompiler.

* Fix problem on non-vc builds with not having return on locateCompilers for VS.

* Change so no warning for code not reachable on locateCompilers for vs.

4 files changed, 172 insertions, 23 deletions

diff --git a/source/core/slang-downstream-compiler.cpp b/source/core/slang-downstream-compiler.cpp
index 6ce526ef0..2e78ea22b 100644
--- a/source/core/slang-downstream-compiler.cpp
+++ b/source/core/slang-downstream-compiler.cpp
@@ -20,6 +20,29 @@
 namespace Slang
 {
 
+static DownstreamCompiler::Infos _calcInfos()
+{
+    typedef DownstreamCompiler::Info Info;
+    typedef DownstreamCompiler::SourceLanguageFlag SourceLanguageFlag;
+    typedef DownstreamCompiler::SourceLanguageFlags SourceLanguageFlags;
+
+    DownstreamCompiler::Infos infos;
+
+    infos.infos[int(SLANG_PASS_THROUGH_CLANG)] = Info(SourceLanguageFlag::CPP | SourceLanguageFlag::C);
+    infos.infos[int(SLANG_PASS_THROUGH_VISUAL_STUDIO)] = Info(SourceLanguageFlag::CPP | SourceLanguageFlag::C);
+    infos.infos[int(SLANG_PASS_THROUGH_GCC)] = Info(SourceLanguageFlag::CPP | SourceLanguageFlag::C);
+
+    infos.infos[int(SLANG_PASS_THROUGH_NVRTC)] = Info(SourceLanguageFlag::CUDA);
+
+    infos.infos[int(SLANG_PASS_THROUGH_DXC)] = Info(SourceLanguageFlag::HLSL);
+    infos.infos[int(SLANG_PASS_THROUGH_FXC)] = Info(SourceLanguageFlag::HLSL);
+    infos.infos[int(SLANG_PASS_THROUGH_GLSLANG)] = Info(SourceLanguageFlag::GLSL);
+
+    return infos;
+}
+
+/* static */DownstreamCompiler::Infos DownstreamCompiler::s_infos = _calcInfos();
+
 /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DownstreamCompiler::Desc !!!!!!!!!!!!!!!!!!!!!!*/
 
 void DownstreamCompiler::Desc::appendAsText(StringBuilder& out) const
@@ -67,6 +90,98 @@ void DownstreamCompiler::Desc::appendAsText(StringBuilder& out) const
     }
 }
 
+/* static */bool DownstreamCompiler::canCompile(SlangPassThrough compiler, SlangSourceLanguage sourceLanguage)
+{
+    const auto& info = getInfo(compiler);
+    return (info.sourceLanguageFlags & (SourceLanguageFlags(1) << int(sourceLanguage))) != 0;
+}
+
+/* static */SlangSourceLanguage DownstreamCompiler::getSourceLanguageFromName(const UnownedStringSlice& text)
+{
+    if (text == "c" || text == "C")
+    {
+        return SLANG_SOURCE_LANGUAGE_C;
+    }
+    else if (text == "cpp" || text == "c++" || text == "C++" || text == "cxx")
+    {
+        return SLANG_SOURCE_LANGUAGE_CPP;
+    }
+    else if (text == "slang")
+    {
+        return SLANG_SOURCE_LANGUAGE_SLANG;
+    }
+    else if (text == "glsl")
+    {
+        return SLANG_SOURCE_LANGUAGE_GLSL;
+    }
+    else if (text == "hlsl")
+    {
+        return SLANG_SOURCE_LANGUAGE_HLSL;
+    }
+    else if (text == "cu" || text == "cuda")
+    {
+        return SLANG_SOURCE_LANGUAGE_CUDA;
+    }
+    return SLANG_SOURCE_LANGUAGE_UNKNOWN;
+}
+
+#define SLANG_PASS_THROUGH_TYPES(x) \
+        x(none, NONE) \
+        x(fxc, FXC) \
+        x(dxc, DXC) \
+        x(glslang, GLSLANG) \
+        x(visualstudio, VISUAL_STUDIO) \
+        x(clang, CLANG) \
+        x(gcc, GCC) \
+        x(genericcpp, GENERIC_C_CPP) \
+        x(nvrtc, NVRTC)
+
+
+
+/* static */SlangPassThrough DownstreamCompiler::getPassThroughFromName(const UnownedStringSlice& slice)
+{
+#define SLANG_PASS_THROUGH_NAME_TO_TYPE(x, y) \
+    if (slice == UnownedStringSlice::fromLiteral(#x)) return SLANG_PASS_THROUGH_##y;
+
+    SLANG_PASS_THROUGH_TYPES(SLANG_PASS_THROUGH_NAME_TO_TYPE)
+
+    // Other options
+    if (slice == "c" || slice == "cpp")
+    {
+        return SLANG_PASS_THROUGH_GENERIC_C_CPP;
+    }
+    else if (slice == "vs")
+    {
+        return SLANG_PASS_THROUGH_VISUAL_STUDIO;
+    }
+
+    return SLANG_PASS_THROUGH_NONE;
+}
+
+/* static */SlangResult DownstreamCompiler::getPassThroughFromName(const UnownedStringSlice& slice, SlangPassThrough& outPassThrough)
+{
+    outPassThrough = getPassThroughFromName(slice);
+    // It could be none on error - if it's not equal to "none" then it msut be an error
+    if (outPassThrough == SLANG_PASS_THROUGH_NONE && slice != UnownedStringSlice::fromLiteral("none"))
+    {
+        return SLANG_FAIL;
+    }
+    return SLANG_OK;
+}
+
+/* static */UnownedStringSlice DownstreamCompiler::getPassThroughName(SlangPassThrough passThru)
+{
+#define SLANG_PASS_THROUGH_TYPE_TO_NAME(x, y) \
+    case SLANG_PASS_THROUGH_##y: return UnownedStringSlice::fromLiteral(#x);
+
+    switch (passThru)
+    {
+        SLANG_PASS_THROUGH_TYPES(SLANG_PASS_THROUGH_TYPE_TO_NAME)
+        default: break;
+    }
+    return UnownedStringSlice::fromLiteral("unknown");
+}
+
 /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DownstreamDiagnostics !!!!!!!!!!!!!!!!!!!!!!*/
 
 Index DownstreamDiagnostics::getCountByType(Diagnostic::Type type) const
@@ -284,7 +399,7 @@ SlangResult CommandLineDownstreamCompiler::compile(const CompileOptions& inOptio
             compileSourcePath.append("-src");
 
             // Make the temporary filename have the appropriate extension.
-            if (options.sourceType == DownstreamCompiler::SourceType::C)
+            if (options.sourceLanguage == SLANG_SOURCE_LANGUAGE_C)
             {
                 compileSourcePath.append(".c");
             }
@@ -515,19 +630,19 @@ const DownstreamCompiler::Desc& DownstreamCompilerUtil::getCompiledWithDesc()
     return findClosestCompiler(compilers, desc);
 }
 
-/* static */void DownstreamCompilerUtil::updateDefault(DownstreamCompilerSet* set, DownstreamCompiler::SourceType type)
+/* static */void DownstreamCompilerUtil::updateDefault(DownstreamCompilerSet* set, SlangSourceLanguage sourceLanguage)
 {
     DownstreamCompiler* compiler = nullptr;
 
-    switch (type)
+    switch (sourceLanguage)
     {
-        case DownstreamCompiler::SourceType::CPP:
-        case DownstreamCompiler::SourceType::C:
+        case SLANG_SOURCE_LANGUAGE_CPP:
+        case SLANG_SOURCE_LANGUAGE_C:
         {
             compiler = findClosestCompiler(set, getCompiledWithDesc());
             break;
         }
-        case DownstreamCompiler::SourceType::CUDA:
+        case SLANG_SOURCE_LANGUAGE_CUDA:
         {
             DownstreamCompiler::Desc desc;
             desc.type = SLANG_PASS_THROUGH_NVRTC;
@@ -537,14 +652,14 @@ const DownstreamCompiler::Desc& DownstreamCompilerUtil::getCompiledWithDesc()
         default: break;
     }
 
-    set->setDefaultCompiler(type, compiler);
+    set->setDefaultCompiler(sourceLanguage, compiler);
 }
 
 /* static */void DownstreamCompilerUtil::updateDefaults(DownstreamCompilerSet* set)
 {
-    for (Index i = 0; i < Index(DownstreamCompiler::SourceType::CountOf); ++i)
+    for (Index i = 0; i < Index(SLANG_SOURCE_LANGUAGE_COUNT_OF); ++i)
     {
-        updateDefault(set, DownstreamCompiler::SourceType(i));
+        updateDefault(set, SlangSourceLanguage(i));
     }
 }
 
diff --git a/source/core/slang-downstream-compiler.h b/source/core/slang-downstream-compiler.h
index 075f8f26a..57fb88a3c 100644
--- a/source/core/slang-downstream-compiler.h
+++ b/source/core/slang-downstream-compiler.h
@@ -120,12 +120,33 @@ public:
 
     typedef DownstreamCompileResult CompileResult;
 
-    enum class SourceType
+    typedef uint32_t SourceLanguageFlags;
+    struct SourceLanguageFlag
     {
-        C,              ///< C source
-        CPP,            ///< C++ source
-        CUDA,           ///< The CUDA language
-        CountOf,
+        enum Enum : SourceLanguageFlags
+        {
+            Unknown = SourceLanguageFlags(1) <<  SLANG_SOURCE_LANGUAGE_UNKNOWN,
+            Slang   = SourceLanguageFlags(1) <<  SLANG_SOURCE_LANGUAGE_SLANG,
+            HLSL    = SourceLanguageFlags(1) <<  SLANG_SOURCE_LANGUAGE_HLSL,
+            GLSL    = SourceLanguageFlags(1) << SLANG_SOURCE_LANGUAGE_GLSL,
+            C       = SourceLanguageFlags(1) << SLANG_SOURCE_LANGUAGE_C,
+            CPP     = SourceLanguageFlags(1) << SLANG_SOURCE_LANGUAGE_CPP,
+            CUDA    = SourceLanguageFlags(1) << SLANG_SOURCE_LANGUAGE_CUDA,
+        };
+    };
+
+    struct Info
+    {
+        Info():sourceLanguageFlags(0) {}
+
+        Info(SourceLanguageFlags inSourceLanguageFlags):
+            sourceLanguageFlags(inSourceLanguageFlags)
+        {}
+        SourceLanguageFlags sourceLanguageFlags;
+    };
+    struct Infos
+    {
+        Info infos[int(SLANG_PASS_THROUGH_COUNT_OF)];
     };
 
     struct Desc
@@ -202,7 +223,7 @@ public:
         OptimizationLevel optimizationLevel = OptimizationLevel::Default;
         DebugInfoType debugInfoType = DebugInfoType::Standard;
         TargetType targetType = TargetType::Executable;
-        SourceType sourceType = SourceType::CPP;
+        SlangSourceLanguage sourceLanguage = SLANG_SOURCE_LANGUAGE_CPP;
         FloatingPointMode floatingPointMode = FloatingPointMode::Default;
 
         Flags flags = Flag::EnableExceptionHandling;
@@ -262,7 +283,21 @@ public:
         /// Return the compiler type as name
     static UnownedStringSlice getCompilerTypeAsText(SlangPassThrough type);
 
+        /// Get info for a compiler type
+    static const Info& getInfo(SlangPassThrough compiler) { return s_infos.infos[int(compiler)]; }
+        /// True if this compiler can compile the specified language
+    static bool canCompile(SlangPassThrough compiler, SlangSourceLanguage sourceLanguage);
+
+        /// Given a source language name returns a source language. Name here is distinct from extension
+    static SlangSourceLanguage getSourceLanguageFromName(const UnownedStringSlice& text);
+        /// Given a name returns the pass through
+    static SlangPassThrough getPassThroughFromName(const UnownedStringSlice& slice);
+    static SlangResult getPassThroughFromName(const UnownedStringSlice& slice, SlangPassThrough& outPassThrough);
+        /// Get the compilers name
+    static UnownedStringSlice getPassThroughName(SlangPassThrough passThru);
+
 protected:
+    static Infos s_infos;
 
     DownstreamCompiler(const Desc& desc) :
         m_desc(desc)
@@ -369,9 +404,9 @@ public:
     void addCompiler(DownstreamCompiler* compiler);
 
         /// Get a default compiler
-    DownstreamCompiler* getDefaultCompiler(DownstreamCompiler::SourceType sourceType) const { return m_defaultCompilers[int(sourceType)];  }
+    DownstreamCompiler* getDefaultCompiler(SlangSourceLanguage sourceLanguage) const { return m_defaultCompilers[int(sourceLanguage)];  }
         /// Set the default compiler
-    void setDefaultCompiler(DownstreamCompiler::SourceType sourceType, DownstreamCompiler* compiler) { m_defaultCompilers[int(sourceType)] = compiler;  }
+    void setDefaultCompiler(SlangSourceLanguage sourceLanguage, DownstreamCompiler* compiler) { m_defaultCompilers[int(sourceLanguage)] = compiler;  }
 
         /// True if has a compiler of the specified type
     bool hasCompiler(SlangPassThrough compilerType) const;
@@ -384,7 +419,7 @@ protected:
 
     Index _findIndex(const DownstreamCompiler::Desc& desc) const;
 
-    RefPtr<DownstreamCompiler> m_defaultCompilers[int(DownstreamCompiler::SourceType::CountOf)];
+    RefPtr<DownstreamCompiler> m_defaultCompilers[int(SLANG_SOURCE_LANGUAGE_COUNT_OF)];
     // This could be a dictionary/map - but doing a linear search is going to be fine and it makes
     // somethings easier.
     List<RefPtr<DownstreamCompiler>> m_compilers;
@@ -399,7 +434,6 @@ struct DownstreamCompilerBaseUtil
     typedef DownstreamCompiler::OptimizationLevel OptimizationLevel;
     typedef DownstreamCompiler::TargetType TargetType;
     typedef DownstreamCompiler::DebugInfoType DebugInfoType;
-    typedef DownstreamCompiler::SourceType SourceType;
     
     typedef DownstreamDiagnostics::Diagnostic Diagnostic;
 
@@ -428,7 +462,7 @@ struct DownstreamCompilerUtil: public DownstreamCompilerBaseUtil
         /// Get the information on the compiler used to compile this source
     static const DownstreamCompiler::Desc& getCompiledWithDesc();
 
-    static void updateDefault(DownstreamCompilerSet* set, DownstreamCompiler::SourceType type);
+    static void updateDefault(DownstreamCompilerSet* set, SlangSourceLanguage sourceLanguage);
     static void updateDefaults(DownstreamCompilerSet* set);
 
     static void setDefaultLocators(DownstreamCompilerLocatorFunc outFuncs[int(SLANG_PASS_THROUGH_COUNT_OF)]);
diff --git a/source/core/slang-gcc-compiler-util.cpp b/source/core/slang-gcc-compiler-util.cpp
index 7c2ec3775..833ae884e 100644
--- a/source/core/slang-gcc-compiler-util.cpp
+++ b/source/core/slang-gcc-compiler-util.cpp
@@ -406,7 +406,7 @@ static SlangResult _parseGCCFamilyLine(const UnownedStringSlice& line, LineParse
 
     PlatformKind platformKind = (options.platform == PlatformKind::Unknown) ? PlatformUtil::getPlatformKind() : options.platform;
         
-    if (options.sourceType == SourceType::CPP)
+    if (options.sourceLanguage == SLANG_SOURCE_LANGUAGE_CPP)
     {
         cmdLine.addArg("-fvisibility=hidden");
 
@@ -570,7 +570,7 @@ static SlangResult _parseGCCFamilyLine(const UnownedStringSlice& line, LineParse
         cmdLine.addArg(libPath);
     }
 
-    if (options.sourceType == SourceType::CPP && !PlatformUtil::isFamily(PlatformFamily::Windows, platformKind))
+    if (options.sourceLanguage == SLANG_SOURCE_LANGUAGE_CPP && !PlatformUtil::isFamily(PlatformFamily::Windows, platformKind))
     {
         // Make STD libs available
         cmdLine.addArg("-lstdc++");
diff --git a/source/core/slang-visual-studio-compiler-util.cpp b/source/core/slang-visual-studio-compiler-util.cpp
index 1a42e9b6e..8dcc97654 100644
--- a/source/core/slang-visual-studio-compiler-util.cpp
+++ b/source/core/slang-visual-studio-compiler-util.cpp
@@ -94,7 +94,7 @@ namespace Slang
 
     if (options.flags & CompileOptions::Flag::EnableExceptionHandling)
     {
-        if (options.sourceType == SourceType::CPP)
+        if (options.sourceLanguage == SLANG_SOURCE_LANGUAGE_CPP)
         {
             // https://docs.microsoft.com/en-us/cpp/build/reference/eh-exception-handling-model?view=vs-2019
             // Assumes c functions cannot throw