FXC as DownstreamCompiler (#1844)

* #include an absolute path didn't work - because paths were taken to always be relative.

* WIP Fxc as downstream compiler.

* First pass FXC downstream compiler working.

* GCC compile fix.

* Fix FXC parsing issue.

* Special case filesystem access.

* Use StringUtil getSlice.

* Fix isses with not emitting source for FXC.

* Small fixes for DXBC handling.

9 files changed, 780 insertions, 30 deletions

diff --git a/source/compiler-core/slang-downstream-compiler.cpp b/source/compiler-core/slang-downstream-compiler.cpp
index 696376b49..d89434896 100644
--- a/source/compiler-core/slang-downstream-compiler.cpp
+++ b/source/compiler-core/slang-downstream-compiler.cpp
@@ -18,6 +18,7 @@
 #include "slang-visual-studio-compiler-util.h"
 #include "slang-gcc-compiler-util.h"
 #include "slang-nvrtc-compiler.h"
+#include "slang-fxc-compiler.h"
 
 namespace Slang
 {
@@ -74,8 +75,56 @@ void DownstreamCompiler::Desc::appendAsText(StringBuilder& out) const
     }
 }
 
+/* static */SlangResult DownstreamDiagnostic::splitPathLocation(const UnownedStringSlice& pathLocation, DownstreamDiagnostic& outDiagnostic)
+{
+    const Index lineStartIndex = pathLocation.lastIndexOf('(');
+    if (lineStartIndex >= 0)
+    {
+        outDiagnostic.filePath = UnownedStringSlice(pathLocation.head(lineStartIndex).trim());
+
+        const UnownedStringSlice tail = pathLocation.tail(lineStartIndex + 1);
+        const Index lineEndIndex = tail.indexOf(')');
+
+        if (lineEndIndex >= 0)
+        {
+            // Extract the location info
+            UnownedStringSlice locationSlice(tail.begin(), tail.begin() + lineEndIndex);
+
+            UnownedStringSlice slices[2];
+            const Index numSlices = StringUtil::split(locationSlice, ',', 2, slices);
+
+            // NOTE! FXC actually outputs a range of columns in the form of START-END in the column position
+            // We don't need to parse here, because we only care about the line number
+
+            Int lineNumber = 0;
+            if (numSlices > 0)
+            {
+                SLANG_RETURN_ON_FAIL(StringUtil::parseInt(slices[0], lineNumber));
+            }
+
+            // Store the line
+            outDiagnostic.fileLine = lineNumber;
+        }
+    }
+    else
+    {
+        outDiagnostic.filePath = pathLocation;
+    }
+    return SLANG_OK;
+}
+
 /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DownstreamCompiler !!!!!!!!!!!!!!!!!!!!!!!!!!!!!*/
 
+SlangResult DownstreamCompiler::dissassemble(SlangCompileTarget sourceBlobTarget, const void* blob, size_t blobSize, ISlangBlob** out)
+{
+    SLANG_UNUSED(sourceBlobTarget);
+    SLANG_UNUSED(blob);
+    SLANG_UNUSED(blobSize);
+    SLANG_UNUSED(out);
+
+    return SLANG_E_NOT_AVAILABLE;
+}
+
 
 /* static */bool DownstreamCompiler::canCompile(SlangPassThrough compiler, SlangSourceLanguage sourceLanguage)
 {
@@ -92,6 +141,7 @@ void DownstreamCompiler::Desc::appendAsText(StringBuilder& out) const
         case SLANG_SOURCE_LANGUAGE_C:       return SLANG_C_SOURCE;
         case SLANG_SOURCE_LANGUAGE_CPP:     return SLANG_CPP_SOURCE;
         case SLANG_SOURCE_LANGUAGE_CUDA:    return SLANG_CUDA_SOURCE;
+        
         default:                            return SLANG_TARGET_UNKNOWN;
     }
 }
@@ -196,6 +246,34 @@ void DownstreamDiagnostics::removeBySeverity(Diagnostic::Severity severity)
     }
 }
 
+/* static */void DownstreamDiagnostics::addNote(const UnownedStringSlice& in, List<DownstreamDiagnostic>& ioDiagnostics)
+{
+    // Don't bother adding an empty line
+    if (in.trim().getLength() == 0)
+    {
+        return;
+    }
+
+    // If there's nothing previous, we'll ignore too, as note should be in addition to
+    // a pre-existing error/warning
+    if (ioDiagnostics.getCount() == 0)
+    {
+        return;
+    }
+
+    // Make it a note on the output
+    DownstreamDiagnostic diagnostic;
+    diagnostic.reset();
+    diagnostic.severity = DownstreamDiagnostic::Severity::Info;
+    diagnostic.text = in;
+    ioDiagnostics.add(diagnostic);
+}
+
+void DownstreamDiagnostics::addNote(const UnownedStringSlice& in)
+{
+    addNote(in, diagnostics);
+}
+
 /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! CommandLineDownstreamCompileResult !!!!!!!!!!!!!!!!!!!!!!*/
 
 SlangResult CommandLineDownstreamCompileResult::getHostCallableSharedLibrary(ComPtr<ISlangSharedLibrary>& outLibrary)
@@ -595,19 +673,6 @@ static SlangResult _locateDXCCompilers(const String& path, ISlangSharedLibraryLo
     return SLANG_OK;
 }
 
-static SlangResult _locateFXCCompilers(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set)
-{
-    ComPtr<ISlangSharedLibrary> sharedLibrary;
-    if (SLANG_SUCCEEDED(DefaultSharedLibraryLoader::load(loader, path, "d3dcompiler_47", sharedLibrary.writeRef())))
-    {
-        // Can we determine the version?
-        DownstreamCompiler::Desc desc(SLANG_PASS_THROUGH_FXC);
-        RefPtr<DownstreamCompiler> compiler(new SharedLibraryDownstreamCompiler(desc, sharedLibrary));
-        set->addCompiler(compiler);
-    }
-    return SLANG_OK;
-}
-
 static SlangResult _locateGlslangCompilers(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set)
 {
 #if SLANG_UNIX_FAMILY
@@ -633,7 +698,7 @@ static SlangResult _locateGlslangCompilers(const String& path, ISlangSharedLibra
     outFuncs[int(SLANG_PASS_THROUGH_GCC)] = &GCCDownstreamCompilerUtil::locateGCCCompilers;
     outFuncs[int(SLANG_PASS_THROUGH_NVRTC)] = &NVRTCDownstreamCompilerUtil::locateCompilers;
     outFuncs[int(SLANG_PASS_THROUGH_DXC)] = &_locateDXCCompilers;
-    outFuncs[int(SLANG_PASS_THROUGH_FXC)] = &_locateFXCCompilers;
+    outFuncs[int(SLANG_PASS_THROUGH_FXC)] = &FXCDownstreamCompilerUtil::locateCompilers;
     outFuncs[int(SLANG_PASS_THROUGH_GLSLANG)] = &_locateGlslangCompilers;
 }
 
diff --git a/source/compiler-core/slang-downstream-compiler.h b/source/compiler-core/slang-downstream-compiler.h
index def13993e..b7f7b0cde 100644
--- a/source/compiler-core/slang-downstream-compiler.h
+++ b/source/compiler-core/slang-downstream-compiler.h
@@ -16,6 +16,8 @@
 namespace Slang
 {
 
+struct SourceManager;
+
 struct DownstreamDiagnostic
 {
     typedef DownstreamDiagnostic ThisType;
@@ -54,6 +56,9 @@ struct DownstreamDiagnostic
 
     static UnownedStringSlice getSeverityText(Severity severity);
 
+        /// Given a path, that holds line number and potentially column number in () after path, writes result into outDiagnostic
+    static SlangResult splitPathLocation(const UnownedStringSlice& pathLocation, DownstreamDiagnostic& outDiagnostic);
+
     Severity severity;              ///< The severity of error
     Stage stage;                    ///< The stage the error came from
     String text;                    ///< The text of the error
@@ -85,6 +90,11 @@ struct DownstreamDiagnostics
         /// Remove all diagnostics of the type
     void removeBySeverity(Diagnostic::Severity severity);
 
+        /// Add a note
+    void addNote(const UnownedStringSlice& in);
+
+    static void addNote(const UnownedStringSlice& in, List<DownstreamDiagnostic>& ioDiagnostics);
+
     String rawDiagnostics;
 
     SlangResult result;
@@ -209,12 +219,14 @@ public:
         Precise,
     };
 
+#if 0
     enum TargetType
     {
         Executable,         ///< Produce an executable
         SharedLibrary,      ///< Produce a shared library object/dll 
         Object,             ///< Produce an object file
     };
+#endif
 
     enum PipelineType
     {
@@ -256,7 +268,7 @@ public:
 
         OptimizationLevel optimizationLevel = OptimizationLevel::Default;
         DebugInfoType debugInfoType = DebugInfoType::Standard;
-        TargetType targetType = TargetType::Executable;
+        SlangCompileTarget targetType = SLANG_EXECUTABLE;
         SlangSourceLanguage sourceLanguage = SLANG_SOURCE_LANGUAGE_CPP;
         FloatingPointMode floatingPointMode = FloatingPointMode::Default;
         PipelineType pipelineType = PipelineType::Unknown;
@@ -284,6 +296,16 @@ public:
         List<String> libraryPaths;
 
         List<CapabilityVersion> requiredCapabilityVersions;
+
+            /// For compilers/compiles that require an entry point name, else can be empty
+        String entryPointName;
+            /// Profile name to use, only required for compiles that need to compile against a a specific profiles.
+            /// Profile names are tied to compilers and targets.
+        String profileName; 
+
+            /// NOTE! Not all downstream compilers can use the fileSystemExt/sourceManager. This option will be ignored in those scenarios.
+        ISlangFileSystemExt* fileSystemExt = nullptr;
+        SourceManager* sourceManager = nullptr;
     };
 
     typedef uint32_t ProductFlags;
@@ -317,6 +339,12 @@ public:
         /// Some downstream compilers are backed by a shared library. This allows access to the shared library to access internal functions. 
     virtual ISlangSharedLibrary* getSharedLibrary() { return nullptr; }
 
+        /// Some compilers have support converting a binary blob into disassembly. Output disassembly is held in the output blob
+    virtual SlangResult dissassemble(SlangCompileTarget sourceBlobTarget, const void* blob, size_t blobSize, ISlangBlob** out);
+
+        /// True if underlying compiler uses file system to pass source
+    virtual bool isFileBased() = 0;
+
         /// Get info for a compiler type
     static const Info& getInfo(SlangPassThrough compiler) { return s_infos.infos[int(compiler)]; }
         /// True if this compiler can compile the specified language
@@ -369,7 +397,8 @@ public:
 
     // DownstreamCompiler
     virtual SlangResult compile(const CompileOptions& options, RefPtr<DownstreamCompileResult>& outResult) SLANG_OVERRIDE;
-    
+    virtual bool isFileBased() SLANG_OVERRIDE { return true; }
+
     // Functions to be implemented for a specific CommandLine
 
         /// Given the compilation options and the module name, determines the actual file name used for output
@@ -405,6 +434,7 @@ public:
 
     // DownstreamCompiler
     virtual SlangResult compile(const CompileOptions& options, RefPtr<DownstreamCompileResult>& outResult) SLANG_OVERRIDE { SLANG_UNUSED(options); SLANG_UNUSED(outResult); return SLANG_E_NOT_IMPLEMENTED; }
+    virtual bool isFileBased() SLANG_OVERRIDE { return true; }
     virtual ISlangSharedLibrary* getSharedLibrary() SLANG_OVERRIDE { return m_library; }
 
     SharedLibraryDownstreamCompiler(const Desc& desc, ISlangSharedLibrary* library):
@@ -461,7 +491,6 @@ struct DownstreamCompilerBaseUtil
 {
     typedef DownstreamCompiler::CompileOptions CompileOptions;
     typedef DownstreamCompiler::OptimizationLevel OptimizationLevel;
-    typedef DownstreamCompiler::TargetType TargetType;
     typedef DownstreamCompiler::DebugInfoType DebugInfoType;
     
     typedef DownstreamDiagnostics::Diagnostic Diagnostic;
diff --git a/source/compiler-core/slang-fxc-compiler.cpp b/source/compiler-core/slang-fxc-compiler.cpp
new file mode 100644
index 000000000..8e190bbca
--- /dev/null
+++ b/source/compiler-core/slang-fxc-compiler.cpp
@@ -0,0 +1,417 @@
+// slang-fxc-compiler.cpp
+#include "slang-fxc-compiler.h"
+
+#include "../core/slang-common.h"
+#include "../../slang-com-helper.h"
+
+#include "../core/slang-blob.h"
+
+#include "../core/slang-string-util.h"
+#include "../core/slang-string-slice-pool.h"
+
+#include "../core/slang-io.h"
+#include "../core/slang-shared-library.h"
+#include "../core/slang-semantic-version.h"
+#include "../core/slang-char-util.h"
+
+#include "slang-include-system.h"
+#include "slang-source-loc.h"
+
+#include "../core/slang-shared-library.h"
+
+// Enable calling through to `fxc` or `dxc` to
+// generate code on Windows.
+#ifdef _WIN32
+#   define WIN32_LEAN_AND_MEAN
+#   define NOMINMAX
+#   include <Windows.h>
+#   undef WIN32_LEAN_AND_MEAN
+#   undef NOMINMAX
+#   include <d3dcompiler.h>
+
+#   ifndef SLANG_ENABLE_DXBC_SUPPORT
+#       define SLANG_ENABLE_DXBC_SUPPORT 1
+#   endif
+#endif
+
+#ifndef SLANG_ENABLE_DXBC_SUPPORT
+#   define SLANG_ENABLE_DXBC_SUPPORT 0
+#endif
+
+#if SLANG_ENABLE_DXBC_SUPPORT
+    // Some of the `D3DCOMPILE_*` constants aren't available in all
+    // versions of `d3dcompiler.h`, so we define them here just in case
+#   ifndef D3DCOMPILE_ENABLE_UNBOUNDED_DESCRIPTOR_TABLES
+#       define D3DCOMPILE_ENABLE_UNBOUNDED_DESCRIPTOR_TABLES (1 << 20)
+#   endif
+
+#   ifndef D3DCOMPILE_ALL_RESOURCES_BOUND
+#       define D3DCOMPILE_ALL_RESOURCES_BOUND (1 << 21)
+#   endif
+#endif
+
+namespace Slang
+{
+
+#if SLANG_ENABLE_DXBC_SUPPORT
+
+static UnownedStringSlice _getSlice(ID3DBlob* blob) { return StringUtil::getSlice((ISlangBlob*)blob);  }
+
+struct FxcIncludeHandler : ID3DInclude
+{
+
+    STDMETHOD(Open)(D3D_INCLUDE_TYPE includeType, LPCSTR fileName, LPCVOID parentData, LPCVOID* outData, UINT* outSize) override
+    {
+        SLANG_UNUSED(includeType);
+        // NOTE! The pParentData means the *text* of any previous include.
+        // In order to work out what *path* that came from, we need to seach which source file it came from, and
+        // use it's path
+
+        // Assume the root pathInfo initially 
+        PathInfo includedFromPathInfo = m_rootPathInfo;
+
+        // Lets try and find the parent source if there is any
+        if (parentData)
+        {
+            SourceFile* foundSourceFile = m_system.getSourceManager()->findSourceFileByContentRecursively((const char*)parentData);
+            if (foundSourceFile)
+            {
+                includedFromPathInfo = foundSourceFile->getPathInfo();
+            }
+        }
+
+        String path(fileName);
+        PathInfo pathInfo;
+        ComPtr<ISlangBlob> blob;
+
+        SLANG_RETURN_ON_FAIL(m_system.findAndLoadFile(path, includedFromPathInfo.foundPath, pathInfo, blob));
+
+        // Return the data
+        *outData = blob->getBufferPointer();
+        *outSize = (UINT)blob->getBufferSize();
+
+        return S_OK;
+    }
+
+    STDMETHOD(Close)(LPCVOID pData) override
+    {
+        SLANG_UNUSED(pData);
+        return S_OK;
+    }
+    FxcIncludeHandler(SearchDirectoryList* searchDirectories, ISlangFileSystemExt* fileSystemExt, SourceManager* sourceManager) :
+        m_system(searchDirectories, fileSystemExt, sourceManager)
+    {
+    }
+
+    PathInfo m_rootPathInfo;
+    IncludeSystem m_system;
+};
+
+class FXCDownstreamCompiler : public DownstreamCompiler
+{
+public:
+    typedef DownstreamCompiler Super;
+
+    // DownstreamCompiler
+    virtual SlangResult compile(const CompileOptions& options, RefPtr<DownstreamCompileResult>& outResult) SLANG_OVERRIDE;
+    virtual ISlangSharedLibrary* getSharedLibrary() SLANG_OVERRIDE { return m_sharedLibrary; }
+    virtual SlangResult dissassemble(SlangCompileTarget sourceBlobTarget, const void* blob, size_t blobSize, ISlangBlob** out) SLANG_OVERRIDE;
+    virtual bool isFileBased() SLANG_OVERRIDE { return false; }
+
+        /// Must be called before use
+    SlangResult init(ISlangSharedLibrary* library);
+
+    FXCDownstreamCompiler() {}
+    
+protected:
+
+    pD3DCompile m_compile = nullptr;
+    pD3DDisassemble m_disassemble = nullptr;
+
+    ComPtr<ISlangSharedLibrary> m_sharedLibrary;  
+};
+
+SlangResult FXCDownstreamCompiler::init(ISlangSharedLibrary* library)
+{
+    m_compile = (pD3DCompile)library->findFuncByName("D3DCompile");
+    m_disassemble = (pD3DDisassemble)library->findFuncByName("D3DDisassemble");
+
+    if (!m_compile || !m_disassemble)
+    {
+        return SLANG_FAIL;
+    }
+
+    m_sharedLibrary = library;
+
+    // It's not clear how to query for a version, but we can get a version number from the header
+    m_desc = Desc(SLANG_PASS_THROUGH_FXC, D3D_COMPILER_VERSION);
+
+    return SLANG_OK;
+}
+
+static SlangResult _parseDiagnosticLine(const UnownedStringSlice& line, List<UnownedStringSlice>& lineSlices, DownstreamDiagnostic& outDiagnostic)
+{
+    /* tests/diagnostics/syntax-error-intrinsic.slang(14,2): error X3000: syntax error: unexpected token '@' */
+    if (lineSlices.getCount() < 3)
+    {
+        return SLANG_FAIL;
+    }
+
+    SLANG_RETURN_ON_FAIL(DownstreamDiagnostic::splitPathLocation(lineSlices[0], outDiagnostic));
+
+    {
+        const UnownedStringSlice severityAndCodeSlice = lineSlices[1].trim();
+        const UnownedStringSlice severitySlice = StringUtil::getAtInSplit(severityAndCodeSlice, ' ', 0);
+
+        outDiagnostic.code = StringUtil::getAtInSplit(severityAndCodeSlice, ' ', 1);
+
+        outDiagnostic.severity = DownstreamDiagnostic::Severity::Error;
+        if (severitySlice == "warning")
+        {
+            outDiagnostic.severity = DownstreamDiagnostic::Severity::Warning;
+        }
+    }
+
+    outDiagnostic.text = UnownedStringSlice(lineSlices[2].begin(), line.end());
+    return SLANG_OK;
+}
+
+static SlangResult _splitDiagnosticLine(const UnownedStringSlice& line, List<UnownedStringSlice>& outSlices)
+{
+    StringUtil::split(line, ':', outSlices);
+
+    /*
+    tests/diagnostics/syntax-error-intrinsic.slang(14,2): error X3000: syntax error: unexpected token '@'
+    */
+
+    const Int pathIndex = 0;
+
+    // Now we want to fix up a path as might have drive letter, and therefore :
+    // If this is the situation then we need to have a slice after the one at the index
+    if (outSlices.getCount() > pathIndex + 1)
+    {
+        const UnownedStringSlice pathStart = outSlices[pathIndex].trim();
+        if (pathStart.getLength() == 1 && CharUtil::isAlpha(pathStart[0]))
+        {
+            // Splice back together
+            outSlices[pathIndex] = UnownedStringSlice(outSlices[pathIndex].begin(), outSlices[pathIndex + 1].end());
+            outSlices.removeAt(pathIndex + 1);
+        }
+    }
+
+    return SLANG_OK;
+}
+
+static SlangResult _parseDiagnostics(const UnownedStringSlice& inText, List<DownstreamDiagnostic>& outDiagnostics)
+{
+    List<UnownedStringSlice> splitLine;
+
+    UnownedStringSlice text(inText), line;
+    while (StringUtil::extractLine(text, line))
+    {
+        SLANG_RETURN_ON_FAIL(_splitDiagnosticLine(line, splitLine));
+
+        DownstreamDiagnostic diagnostic;
+        diagnostic.severity = DownstreamDiagnostic::Severity::Error;
+        diagnostic.stage = DownstreamDiagnostic::Stage::Compile;
+        diagnostic.fileLine = 0;
+
+        if (SLANG_SUCCEEDED(_parseDiagnosticLine(line, splitLine, diagnostic)))
+        {
+            outDiagnostics.add(diagnostic);
+        }
+        else
+        {
+            // If couldn't parse, just add as a note
+            DownstreamDiagnostics::addNote(line, outDiagnostics);
+        }
+    }
+
+    return SLANG_OK;
+}
+
+SlangResult FXCDownstreamCompiler::compile(const CompileOptions& options, RefPtr<DownstreamCompileResult>& outResult)
+{
+    // This compiler doesn't read files, they should be read externally and stored in sourceContents/sourceContentsPath
+    if (options.sourceFiles.getCount() > 0)
+    {
+        return SLANG_FAIL;
+    }
+
+    if (options.sourceLanguage != SLANG_SOURCE_LANGUAGE_HLSL || options.targetType != SLANG_DXBC)
+    {
+        SLANG_ASSERT(!"Can only compile HLSL to DXBC");
+        return SLANG_FAIL;
+    }
+
+    // If we have been invoked in a pass-through mode, then we need to make sure
+    // that the downstream compiler sees whatever options were passed to Slang
+    // via the command line or API.
+    //
+    // TODO: more pieces of information should be added here as needed.
+    //
+
+    SearchDirectoryList searchDirectories;
+    for (const auto& includePath : options.includePaths)
+    {
+        searchDirectories.searchDirectories.add(includePath);
+    }
+
+    // Use the default fileSystemExt is not set
+    ID3DInclude* includeHandler = nullptr;
+
+    FxcIncludeHandler fxcIncludeHandlerStorage(&searchDirectories, options.fileSystemExt, options.sourceManager);
+    if (options.fileSystemExt)
+    {
+        if (options.sourceContentsPath.getLength() > 0)
+        {
+            fxcIncludeHandlerStorage.m_rootPathInfo = PathInfo::makePath(options.sourceContentsPath);
+        }
+        includeHandler = &fxcIncludeHandlerStorage;
+    }
+
+    List<D3D_SHADER_MACRO> dxMacrosStorage;
+    D3D_SHADER_MACRO const* dxMacros = nullptr;
+
+    if (options.defines.getCount() > 0)
+    {
+        for (const auto& define : options.defines)
+        {
+            D3D_SHADER_MACRO dxMacro;
+            dxMacro.Name = define.nameWithSig.getBuffer();
+            dxMacro.Definition = define.value.getBuffer();
+            dxMacrosStorage.add(dxMacro);
+        }
+        D3D_SHADER_MACRO nullTerminator = { 0, 0 };
+        dxMacrosStorage.add(nullTerminator);
+
+        dxMacros = dxMacrosStorage.getBuffer();
+    }
+
+    DWORD flags = 0;
+
+    switch (options.floatingPointMode)
+    {
+        default:
+            break;
+
+        case FloatingPointMode::Precise:
+            flags |= D3DCOMPILE_IEEE_STRICTNESS;
+            break;
+    }
+
+    flags |= D3DCOMPILE_ENABLE_STRICTNESS;
+    flags |= D3DCOMPILE_ENABLE_UNBOUNDED_DESCRIPTOR_TABLES;
+
+    switch (options.optimizationLevel)
+    {
+        default:
+            break;
+
+        case OptimizationLevel::None:       flags |= D3DCOMPILE_OPTIMIZATION_LEVEL0; break;
+        case OptimizationLevel::Default:    flags |= D3DCOMPILE_OPTIMIZATION_LEVEL1; break;
+        case OptimizationLevel::High:       flags |= D3DCOMPILE_OPTIMIZATION_LEVEL2; break;
+        case OptimizationLevel::Maximal:    flags |= D3DCOMPILE_OPTIMIZATION_LEVEL3; break;
+    }
+
+    switch (options.debugInfoType)
+    {
+        case DebugInfoType::None:
+            break;
+
+        default:
+            flags |= D3DCOMPILE_DEBUG;
+            break;
+    }
+
+    ComPtr<ID3DBlob> codeBlob;
+    ComPtr<ID3DBlob> diagnosticsBlob;
+    HRESULT hr = m_compile(
+        options.sourceContents.begin(),
+        options.sourceContents.getLength(),
+        options.sourceContentsPath.getBuffer(),
+        dxMacros,
+        includeHandler,
+        options.entryPointName.getBuffer(),
+        options.profileName.getBuffer(),
+        flags,
+        0, // unused: effect flags
+        codeBlob.writeRef(),
+        diagnosticsBlob.writeRef());
+
+    DownstreamDiagnostics diagnostics;
+
+    // HRESULT is compatible with SlangResult
+    diagnostics.result = hr;
+
+    if (diagnosticsBlob)
+    {
+        UnownedStringSlice diagnosticText = _getSlice(diagnosticsBlob);
+        diagnostics.rawDiagnostics = diagnosticText;
+        _parseDiagnostics(diagnosticText, diagnostics.diagnostics);
+    }
+
+    // ID3DBlob is compatible with ISlangBlob, so just cast away...
+    ISlangBlob* slangCodeBlob = (ISlangBlob*)codeBlob.get();
+
+    outResult = new BlobDownstreamCompileResult(diagnostics, slangCodeBlob);
+    return SLANG_OK;
+}
+
+SlangResult FXCDownstreamCompiler::dissassemble(SlangCompileTarget sourceBlobTarget, const void* blob, size_t blobSize, ISlangBlob** out)
+{
+    // Can only disassemble blobs that are DXBC
+    if (sourceBlobTarget != SLANG_DXBC)
+    {
+        return SLANG_FAIL;
+    }
+
+    ComPtr<ID3DBlob> codeBlob;
+    SLANG_RETURN_ON_FAIL(m_disassemble(blob, blobSize, 0, nullptr, codeBlob.writeRef()));
+
+    // ISlangBlob is compatible with ID3DBlob
+    *out = (ISlangBlob*)codeBlob.detach();
+    return SLANG_OK;
+}
+
+/* static */SlangResult FXCDownstreamCompilerUtil::locateCompilers(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set)
+{
+    ComPtr<ISlangSharedLibrary> library;
+
+    // If the user supplies a path to their preferred version of FXC,
+    // we just use this.
+    if (path.getLength() != 0)
+    {
+        SLANG_RETURN_ON_FAIL(loader->loadSharedLibrary(path.getBuffer(), library.writeRef()));
+    }
+    else
+    {
+        SLANG_RETURN_ON_FAIL(loader->loadSharedLibrary("d3dcompiler_47", library.writeRef()));
+    }
+
+    SLANG_ASSERT(library);
+    if (!library)
+    {
+        return SLANG_FAIL;
+    }
+
+    RefPtr<FXCDownstreamCompiler> compiler(new FXCDownstreamCompiler);
+    SLANG_RETURN_ON_FAIL(compiler->init(library));
+
+    set->addCompiler(compiler);
+    return SLANG_OK;
+}
+
+#else // SLANG_ENABLE_DXBC_SUPPORT
+
+/* static */SlangResult FXCDownstreamCompilerUtil::locateCompilers(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set)
+{
+    SLANG_UNUSED(path);
+    SLANG_UNUSED(loader);
+    SLANG_UNUSED(set);
+    return SLANG_E_NOT_AVAILABLE;
+}
+
+#endif // SLANG_ENABLE_DXBC_SUPPORT
+
+}
diff --git a/source/compiler-core/slang-fxc-compiler.h b/source/compiler-core/slang-fxc-compiler.h
new file mode 100644
index 000000000..0718f6db4
--- /dev/null
+++ b/source/compiler-core/slang-fxc-compiler.h
@@ -0,0 +1,18 @@
+#ifndef SLANG_FXC_COMPILER_UTIL_H
+#define SLANG_FXC_COMPILER_UTIL_H
+
+#include "slang-downstream-compiler.h"
+
+#include "../core/slang-platform.h"
+
+namespace Slang
+{
+
+struct FXCDownstreamCompilerUtil
+{
+    static SlangResult locateCompilers(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set);
+};
+
+}
+
+#endif
diff --git a/source/compiler-core/slang-gcc-compiler-util.cpp b/source/compiler-core/slang-gcc-compiler-util.cpp
index a51d5735f..971416ddb 100644
--- a/source/compiler-core/slang-gcc-compiler-util.cpp
+++ b/source/compiler-core/slang-gcc-compiler-util.cpp
@@ -377,18 +377,18 @@ static SlangResult _parseGCCFamilyLine(const UnownedStringSlice& line, LineParse
 
     switch (options.targetType)
     {
-        case TargetType::SharedLibrary:
+        case SLANG_SHARED_LIBRARY:
         {
             outPath << SharedLibrary::calcPlatformPath(options.modulePath.getUnownedSlice());
             return SLANG_OK;
         }
-        case TargetType::Executable:
+        case SLANG_EXECUTABLE:
         {
             outPath << options.modulePath;
             outPath << ProcessUtil::getExecutableSuffix();
             return SLANG_OK;
         }
-        case TargetType::Object:
+        case SLANG_OBJECT_CODE:
         {
 #if __CYGWIN__
             outPath << options.modulePath << ".obj";
@@ -509,7 +509,7 @@ static SlangResult _parseGCCFamilyLine(const UnownedStringSlice& line, LineParse
 
     switch (options.targetType)
     {
-        case TargetType::SharedLibrary:
+        case SLANG_SHARED_LIBRARY:
         {
             // Shared library
             cmdLine.addArg("-shared");
@@ -521,11 +521,11 @@ static SlangResult _parseGCCFamilyLine(const UnownedStringSlice& line, LineParse
             }
             break;
         }
-        case TargetType::Executable:
+        case SLANG_EXECUTABLE:
         {
             break;
         }
-        case TargetType::Object:
+        case SLANG_OBJECT_CODE:
         {
             // Don't link, just produce object file
             cmdLine.addArg("-c");
@@ -563,7 +563,7 @@ static SlangResult _parseGCCFamilyLine(const UnownedStringSlice& line, LineParse
         //cmdLine.addArg(linkOptions);
     }
 
-    if (options.targetType == TargetType::SharedLibrary)
+    if (options.targetType == SLANG_SHARED_LIBRARY)
     {
         if (!PlatformUtil::isFamily(PlatformFamily::Apple, platformKind))
         {
diff --git a/source/compiler-core/slang-include-system.cpp b/source/compiler-core/slang-include-system.cpp
new file mode 100644
index 000000000..891d376f6
--- /dev/null
+++ b/source/compiler-core/slang-include-system.cpp
@@ -0,0 +1,158 @@
+// slang-include-system.cpp
+#include "slang-include-system.h"
+
+#include "../core/slang-io.h"
+#include "../core/slang-string-util.h"
+
+namespace Slang
+{
+
+SlangResult IncludeSystem::findFile(SlangPathType fromPathType, const String& fromPath, const String& path, PathInfo& outPathInfo)
+{
+    String combinedPath;
+
+    if (fromPath.getLength() == 0 || Path::isAbsolute(path))
+    {
+        // If the path is absolute or the fromPath is empty, the combined path is just the path
+        combinedPath = path;
+    }
+    else
+    {
+        // Get relative path
+        ComPtr<ISlangBlob> combinedPathBlob;
+        SLANG_RETURN_ON_FAIL(m_fileSystemExt->calcCombinedPath(fromPathType, fromPath.begin(), path.begin(), combinedPathBlob.writeRef()));
+        combinedPath = StringUtil::getString(combinedPathBlob);
+        if (combinedPath.getLength() <= 0)
+        {
+            return SLANG_FAIL;
+        }
+    }
+
+    // This checks the path exists
+    SlangPathType pathType;
+    SLANG_RETURN_ON_FAIL(m_fileSystemExt->getPathType(combinedPath.begin(), &pathType));
+    if (pathType != SLANG_PATH_TYPE_FILE)
+    {
+        return SLANG_E_NOT_FOUND;
+    }
+
+    // Get the uniqueIdentity
+    ComPtr<ISlangBlob> uniqueIdentityBlob;
+    SLANG_RETURN_ON_FAIL(m_fileSystemExt->getFileUniqueIdentity(combinedPath.begin(), uniqueIdentityBlob.writeRef()));
+
+    // If the rel path exists -> a uniqueIdentity MUST exists too
+    String uniqueIdentity(StringUtil::getString(uniqueIdentityBlob));
+    if (uniqueIdentity.getLength() <= 0)
+    {
+        // Unique identity can't be empty
+        return SLANG_FAIL;
+    }
+
+    outPathInfo = PathInfo::makeNormal(combinedPath, uniqueIdentity);
+    return SLANG_OK;
+}
+
+String IncludeSystem::simplifyPath(const String& path)
+{
+    ComPtr<ISlangBlob> simplifiedPath;
+    if (SLANG_FAILED(m_fileSystemExt->getSimplifiedPath(path.getBuffer(), simplifiedPath.writeRef())))
+    {
+        return path;
+    }
+    return StringUtil::getString(simplifiedPath);
+}
+
+SlangResult IncludeSystem::findFile(String const& pathToInclude, String const& pathIncludedFrom, PathInfo& outPathInfo)
+{
+    outPathInfo.type = PathInfo::Type::Unknown;
+
+    // If it's absolute we only have to try and find if it's there - no need to look at search paths
+    if (Path::isAbsolute(pathToInclude))
+    {
+        // We pass in "" as the from path, so ensure no from path is taken into account
+        // and to allow easy identification that this is in effect absolute
+        return findFile(SLANG_PATH_TYPE_DIRECTORY, UnownedStringSlice::fromLiteral(""), pathToInclude, outPathInfo);
+    }
+
+    // Try just relative to current path
+    {
+        SlangResult res = findFile(SLANG_PATH_TYPE_FILE, pathIncludedFrom, pathToInclude, outPathInfo);
+        // It either succeeded or wasn't found, anything else is a failure passed back
+        if (SLANG_SUCCEEDED(res) || res != SLANG_E_NOT_FOUND)
+        {
+            return res;
+        }
+    }
+
+    // Search all the searchDirectories
+    for (auto sd = m_searchDirectories; sd; sd = sd->parent)
+    {
+        for (auto& dir : sd->searchDirectories)
+        {
+            SlangResult res = findFile(SLANG_PATH_TYPE_DIRECTORY, dir.path, pathToInclude, outPathInfo);
+            if (SLANG_SUCCEEDED(res) || res != SLANG_E_NOT_FOUND)
+            {
+                return res;
+            }
+        }
+    }
+
+    return SLANG_E_NOT_FOUND;
+}
+
+SlangResult IncludeSystem::loadFile(const PathInfo& pathInfo, ComPtr<ISlangBlob>& outBlob)
+{
+    if (m_sourceManager)
+    {
+        // See if this an already loaded source file
+        SourceFile* sourceFile = m_sourceManager->findSourceFileRecursively(pathInfo.uniqueIdentity);
+
+        // If not create a new one, and add to the list of known source files
+        if (!sourceFile)
+        {
+            ComPtr<ISlangBlob> foundSourceBlob;
+            if (SLANG_FAILED(m_fileSystemExt->loadFile(pathInfo.foundPath.getBuffer(), foundSourceBlob.writeRef())))
+            {
+                return SLANG_E_CANNOT_OPEN;
+            }
+
+            sourceFile = m_sourceManager->createSourceFileWithBlob(pathInfo, foundSourceBlob);
+            m_sourceManager->addSourceFile(pathInfo.uniqueIdentity, sourceFile);
+
+            outBlob = foundSourceBlob;
+            return SLANG_OK;
+        }
+        else
+        {
+            if (sourceFile->getContentBlob())
+            {
+                outBlob = sourceFile->getContentBlob();
+                return SLANG_OK;
+            }
+
+            ComPtr<ISlangBlob> foundSourceBlob;
+            if (SLANG_FAILED(m_fileSystemExt->loadFile(pathInfo.foundPath.getBuffer(), foundSourceBlob.writeRef())))
+            {
+                return SLANG_E_CANNOT_OPEN;
+            }
+
+            sourceFile->setContents(foundSourceBlob);
+            outBlob = foundSourceBlob;
+            return SLANG_OK;
+        }
+    }
+    else
+    {
+        // If we don't have the source manager, just load
+        return m_fileSystemExt->loadFile(pathInfo.foundPath.getBuffer(), outBlob.writeRef());
+    }
+}
+
+SlangResult IncludeSystem::findAndLoadFile(const String& pathToInclude, const String& pathIncludedFrom, PathInfo& outPathInfo, ComPtr<ISlangBlob>& outBlob)
+{
+    SLANG_RETURN_ON_FAIL(findFile(pathToInclude, pathIncludedFrom, outPathInfo));
+    SLANG_RETURN_ON_FAIL(loadFile(outPathInfo, outBlob));
+    return SLANG_OK;
+}
+
+} // namespace Slang
diff --git a/source/compiler-core/slang-include-system.h b/source/compiler-core/slang-include-system.h
new file mode 100644
index 000000000..70f6dd81e
--- /dev/null
+++ b/source/compiler-core/slang-include-system.h
@@ -0,0 +1,62 @@
+#ifndef SLANG_INCLUDE_SYSTEM_H
+#define SLANG_INCLUDE_SYSTEM_H
+// slang-include-system.h
+
+#include "../compiler-core/slang-source-loc.h"
+
+namespace Slang
+{
+
+// A directory to be searched when looking for files (e.g., `#include`)
+struct SearchDirectory
+{
+    SearchDirectory() = default;
+    SearchDirectory(SearchDirectory const& other) = default;
+    SearchDirectory(String const& path)
+        : path(path)
+    {}
+
+    String path;
+};
+
+/// A list of directories to search for files (e.g., `#include`)
+struct SearchDirectoryList
+{
+    // A parent list that should also be searched
+    SearchDirectoryList*    parent = nullptr;
+
+    // Directories to be searched
+    List<SearchDirectory>   searchDirectories;
+};
+
+/* A helper class that builds basic include handling on top of searchDirectories/fileSystemExt and optionally a sourceManager */
+struct IncludeSystem
+{
+    IncludeSystem(SearchDirectoryList* searchDirectories, ISlangFileSystemExt* fileSystemExt, SourceManager* sourceManager = nullptr) :
+        m_searchDirectories(searchDirectories),
+        m_fileSystemExt(fileSystemExt),
+        m_sourceManager(sourceManager)
+    {
+    }
+
+    SlangResult findFile(const String& pathToInclude, const String& pathIncludedFrom, PathInfo& outPathInfo);
+    SlangResult findFile(SlangPathType fromPathType, const String& fromPath, const String& path, PathInfo& outPathInfo);
+    String simplifyPath(const String& path);
+    SlangResult loadFile(const PathInfo& pathInfo, ComPtr<ISlangBlob>& outBlob);
+
+    SlangResult findAndLoadFile(const String& pathToInclude, const String& pathIncludedFrom, PathInfo& outPathInfo, ComPtr<ISlangBlob>& outBlob);
+
+    SearchDirectoryList* getSearchDirectoryList() const { return m_searchDirectories; }
+    ISlangFileSystemExt* getFileSystem() const { return m_fileSystemExt; }
+    SourceManager* getSourceManager() const { return m_sourceManager; }
+
+protected:
+    
+    SearchDirectoryList* m_searchDirectories;
+    ISlangFileSystemExt* m_fileSystemExt;
+    SourceManager* m_sourceManager;                 ///< If not set, will not look up the content in the source manager
+};
+
+}
+
+#endif  // SLANG_INCLUDE_HANDLER_H
diff --git a/source/compiler-core/slang-nvrtc-compiler.cpp b/source/compiler-core/slang-nvrtc-compiler.cpp
index 79280e033..029f3c71b 100644
--- a/source/compiler-core/slang-nvrtc-compiler.cpp
+++ b/source/compiler-core/slang-nvrtc-compiler.cpp
@@ -100,6 +100,7 @@ public:
     // DownstreamCompiler
     virtual SlangResult compile(const CompileOptions& options, RefPtr<DownstreamCompileResult>& outResult) SLANG_OVERRIDE;
     virtual ISlangSharedLibrary* getSharedLibrary() SLANG_OVERRIDE { return m_sharedLibrary; }
+    virtual bool isFileBased() SLANG_OVERRIDE { return false; }
 
         /// Must be called before use
     SlangResult init(ISlangSharedLibrary* library);
diff --git a/source/compiler-core/slang-visual-studio-compiler-util.cpp b/source/compiler-core/slang-visual-studio-compiler-util.cpp
index fd879e63f..e2c488ee9 100644
--- a/source/compiler-core/slang-visual-studio-compiler-util.cpp
+++ b/source/compiler-core/slang-visual-studio-compiler-util.cpp
@@ -23,17 +23,17 @@ namespace Slang
 
     switch (options.targetType)
     {
-        case TargetType::SharedLibrary:
+        case SLANG_SHARED_LIBRARY:
         {
             outPath << options.modulePath << ".dll";
             return SLANG_OK;
         }
-        case TargetType::Executable:
+        case SLANG_EXECUTABLE:
         {
             outPath << options.modulePath << ".exe";
             return SLANG_OK;
         }
-        case TargetType::Object:
+        case SLANG_OBJECT_CODE:
         {
             outPath << options.modulePath << ".obj";
             return SLANG_OK;
@@ -60,7 +60,7 @@ namespace Slang
     {
         outPaths.add(options.modulePath + ".ilk");
 
-        if (options.targetType == TargetType::SharedLibrary)
+        if (options.targetType == SLANG_SHARED_LIBRARY)
         {
             outPaths.add(options.modulePath + ".exp");
             outPaths.add(options.modulePath + ".lib");
@@ -190,7 +190,7 @@ namespace Slang
 
     switch (options.targetType)
     {
-        case TargetType::SharedLibrary:
+        case SLANG_SHARED_LIBRARY:
         {
             // Create dynamic link library
             if (options.debugInfoType == DebugInfoType::None)
@@ -205,7 +205,7 @@ namespace Slang
             cmdLine.addPrefixPathArg("/Fe", options.modulePath, ".dll");
             break;
         }
-        case TargetType::Executable:
+        case SLANG_EXECUTABLE:
         {
             cmdLine.addPrefixPathArg("/Fe", options.modulePath, ".exe");
             break;