7 files changed, 388 insertions, 40 deletions

diff --git a/source/slang/compiler.cpp b/source/slang/compiler.cpp
index 66df01db0..9c1823f91 100644
--- a/source/slang/compiler.cpp
+++ b/source/slang/compiler.cpp
@@ -52,7 +52,7 @@
 
 // Includes to allow us to control console
 // output when writing assembly dumps.
-#include <fcntl.h> 
+#include <fcntl.h>
 #ifdef _WIN32
 #include <io.h>
 #else
@@ -97,6 +97,28 @@ namespace Slang
         }
     }
 
+    ComPtr<ISlangBlob> CompileResult::getBlob()
+    {
+        if(!blob)
+        {
+            switch(format)
+            {
+            case ResultFormat::None:
+            default:
+                break;
+
+            case ResultFormat::Text:
+                blob = createStringBlob(outputString);
+                break;
+
+            case ResultFormat::Binary:
+                blob = createRawBlob(outputBinary.Buffer(), outputBinary.Count());
+                break;
+            }
+        }
+        return blob;
+    }
+
     // EntryPointRequest
 
     TranslationUnitRequest* EntryPointRequest::getTranslationUnit()
@@ -149,6 +171,7 @@ namespace Slang
                     }
                 }
                 codeBuilder << "\"\n";
+
                 codeBuilder << sourceFile->content << "\n";
             }
 
@@ -886,7 +909,7 @@ String dissassembleDXILUsingDXC(
     void emitEntryPoints(
         TargetRequest*          /*targetReq*/)
     {
-        
+
     }
 
     void generateOutputForTarget(
diff --git a/source/slang/compiler.h b/source/slang/compiler.h
index 3a5f888fe..acbad9a7e 100644
--- a/source/slang/compiler.h
+++ b/source/slang/compiler.h
@@ -2,6 +2,7 @@
 #define RASTER_SHADER_COMPILER_H
 
 #include "../core/basic.h"
+#include "../core/slang-com-ptr.h"
 
 #include "diagnostics.h"
 #include "name.h"
@@ -99,9 +100,13 @@ namespace Slang
 
         void append(CompileResult const& result);
 
+        ComPtr<ISlangBlob> getBlob();
+
         ResultFormat format = ResultFormat::None;
         String outputString;
         List<uint8_t> outputBinary;
+
+        ComPtr<ISlangBlob> blob;
     };
 
     // Describes an entry point that we've been requested to compile
@@ -281,6 +286,14 @@ namespace Slang
 
     class Session;
 
+    /// Create a blob that will retain a string for its lifetime.
+    ///
+    ComPtr<ISlangBlob> createStringBlob(String const& string);
+
+    /// Create a blob that will retain (a copy of) raw data.
+    ///
+    ComPtr<ISlangBlob> createRawBlob(void const* data, size_t size);
+
     class CompileRequest : public RefObject
     {
     public:
@@ -348,6 +361,9 @@ namespace Slang
         DiagnosticSink mSink;
         String mDiagnosticOutput;
 
+        /// A blob holding the diagnostic output
+        ComPtr<ISlangBlob> diagnosticOutputBlob;
+
         // Files that compilation depended on
         List<String> mDependencyFilePaths;
 
@@ -368,6 +384,21 @@ namespace Slang
         // The resulting specialized IR module for each entry point request
         List<RefPtr<IRModule>> compiledModules;
 
+        /// File system implementation to use when loading files from disk.
+        ///
+        /// If this member is `null`, a default implementation that tries
+        /// to use the native OS filesystem will be used instead.
+        ///
+        ComPtr<ISlangFileSystem> fileSystem;
+
+        /// Load a file into memory using the configured file system.
+        ///
+        /// @param path The path to attempt to load from
+        /// @param outBlob A destination pointer to receive the loaded blob
+        /// @returns A `SlangResult` to indicate success or failure.
+        ///
+        SlangResult loadFile(String const& path, ISlangBlob** outBlob);
+
         CompileRequest(Session* session);
 
         ~CompileRequest();
@@ -394,6 +425,11 @@ namespace Slang
             int             translationUnitIndex,
             SourceFile*     sourceFile);
 
+        void addTranslationUnitSourceBlob(
+            int             translationUnitIndex,
+            String const&   path,
+            ISlangBlob*     sourceBlob);
+
         void addTranslationUnitSourceString(
             int             translationUnitIndex,
             String const&   path,
@@ -415,7 +451,7 @@ namespace Slang
         RefPtr<ModuleDecl> loadModule(
             Name*               name,
             String const&       path,
-            String const&       source,
+            ISlangBlob*         sourceBlob,
             SourceLoc const& loc);
 
         void loadParsedModule(
diff --git a/source/slang/preprocessor.cpp b/source/slang/preprocessor.cpp
index 301264632..3ec44fc28 100644
--- a/source/slang/preprocessor.cpp
+++ b/source/slang/preprocessor.cpp
@@ -1581,7 +1581,7 @@ static void HandleIncludeDirective(PreprocessorDirectiveContext* context)
     auto expandedDirectiveLoc = context->preprocessor->translationUnit->compileRequest->getSourceManager()->expandSourceLoc(directiveLoc);
     String pathIncludedFrom = expandedDirectiveLoc.getSpellingPath();
     String foundPath;
-    String foundSource;
+    ComPtr<ISlangBlob> foundSourceBlob;
 
     IncludeHandler* includeHandler = context->preprocessor->includeHandler;
     if (!includeHandler)
@@ -1590,7 +1590,7 @@ static void HandleIncludeDirective(PreprocessorDirectiveContext* context)
         GetSink(context)->diagnose(pathToken.loc, Diagnostics::noIncludeHandlerSpecified);
         return;
     }
-    auto includeResult = includeHandler->TryToFindIncludeFile(path, pathIncludedFrom, &foundPath, &foundSource);
+    auto includeResult = includeHandler->TryToFindIncludeFile(path, pathIncludedFrom, &foundPath, foundSourceBlob.writeRef());
 
     switch (includeResult)
     {
@@ -1611,7 +1611,7 @@ static void HandleIncludeDirective(PreprocessorDirectiveContext* context)
     // Push the new file onto our stack of input streams
     // TODO(tfoley): check if we have made our include stack too deep
 
-    SourceFile* sourceFile = context->preprocessor->getCompileRequest()->getSourceManager()->allocateSourceFile(foundPath, foundSource);
+    SourceFile* sourceFile = context->preprocessor->getCompileRequest()->getSourceManager()->allocateSourceFile(foundPath, foundSourceBlob);
 
     PreprocessorInputStream* inputStream = CreateInputStreamForSource(context->preprocessor, sourceFile);
     inputStream->parent = context->preprocessor->inputStream;
diff --git a/source/slang/preprocessor.h b/source/slang/preprocessor.h
index 64591ef03..0e30038bf 100644
--- a/source/slang/preprocessor.h
+++ b/source/slang/preprocessor.h
@@ -26,7 +26,7 @@ struct IncludeHandler
         String const& pathToInclude,
         String const& pathIncludedFrom,
         String* outFoundPath,
-        String* outFoundSource) = 0;
+        ISlangBlob** outFoundSourceBlob) = 0;
 };
 
 // Take a string of source code and preprocess it into a list of tokens.
diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp
index 2d3e9ee77..24dff88a8 100644
--- a/source/slang/slang.cpp
+++ b/source/slang/slang.cpp
@@ -81,32 +81,29 @@ struct IncludeHandlerImpl : IncludeHandler
         String const& pathToInclude,
         String const& pathIncludedFrom,
         String* outFoundPath,
-        String* outFoundSource) override
+        ISlangBlob** outFoundSourceBlob) override
     {
         String path = Path::Combine(Path::GetDirectoryName(pathIncludedFrom), pathToInclude);
-        if (File::Exists(path))
+
+        if(SLANG_SUCCEEDED(request->loadFile(path, outFoundSourceBlob)))
         {
             *outFoundPath = path;
-            *outFoundSource = File::ReadAllText(path);
-
             request->mDependencyFilePaths.Add(path);
-
             return IncludeResult::Found;
         }
 
         for (auto & dir : request->searchDirectories)
         {
             path = Path::Combine(dir.path, pathToInclude);
-            if (File::Exists(path))
+
+            if(SLANG_SUCCEEDED(request->loadFile(path, outFoundSourceBlob)))
             {
                 *outFoundPath = path;
-                *outFoundSource = File::ReadAllText(path);
-
                 request->mDependencyFilePaths.Add(path);
-
                 return IncludeResult::Found;
             }
         }
+
         return IncludeResult::NotFound;
     }
 };
@@ -205,11 +202,168 @@ CompileRequest::~CompileRequest()
     types = decltype(types)();
 }
 
+// Allocate static const storage for the various interface IDs that the Slang API needs to expose
+static const Guid IID_ISlangUnknown = SLANG_UUID_ISlangUnknown;
+static const Guid IID_ISlangBlob    = SLANG_UUID_ISlangBlob;
+
+/** Base class for simple blobs.
+*/
+class BlobBase : public ISlangBlob
+{
+public:
+    BlobBase() {}
+    virtual ~BlobBase() {}
+
+    uint32_t referenceCount = 0;
+
+    // ISlangUnknown
+
+    SLANG_NO_THROW SlangResult SLANG_MCALL queryInterface(SlangUUID const& uuid, void** outObject) SLANG_OVERRIDE
+    {
+        if(uuid == IID_IComUnknown)
+        {
+            *(ISlangUnknown**)outObject = this;
+            addRef();
+            return SLANG_OK;
+        }
+        else if(uuid == IID_ISlangBlob)
+        {
+            *(ISlangBlob**)outObject = this;
+            addRef();
+            return SLANG_OK;
+        }
+
+        return SLANG_FAIL;
+    }
+
+    SLANG_NO_THROW uint32_t SLANG_MCALL addRef() SLANG_OVERRIDE
+    {
+        referenceCount++;
+        return 0;
+    }
+
+    SLANG_NO_THROW uint32_t SLANG_MCALL release() SLANG_OVERRIDE
+    {
+        if(--referenceCount == 0)
+        {
+            delete this;
+        }
+        return 0;
+    }
+
+};
+
+/** A blob that uses a `String` for its storage.
+*/
+class StringBlob : public BlobBase
+{
+public:
+    String string;
+
+    explicit StringBlob(String const& string)
+        : string(string)
+    {}
+
+    // ISlangBlob
+
+    SLANG_NO_THROW void const* SLANG_MCALL getBufferPointer() SLANG_OVERRIDE
+    {
+        return string.Buffer();
+    }
+
+    SLANG_NO_THROW size_t SLANG_MCALL getBufferSize() SLANG_OVERRIDE
+    {
+        return string.Length();
+    }
+};
+
+ComPtr<ISlangBlob> createStringBlob(String const& string)
+{
+    return ComPtr<ISlangBlob>(new StringBlob(string));
+}
+
+/** A blob that manages some raw data that it owns.
+*/
+class RawBlob : public BlobBase
+{
+public:
+    void* data;
+    size_t size;
+
+    ~RawBlob()
+    {
+        free(data);
+    }
+
+    // ISlangBlob
+
+    SLANG_NO_THROW void const* SLANG_MCALL getBufferPointer() SLANG_OVERRIDE
+    {
+        return data;
+    }
+
+    SLANG_NO_THROW size_t SLANG_MCALL getBufferSize() SLANG_OVERRIDE
+    {
+        return size;
+    }
+};
+
+
+ComPtr<ISlangBlob> createRawBlob(void const* inData, size_t size)
+{
+    void* dataCopy = malloc(size);
+    memcpy(dataCopy, inData, size);
+
+    RawBlob* rawBlob = new RawBlob();
+    rawBlob->data = dataCopy;
+    rawBlob->size = size;
+
+    return ComPtr<ISlangBlob>(rawBlob);
+}
+
+
+SlangResult CompileRequest::loadFile(String const& path, ISlangBlob** outBlob)
+{
+    // If there is a used-defined filesystem, then use that to load files.
+    //
+    if(fileSystem)
+    {
+        return fileSystem->loadFile(path.Buffer(), outBlob);
+    }
+
+    // Otherwise, fall back to a default implementation that uses the `core`
+    // libraries facilities for talking to the OS filesystem.
+    //
+    // TODO: we might want to conditionally compile these in, so that
+    // a user could create a build of Slang that doesn't include any OS
+    // filesystem calls.
+    //
+
+    if (!File::Exists(path))
+    {
+        return SLANG_FAIL;
+    }
+
+    try
+    {
+        String sourceString = File::ReadAllText(path);
+        ComPtr<ISlangBlob> sourceBlob = createStringBlob(sourceString);
+        *outBlob = sourceBlob.detach();
+
+        return SLANG_OK;
+    }
+    catch(...)
+    {
+    }
+    return SLANG_FAIL;
+
+}
+
 
 RefPtr<Expr> CompileRequest::parseTypeString(TranslationUnitRequest * translationUnit, String typeStr, RefPtr<Scope> scope)
 {
     Slang::SourceFile srcFile;
-    srcFile.content = typeStr;
+    srcFile.content = UnownedStringSlice(typeStr.begin(), typeStr.end());
     DiagnosticSink sink;
     sink.sourceManager = sourceManager;
     auto tokens = preprocessSource(
@@ -484,6 +638,16 @@ void CompileRequest::addTranslationUnitSourceFile(
     translationUnits[translationUnitIndex]->sourceFiles.Add(sourceFile);
 }
 
+void CompileRequest::addTranslationUnitSourceBlob(
+    int             translationUnitIndex,
+    String const&   path,
+    ISlangBlob*     sourceBlob)
+{
+    RefPtr<SourceFile> sourceFile = getSourceManager()->allocateSourceFile(path, sourceBlob);
+
+    addTranslationUnitSourceFile(translationUnitIndex, sourceFile);
+}
+
 void CompileRequest::addTranslationUnitSourceString(
     int             translationUnitIndex,
     String const&   path,
@@ -498,12 +662,17 @@ void CompileRequest::addTranslationUnitSourceFile(
     int             translationUnitIndex,
     String const&   path)
 {
-    String source;
-    try
-    {
-        source = File::ReadAllText(path);
-    }
-    catch (...)
+    // TODO: We need to consider whether a relative `path` should cause
+    // us to look things up using the registered search paths.
+    //
+    // This behavior wouldn't make sense for command-line invocations
+    // of `slangc`, but at least one API user wondered by the search
+    // paths were not taken into account by this function.
+    //
+
+    ComPtr<ISlangBlob> sourceBlob;
+    SlangResult result = loadFile(path, sourceBlob.writeRef());
+    if(SLANG_FAILED(result))
     {
         // Emit a diagnostic!
         mSink.diagnose(
@@ -513,10 +682,10 @@ void CompileRequest::addTranslationUnitSourceFile(
         return;
     }
 
-    addTranslationUnitSourceString(
+    addTranslationUnitSourceBlob(
         translationUnitIndex,
         path,
-        source);
+        sourceBlob);
 
     mDependencyFilePaths.Add(path);
 }
@@ -590,7 +759,7 @@ void CompileRequest::loadParsedModule(
 RefPtr<ModuleDecl> CompileRequest::loadModule(
     Name*               name,
     String const&       path,
-    String const&       source,
+    ISlangBlob*         sourceBlob,
     SourceLoc const&    srcLoc)
 {
     RefPtr<TranslationUnitRequest> translationUnit = new TranslationUnitRequest();
@@ -603,7 +772,7 @@ RefPtr<ModuleDecl> CompileRequest::loadModule(
     // TODO: decide which options, if any, should be inherited.
     translationUnit->compileFlags = 0;
 
-    RefPtr<SourceFile> sourceFile = getSourceManager()->allocateSourceFile(path, source);
+    RefPtr<SourceFile> sourceFile = getSourceManager()->allocateSourceFile(path, sourceBlob);
 
     translationUnit->sourceFiles.Add(sourceFile);
 
@@ -687,8 +856,8 @@ RefPtr<ModuleDecl> CompileRequest::findOrImportModule(
     String pathIncludedFrom = expandedLoc.getSpellingPath();
 
     String foundPath;
-    String foundSource;
-    IncludeResult includeResult = includeHandler.TryToFindIncludeFile(fileName, pathIncludedFrom, &foundPath, &foundSource);
+    ComPtr<ISlangBlob> foundSourceBlob;
+    IncludeResult includeResult = includeHandler.TryToFindIncludeFile(fileName, pathIncludedFrom, &foundPath, foundSourceBlob.writeRef());
     switch( includeResult )
     {
     case IncludeResult::NotFound:
@@ -715,7 +884,7 @@ RefPtr<ModuleDecl> CompileRequest::findOrImportModule(
     return loadModule(
         name,
         foundPath,
-        foundSource,
+        foundSourceBlob,
         loc);
 }
 
@@ -832,7 +1001,7 @@ Session::~Session()
     constExprRate = nullptr;
 
     destroyTypeCheckingCache();
-    
+
     builtinTypes = decltype(builtinTypes)();
     // destroy modules next
     loadedModuleCode = decltype(loadedModuleCode)();
@@ -895,6 +1064,16 @@ SLANG_API void spDestroyCompileRequest(
     delete req;
 }
 
+SLANG_API void spSetFileSystem(
+    SlangCompileRequest*    request,
+    ISlangFileSystem*       fileSystem)
+{
+    if(!request) return;
+    auto req = REQ(request);
+    req->fileSystem = fileSystem;
+}
+
+
 SLANG_API void spSetCompileFlags(
     SlangCompileRequest*    request,
     SlangCompileFlags       flags)
@@ -1021,6 +1200,25 @@ SLANG_API char const* spGetDiagnosticOutput(
     return req->mDiagnosticOutput.begin();
 }
 
+SLANG_API SlangResult spGetDiagnosticOutputBlob(
+        SlangCompileRequest*    request,
+        ISlangBlob**            outBlob)
+{
+    if(!request) return SLANG_ERROR_INVALID_PARAMETER;
+    if(!outBlob) return SLANG_ERROR_INVALID_PARAMETER;
+
+    auto req = REQ(request);
+
+    if(!req->diagnosticOutputBlob)
+    {
+        req->diagnosticOutputBlob = Slang::createStringBlob(req->mDiagnosticOutput);
+    }
+
+    Slang::ComPtr<ISlangBlob> resultBlob = req->diagnosticOutputBlob;
+    *outBlob = resultBlob.detach();
+    return SLANG_OK;
+}
+
 // New-fangled compilation API
 
 SLANG_API int spAddTranslationUnit(
@@ -1063,16 +1261,31 @@ SLANG_API void spAddTranslationUnitSourceFile(
         path);
 }
 
-// Add a source string to the given translation unit
 SLANG_API void spAddTranslationUnitSourceString(
     SlangCompileRequest*    request,
     int                     translationUnitIndex,
     char const*             path,
     char const*             source)
 {
+    if(!source) return;
+    spAddTranslationUnitSourceStringSpan(
+        request,
+        translationUnitIndex,
+        path,
+        source,
+        source + strlen(source));
+}
+
+SLANG_API void spAddTranslationUnitSourceStringSpan(
+    SlangCompileRequest*    request,
+    int                     translationUnitIndex,
+    char const*             path,
+    char const*             sourceBegin,
+    char const*             sourceEnd)
+{
     if(!request) return;
     auto req = REQ(request);
-    if(!source) return;
+    if(!sourceBegin) return;
     if(translationUnitIndex < 0) return;
     if(Slang::UInt(translationUnitIndex) >= req->translationUnits.Count()) return;
 
@@ -1081,10 +1294,34 @@ SLANG_API void spAddTranslationUnitSourceString(
     req->addTranslationUnitSourceString(
         translationUnitIndex,
         path,
-        source);
+        Slang::UnownedStringSlice(sourceBegin, sourceEnd));
+}
 
+SLANG_API void spAddTranslationUnitSourceBlob(
+    SlangCompileRequest*    request,
+    int                     translationUnitIndex,
+    char const*             path,
+    ISlangBlob*             sourceBlob)
+{
+    if(!request) return;
+    auto req = REQ(request);
+    if(!sourceBlob) return;
+    if(translationUnitIndex < 0) return;
+    if(Slang::UInt(translationUnitIndex) >= req->translationUnits.Count()) return;
+
+    if(!path) path = "";
+
+    req->addTranslationUnitSourceBlob(
+        translationUnitIndex,
+        path,
+        sourceBlob);
 }
 
+
+
+
+
+
 SLANG_API SlangProfileID spFindProfile(
     SlangSession*,
     char const*     name)
@@ -1267,6 +1504,35 @@ SLANG_API void const* spGetEntryPointCode(
     return data;
 }
 
+SLANG_API SlangResult spGetEntryPointCodeBlob(
+        SlangCompileRequest*    request,
+        int                     entryPointIndex,
+        int                     targetIndex,
+        ISlangBlob**            outBlob)
+{
+    if(!request) return SLANG_ERROR_INVALID_PARAMETER;
+    if(!outBlob) return SLANG_ERROR_INVALID_PARAMETER;
+
+    auto req = REQ(request);
+
+    int targetCount = (int) req->targets.Count();
+    if((targetIndex < 0) || (targetIndex >= targetCount))
+    {
+        return SLANG_ERROR_INVALID_PARAMETER;
+    }
+    auto targetReq = req->targets[targetIndex];
+
+    int entryPointCount = (int) req->entryPoints.Count();
+    if((entryPointIndex < 0) || (entryPointIndex >= entryPointCount))
+    {
+        return SLANG_ERROR_INVALID_PARAMETER;
+    }
+    Slang::CompileResult& result = targetReq->entryPointResults[entryPointIndex];
+
+    *outBlob = result.getBlob().detach();
+    return SLANG_OK;
+}
+
 SLANG_API char const* spGetEntryPointSource(
     SlangCompileRequest*    request,
     int                     entryPointIndex)
diff --git a/source/slang/source-loc.cpp b/source/slang/source-loc.cpp
index 80556ac37..3042c3bd4 100644
--- a/source/slang/source-loc.cpp
+++ b/source/slang/source-loc.cpp
@@ -1,6 +1,8 @@
 // source-loc.cpp
 #include "source-loc.h"
 
+#include "compiler.h"
+
 namespace Slang {
 
 String ExpandedSourceLoc::getPath() const
@@ -70,15 +72,18 @@ SourceRange SourceManager::allocateSourceRange(UInt size)
 
 SourceFile* SourceManager::allocateSourceFile(
     String const&   path,
-    String const&   content)
+    ISlangBlob*     contentBlob)
 {
-    UInt size = content.Length();
+    char const* contentBegin = (char const*) contentBlob->getBufferPointer();
+    UInt contentSize = contentBlob->getBufferSize();
+    char const* contentEnd = contentBegin + contentSize;
 
-    SourceRange sourceRange = allocateSourceRange(size);
+    SourceRange sourceRange = allocateSourceRange(contentSize);
 
     SourceFile* sourceFile = new SourceFile();
     sourceFile->path = path;
-    sourceFile->content = content;
+    sourceFile->contentBlob = contentBlob;
+    sourceFile->content = UnownedStringSlice(contentBegin, contentEnd);
     sourceFile->sourceRange = sourceRange;
 
     Entry entry;
@@ -91,6 +96,14 @@ SourceFile* SourceManager::allocateSourceFile(
     return sourceFile;
 }
 
+SourceFile* SourceManager::allocateSourceFile(
+    String const&   path,
+    String const&   content)
+{
+    ComPtr<ISlangBlob> contentBlob = createStringBlob(content);
+    return allocateSourceFile(path, contentBlob);
+}
+
 SourceLoc SourceManager::allocateSourceFileForLineDirective(
     SourceLoc const&    directiveLoc,
     String const&       path,
diff --git a/source/slang/source-loc.h b/source/slang/source-loc.h
index 83ad87633..88ec6b62a 100644
--- a/source/slang/source-loc.h
+++ b/source/slang/source-loc.h
@@ -3,6 +3,9 @@
 #define SLANG_SOURCE_LOC_H_INCLUDED
 
 #include "../core/basic.h"
+#include "../core/slang-com-ptr.h"
+
+#include "../../slang.h"
 
 namespace Slang {
 
@@ -73,8 +76,11 @@ public:
     // The logical file path to report for locations inside this span.
     String path;
 
-    // The actual contents of the file.
-    String content;
+    /// A blob that owns the storage for the file contents
+    ComPtr<ISlangBlob> contentBlob;
+
+    /// The actual contents of the file.
+    UnownedStringSlice content;
 
     // The range of source locations that the span covers
     SourceRange sourceRange;
@@ -134,6 +140,10 @@ struct SourceManager
 
     SourceFile* allocateSourceFile(
         String const&   path,
+        ISlangBlob*     content);
+
+    SourceFile* allocateSourceFile(
+        String const&   path,
         String const&   content);
 
     SourceLoc allocateSourceFileForLineDirective(