1 files changed, 200 insertions, 73 deletions

diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp
index a7a4fc8dc..18b9b3c99 100644
--- a/source/slang/slang.cpp
+++ b/source/slang/slang.cpp
@@ -249,13 +249,15 @@ void Session::_initCodeGenTransitionMap()
 
 void Session::addBuiltins(
     char const*     sourcePath,
-    char const*     sourceString)
+    char const*     source)
 {
+    auto sourceBlob = StringBlob::moveCreate(String(source));
+
     // TODO(tfoley): Add ability to directly new builtins to the appropriate scope
     addBuiltinSource(
         coreLanguageScope,
         sourcePath,
-        sourceString);
+        sourceBlob);
 }
 
 void Session::setSharedLibraryLoader(ISlangSharedLibraryLoader* loader)
@@ -295,9 +297,9 @@ SlangResult Session::compileStdLib(slang::CompileStdLibFlags compileFlags)
     }
 
     // TODO(JS): Could make this return a SlangResult as opposed to exception
-    addBuiltinSource(coreLanguageScope, "core", getCoreLibraryCode());
-    addBuiltinSource(hlslLanguageScope, "hlsl", getHLSLLibraryCode());
-    addBuiltinSource(autodiffLanguageScope, "diff", getAutodiffLibraryCode());
+    addBuiltinSource(coreLanguageScope, "core", StringBlob::moveCreate(getCoreLibraryCode()));
+    addBuiltinSource(hlslLanguageScope, "hlsl", StringBlob::moveCreate(getHLSLLibraryCode()));
+    addBuiltinSource(autodiffLanguageScope, "diff", StringBlob::moveCreate(getAutodiffLibraryCode()));
 
     if (compileFlags & slang::CompileStdLibFlag::WriteDocumentation)
     {
@@ -1489,7 +1491,120 @@ SourceManager* TranslationUnitRequest::getSourceManager()
     return compileRequest->getSourceManager();
 }
 
-void TranslationUnitRequest::addSourceFile(SourceFile* sourceFile)
+void TranslationUnitRequest::addSourceArtifact(IArtifact* sourceArtifact)
+{
+    SLANG_ASSERT(sourceArtifact);
+    m_sourceArtifacts.add(ComPtr<IArtifact>(sourceArtifact));
+}
+
+
+void TranslationUnitRequest::addSource(IArtifact* sourceArtifact, SourceFile* sourceFile)
+{
+    SLANG_ASSERT(sourceArtifact && sourceFile);
+    // Must be in sync!
+    SLANG_ASSERT(m_sourceFiles.getCount() == m_sourceArtifacts.getCount());
+
+    addSourceArtifact(sourceArtifact);
+    _addSourceFile(sourceFile);
+}
+
+SlangResult TranslationUnitRequest::requireSourceFiles()
+{
+    SLANG_ASSERT(m_sourceFiles.getCount() <= m_sourceArtifacts.getCount());
+
+    if (m_sourceFiles.getCount() == m_sourceArtifacts.getCount())
+    {
+        return SLANG_OK;
+    }
+
+    auto sink = compileRequest->getSink();
+    SourceManager* sourceManager = compileRequest->getSourceManager();
+
+    // Get he linkage file system
+    //ISlangFileSystemExt* linkageFileSystem = compileRequest->getLinkage()->getFileSystemExt();
+
+    for (Index i = m_sourceFiles.getCount(); i < m_sourceArtifacts.getCount(); ++i)
+    {
+        IArtifact* artifact = m_sourceArtifacts[i];
+
+        PathInfo pathInfo = PathInfo::makeUnknown();
+        
+        if (auto extRep = findRepresentation<IExtFileArtifactRepresentation>(artifact))
+        {
+            auto extFileSystem = extRep->getFileSystem();
+
+            // TODO(JS): 
+            // Ideally we'd confirm that the file system was the same, such we could know that the unique 
+            // identity is appropriate for the current file system.
+            //
+            // We just assume compatibility for the moment, because repro will be a different file system
+            // but we need to use the unique identity that is there.
+      
+            //if (extFileSystem == linkageFileSystem)
+            {
+                // Get the unique identity
+                ComPtr<ISlangBlob> uniqueIdentityBlob;
+                if (SLANG_SUCCEEDED(extFileSystem->getFileUniqueIdentity(extRep->getPath(), uniqueIdentityBlob.writeRef())) && uniqueIdentityBlob)
+                {
+                    auto uniqueIdentity = StringUtil::getString(uniqueIdentityBlob);
+
+                    // See if this an already loaded source file
+                    if (auto sourceFile = sourceManager->findSourceFileRecursively(uniqueIdentity))
+                    {
+                        // If the source file has a blob, and the artifact doesn't copy over that representation
+                        if (sourceFile->getContentBlob() && findRepresentation<ISlangBlob>(artifact) == nullptr)
+                        {
+                            artifact->addRepresentationUnknown(sourceFile->getContentBlob());
+                        }
+
+                        _addSourceFile(sourceFile);
+                        continue;
+                    }
+
+                    pathInfo = PathInfo::makeNormal(extRep->getPath(), uniqueIdentity);
+                }
+                else
+                {
+                    pathInfo = PathInfo::makePath(extRep->getPath());
+                }
+            }
+        }
+
+        if (pathInfo.type == PathInfo::Type::Unknown)
+        {
+            const auto path = ArtifactUtil::findPath(artifact);
+
+            // If has a path representation we'll make it a FoundPath
+            if (findRepresentation<IPathArtifactRepresentation>(artifact))
+            {
+                pathInfo = PathInfo::makePath(path);
+            }
+            else
+            {
+                // Otherwise we'll assume it's created from a 'String' even if that's not quite the case
+                pathInfo = PathInfo::makeFromString(path);
+            }
+        }
+
+        ComPtr<ISlangBlob> blob;
+        const SlangResult blobRes = artifact->loadBlob(ArtifactKeep::Yes, blob.writeRef());
+        if (SLANG_FAILED(blobRes))
+        {
+            // Report couldn't load
+            sink->diagnose(SourceLoc(),
+                Diagnostics::cannotOpenFile,
+                pathInfo.getName());
+            return blobRes;
+        }
+        
+        auto sourceFile = sourceManager->createSourceFileWithBlob(pathInfo, blob);
+        _addSourceFile(sourceFile);
+    }
+
+    return SLANG_OK;
+}
+
+void TranslationUnitRequest::_addSourceFile(SourceFile* sourceFile)
 {
     m_sourceFiles.add(sourceFile);
 
@@ -1505,6 +1620,12 @@ void TranslationUnitRequest::addSourceFile(SourceFile* sourceFile)
     }
 }
 
+List<SourceFile*> const& TranslationUnitRequest::getSourceFiles() 
+{ 
+    SLANG_ASSERT(m_sourceArtifacts.getCount() == m_sourceFiles.getCount());
+    return m_sourceFiles; 
+}
+
 EndToEndCompileRequest::~EndToEndCompileRequest()
 {
     // Flush any writers associated with the request
@@ -2246,8 +2367,11 @@ SlangResult FrontEndCompileRequest::executeActionsInner()
     // We currently allow GlSL files on the command line so that we can
     // drive our "pass-through" mode, but we really want to issue an error
     // message if the user is seriously asking us to compile them.
-    for (auto& translationUnit : translationUnits)
+    for (TranslationUnitRequest* translationUnit : translationUnits)
     {
+        // Make sure SourceFile representation is available for all translationUnits
+        SLANG_RETURN_ON_FAIL(translationUnit->requireSourceFiles());
+
         switch(translationUnit->sourceLanguage)
         {
         default:
@@ -2261,9 +2385,9 @@ SlangResult FrontEndCompileRequest::executeActionsInner()
 
 
     // Parse everything from the input files requested
-    for (auto& translationUnit : translationUnits)
+    for (TranslationUnitRequest* translationUnit : translationUnits)
     {
-        parseTranslationUnit(translationUnit.Ptr());
+        parseTranslationUnit(translationUnit);
     }
 
     if (outputPreprocessor)
@@ -2559,47 +2683,14 @@ int FrontEndCompileRequest::addTranslationUnit(TranslationUnitRequest* translati
     return (int) result;
 }
 
-
-void FrontEndCompileRequest::addTranslationUnitSourceFile(
+void FrontEndCompileRequest::addTranslationUnitSourceArtifact(
     int             translationUnitIndex,
-    SourceFile*     sourceFile)
+    IArtifact*      sourceArtifact)
 {
     auto translationUnit = translationUnits[translationUnitIndex];
     
-    // TODO(JS): 
-    // The larger problem here is that a file on a file system *could* be interpretted in different ways.
-    // When the user supplies the source as a string, we use the source type specified for the translation unit.
-    // 
-    // If it's on the file system it could be (say) compiled as HLSL or some other way. A *downstream* compiler
-    // that used the file system may care.  
-    //
-    // We will assume here, that if it's loaded from the file system, it's path extension defines how it should be interpretted
-    // If that wasn't the case we'd have to either *copy*, or do some command line fiddling to tell it for the downstream compiler
-    // what the file is. 
-
-    const auto& pathInfo = sourceFile->getPathInfo();
-    const auto pathType = pathInfo.type;
-
-    switch (pathType)
-    {
-        case PathInfo::Type::FromString:
-        {
-            // Set the artifact type from the the source language type
-            auto sourceDesc = ArtifactDescUtil::makeDescForSourceLanguage(asExternal(translationUnit->sourceLanguage));
-            sourceFile->maybeAddArtifact(&sourceDesc, nullptr);
-            break;
-        }
-        case PathInfo::Type::FoundPath:
-        case PathInfo::Type::Normal:
-        {
-            // We'll *not* use the source for the artifact type. Doing so will lead to the type being determined via extension
-            sourceFile->maybeAddArtifact(nullptr, getLinkage()->getFileSystemExt());
-            break;
-        }
-    }
-
     // Add the source file
-    translationUnit->addSourceFile(sourceFile);
+    translationUnit->addSourceArtifact(sourceArtifact);
 }
 
 void FrontEndCompileRequest::addTranslationUnitSourceBlob(
@@ -2607,21 +2698,13 @@ void FrontEndCompileRequest::addTranslationUnitSourceBlob(
     String const&   path,
     ISlangBlob*     sourceBlob)
 {
-    // The path specified may or may not be a file path - mark as being constructed 'FromString'.
-    SourceFile* sourceFile = getSourceManager()->createSourceFileWithBlob(PathInfo::makeFromString(path), sourceBlob);
-    
-    addTranslationUnitSourceFile(translationUnitIndex, sourceFile);
-}
+    auto translationUnit = translationUnits[translationUnitIndex];
+    auto sourceDesc = ArtifactDescUtil::makeDescForSourceLanguage(asExternal(translationUnit->sourceLanguage));
 
-void FrontEndCompileRequest::addTranslationUnitSourceString(
-    int             translationUnitIndex,
-    String const&   path,
-    String const&   source)
-{
-    // The path specified may or may not be a file path - mark as being constructed 'FromString'.
-    SourceFile* sourceFile = getSourceManager()->createSourceFileWithString(PathInfo::makeFromString(path), source);
+    auto artifact = ArtifactUtil::createArtifact(sourceDesc, path.getBuffer());
+    artifact->addRepresentationUnknown(sourceBlob);
 
-    addTranslationUnitSourceFile(translationUnitIndex, sourceFile);
+    translationUnit->addSourceArtifact(artifact);
 }
 
 void FrontEndCompileRequest::addTranslationUnitSourceFile(
@@ -2636,11 +2719,35 @@ void FrontEndCompileRequest::addTranslationUnitSourceFile(
     // paths were not taken into account by this function.
     //
 
-    PathInfo pathInfo;
+    auto fileSystemExt = getLinkage()->getFileSystemExt();
+    auto translationUnit = getTranslationUnit(translationUnitIndex);
+
+    auto sourceDesc = ArtifactDescUtil::makeDescForSourceLanguage(asExternal(translationUnit->sourceLanguage));
+
+    auto sourceArtifact = ArtifactUtil::createArtifact(sourceDesc);
 
-    ComPtr<ISlangBlob> sourceBlob;
-    SlangResult result = loadFile(path, pathInfo, sourceBlob.writeRef());
-    if(SLANG_FAILED(result))
+    auto extRep = new ExtFileArtifactRepresentation(path.getUnownedSlice(), fileSystemExt);
+    sourceArtifact->addRepresentation(extRep);
+
+    SlangResult existsRes = SLANG_OK;
+
+    // If we require caching, we demand it's loaded here.
+    // 
+    // In practice this probably means repro capture is enabled. So we want to 
+    // load the blob such that it's in the cache, even if it doesn't actually 
+    // have to be loaded for the compilation.
+    if (getLinkage()->m_requireCacheFileSystem)
+    {
+        ComPtr<ISlangBlob> blob;
+        // If we can load the blob, then it exists
+        existsRes = sourceArtifact->loadBlob(ArtifactKeep::Yes, blob.writeRef());
+    }
+    else
+    {
+        existsRes = sourceArtifact->exists() ? SLANG_OK : SLANG_E_NOT_FOUND;
+    }
+
+    if (SLANG_FAILED(existsRes))
     {
         // Emit a diagnostic!
         getSink()->diagnose(
@@ -2650,10 +2757,7 @@ void FrontEndCompileRequest::addTranslationUnitSourceFile(
         return;
     }
 
-    // Was loaded from the specified path
-    SourceFile* sourceFile = getSourceManager()->createSourceFileWithBlob(pathInfo, sourceBlob);
-
-    addTranslationUnitSourceFile(translationUnitIndex, sourceFile);
+    translationUnit->addSourceArtifact(sourceArtifact);
 }
 
 int FrontEndCompileRequest::addEntryPoint(
@@ -2808,10 +2912,29 @@ RefPtr<Module> Linkage::loadModule(
         name,
         srcLoc);
 
+    // Create an artifact for the source
+    auto sourceArtifact = ArtifactUtil::createArtifact(ArtifactDesc::make(ArtifactKind::Source, ArtifactPayload::Slang, ArtifactStyle::Unknown));
+
     // Create with the 'friendly' name
-    SourceFile* sourceFile = getSourceManager()->createSourceFileWithBlob(filePathInfo, sourceBlob);
-    
-    translationUnit->addSourceFile(sourceFile);
+    if (filePathInfo.type == PathInfo::Type::Normal ||
+        filePathInfo.type == PathInfo::Type::FoundPath)
+    {
+        // We create that it was loaded from the file system
+        sourceArtifact->addRepresentation(new ExtFileArtifactRepresentation(filePathInfo.foundPath.getUnownedSlice(), getFileSystemExt()));
+    }
+    else
+    {
+        // Else we say we don't know and just add the blob
+        sourceArtifact->addRepresentationUnknown(sourceBlob);
+    }
+
+    translationUnit->addSourceArtifact(sourceArtifact);
+
+    if (SLANG_FAILED(translationUnit->requireSourceFiles()))
+    {
+        // Some problem accessing source files
+        return nullptr;
+    }
 
     int errorCountBefore = sink->getErrorCount();
     frontEndReq->parseTranslationUnit(translationUnit);
@@ -4234,7 +4357,7 @@ RefPtr<Module> findOrImportModule(
 void Session::addBuiltinSource(
     Scope*                  scope,
     String const&           path,
-    String const&           source)
+    ISlangBlob*             sourceBlob)
 {
     SourceManager* sourceManager = getBuiltinSourceManager();
 
@@ -4254,10 +4377,10 @@ void Session::addBuiltinSource(
     Name* moduleName = getNamePool()->getName(path);
     auto translationUnitIndex = compileRequest->addTranslationUnit(SourceLanguage::Slang, moduleName);
 
-    compileRequest->addTranslationUnitSourceString(
+    compileRequest->addTranslationUnitSourceBlob(
         translationUnitIndex,
         path,
-        source);
+        sourceBlob);
 
     SlangResult res = compileRequest->executeActionsInner();
     if (SLANG_FAILED(res))
@@ -4637,7 +4760,11 @@ void EndToEndCompileRequest::addTranslationUnitSourceStringSpan(int translationU
 
     if (!path) path = "";
 
-    frontEndReq->addTranslationUnitSourceString(translationUnitIndex, path, UnownedStringSlice(sourceBegin, sourceEnd));
+    const auto slice = UnownedStringSlice(sourceBegin, sourceEnd);
+
+    auto blob = RawBlob::create(slice.begin(), slice.getLength());
+
+    frontEndReq->addTranslationUnitSourceBlob(translationUnitIndex, path, blob);
 }
 
 void EndToEndCompileRequest::addTranslationUnitSourceBlob(int translationUnitIndex, char const* path, ISlangBlob* sourceBlob)