Passing source to Downstream compilation as artifacts (#2382)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Make DownstreamCompileOptions use POD types.

* CharSliceAllocator -> SliceAllocator
Added SliceConverter
CharSliceCaster -> SliceCaster

* First attempt at zero terminating around blobs.

* Fix clang warning.

* Add SlangTerminatedChars
Make Blob implementations support it.
Make most blobs 'terminated'.

* Fix bug setting up sourceFiles for CommandLineDownstreamCompiler.

* Traffic in TerminatedCharSlice for sourceFiles.
Use ArtifactDesc to generate temporary file names for source.

* Fix typo in testing for shared library/C++.

* Working with source being passed as artifacts to DownstreamCompiler.

* Use artifacts in SourceManager/SourceFile.

* Support infering extension from the original file extension.

3 files changed, 32 insertions, 15 deletions

diff --git a/source/slang/slang-compiler.cpp b/source/slang/slang-compiler.cpp
index 18c92ec1a..15a1ceb4b 100644
--- a/source/slang/slang-compiler.cpp
+++ b/source/slang/slang-compiler.cpp
@@ -1051,14 +1051,16 @@ namespace Slang
             // If emitted source is required, emit and set the path            
             if (_useEmittedSource(compiler, translationUnit))
             {
-                // If it's not file based we can set an appropriate path name, and it doesn't matter if it doesn't
-                // exist on the file system
-                options.sourceContentsPath = allocator.allocate(calcSourcePathForEntryPoints());
-
                 CodeGenContext sourceCodeGenContext(this, sourceTarget, extensionTracker);
 
                 SLANG_RETURN_ON_FAIL(sourceCodeGenContext.emitEntryPointsSource(sourceArtifact));
 
+                // If it's not file based we can set an appropriate path name, and it doesn't matter if it doesn't
+                // exist on the file system. 
+                // We set the name to the path as this will be used for downstream reporting.
+                auto sourcePath = calcSourcePathForEntryPoints();
+                sourceArtifact->setName(sourcePath.getBuffer());
+
                 sourceCodeGenContext.maybeDumpIntermediate(sourceArtifact);
             }
             else
@@ -1067,10 +1069,17 @@ namespace Slang
                 const auto& sourceFiles = translationUnit->getSourceFiles();
                 SLANG_ASSERT(sourceFiles.getCount() == 1);
 
-                const SourceFile* sourceFile = sourceFiles[0];
-                
-                options.sourceContentsPath = SliceCaster::asTerminatedCharSlice(sourceFile->getPathInfo().foundPath);
-                options.sourceContents = SliceConverter::toTerminatedCharSlice(allocator, sourceFile->getContentBlob());
+                SourceFile* sourceFile = sourceFiles[0];
+
+                // Make it have an artifact if doesn't have one already
+                // This is useful because it will mean any reps will be kept in scope
+                // 
+                // For example if file backing is needed, the file rep will last the lifetime of the 
+                // SourceFile
+                sourceFile->maybeAddArtifact(nullptr);
+
+                sourceArtifact = sourceFile->getArtifact();
+                SLANG_ASSERT(sourceArtifact);
             }
         }
         else
@@ -1087,11 +1096,9 @@ namespace Slang
         if (sourceArtifact)
         {
             metadata = findAssociated<IArtifactPostEmitMetadata>(sourceArtifact);
-            
-            ComPtr<ISlangBlob> blob;
-            SLANG_RETURN_ON_FAIL(sourceArtifact->loadBlob(ArtifactKeep::Yes, blob.writeRef()));
 
-            options.sourceContents = SliceConverter::toTerminatedCharSlice(allocator, blob);
+            // Set the source artifacts
+            options.sourceArtifacts = makeSlice(sourceArtifact.readRef(), 1);
         }
 
         // Add any preprocessor definitions associated with the linkage
@@ -1368,8 +1375,8 @@ namespace Slang
         }
         
         options.compilerSpecificArguments = allocator.allocate(compilerSpecificArguments);
-        options.requiredCapabilityVersions = SliceCaster::asSlice(requiredCapabilityVersions);
-        options.libraries = SliceCaster::asSlice(libraries);
+        options.requiredCapabilityVersions = SliceUtil::asSlice(requiredCapabilityVersions);
+        options.libraries = SliceUtil::asSlice(libraries);
         options.libraryPaths = allocator.allocate(libraryPaths);
 
         // Compile
diff --git a/source/slang/slang-preprocessor.cpp b/source/slang/slang-preprocessor.cpp
index 0167b4a58..f1f7cffa4 100644
--- a/source/slang/slang-preprocessor.cpp
+++ b/source/slang/slang-preprocessor.cpp
@@ -3055,8 +3055,11 @@ static void HandleIncludeDirective(PreprocessorDirectiveContext* context)
             return;
         }
 
-        
         sourceFile = sourceManager->createSourceFileWithBlob(filePathInfo, foundSourceBlob);
+
+        auto fileSystemExt = context->m_preprocessor->fileSystem;
+        sourceFile->maybeAddArtifact(fileSystemExt);
+
         sourceManager->addSourceFile(filePathInfo.uniqueIdentity, sourceFile);
     }
 
diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp
index 6101a5298..a9868898e 100644
--- a/source/slang/slang.cpp
+++ b/source/slang/slang.cpp
@@ -2575,6 +2575,8 @@ void FrontEndCompileRequest::addTranslationUnitSourceBlob(
     // The path specified may or may not be a file path - mark as being constructed 'FromString'.
     SourceFile* sourceFile = getSourceManager()->createSourceFileWithBlob(PathInfo::makeFromString(path), sourceBlob);
     
+    sourceFile->maybeAddArtifact(nullptr);
+
     addTranslationUnitSourceFile(translationUnitIndex, sourceFile);
 }
 
@@ -2586,6 +2588,8 @@ void FrontEndCompileRequest::addTranslationUnitSourceString(
     // The path specified may or may not be a file path - mark as being constructed 'FromString'.
     SourceFile* sourceFile = getSourceManager()->createSourceFileWithString(PathInfo::makeFromString(path), source);
 
+    sourceFile->maybeAddArtifact(nullptr);
+
     addTranslationUnitSourceFile(translationUnitIndex, sourceFile);
 }
 
@@ -2617,6 +2621,9 @@ void FrontEndCompileRequest::addTranslationUnitSourceFile(
 
     // Was loaded from the specified path
     SourceFile* sourceFile = getSourceManager()->createSourceFileWithBlob(pathInfo, sourceBlob);
+
+    sourceFile->maybeAddArtifact(getLinkage()->getFileSystemExt());
+
     addTranslationUnitSourceFile(translationUnitIndex, sourceFile);
 }