Reference IR modules with entry point (#1101)

* Added RiffReadHelper

* Move type to fourCC in Chunk simplifies some code.

* Make MemoryArena able to track external blocks.
Allow ownership of Data to vary.
Changed IR serialization to use moved allocations to avoid copies.

As it turns out all of the array writes could use unowned data, but doing so requires the IRData to stay in scope longer than IRSerialData, which it does at the moment - but perhaps needs better naming or a control for the feature.

* Write out slang-module container.

* WIP on -r option.
Loading modules - with -r.

* Making the serialized-module run (without using imported module).

* Split compiling module from the test.

* Separate module compilation with a function working.

* Remove serialization test as not used.

* Fix warning on gcc.

* Updated test to have types across module boundary.

* Allow entry point declaration.
A test that tries to build with just an entry point declaration and a module.

* Try to make link work with multiple modules.

* Multi module linking first pass working.

* Multi module test working with -module-name option

* Use isDefinition - for determining to add decorations to entry point lowering.

6 files changed, 60 insertions, 34 deletions

diff --git a/source/slang/slang-compiler.h b/source/slang/slang-compiler.h
index cd99c4079..dd90ff7c6 100644
--- a/source/slang/slang-compiler.h
+++ b/source/slang/slang-compiler.h
@@ -1407,20 +1407,14 @@ namespace Slang
             /// Add a translation unit to be compiled.
             ///
             /// @param language The source language that the translation unit will use (e.g., `SourceLanguage::Slang`
-            /// @param moduleName The name that will be used for the module compile from the translation unit.
-            /// @return The zero-based index of the translation unit in this compile request.
-        int addTranslationUnit(SourceLanguage language, Name* moduleName);
-
-            /// Add a translation unit to be compiled.
+            /// @param moduleName The name that will be used for the module compile from the translation unit. 
             ///
-            /// @param language The source language that the translation unit will use (e.g., `SourceLanguage::Slang`
-            /// @return The zero-based index of the translation unit in this compile request.
-            ///
-            /// The module name for the translation unit will be automatically generated.
+            /// If moduleName is passed as nullptr a module name is generated.
             /// If all translation units in a compile request use automatically generated
             /// module names, then they are guaranteed not to conflict with one another.
-            ///
-        int addTranslationUnit(SourceLanguage language);
+            /// 
+            /// @return The zero-based index of the translation unit in this compile request.
+        int addTranslationUnit(SourceLanguage language, Name* moduleName);
 
         void addTranslationUnitSourceFile(
             int             translationUnitIndex,
@@ -1451,6 +1445,8 @@ namespace Slang
             /// Does the code we are compiling represent part of the Slang standard library?
         bool m_isStandardLibraryCode = false;
 
+        Name* m_defaultModuleName = nullptr;
+
     private:
             /// A component type that includes only the global scopes of the translation unit(s) that were compiled.
         RefPtr<ComponentType> m_globalComponentType;
diff --git a/source/slang/slang-ir-link.cpp b/source/slang/slang-ir-link.cpp
index 1102ecb3e..4e1a03bb2 100644
--- a/source/slang/slang-ir-link.cpp
+++ b/source/slang/slang-ir-link.cpp
@@ -806,7 +806,7 @@ IRFunc* specializeIRForEntryPoint(
         SLANG_UNEXPECTED("no matching IR symbol");
         return nullptr;
     }
-
+    
     // Note: it is possible that `sym` shows multiple
     // definitions, coming from different IR modules that
     // were input to the linking process. We don't have
diff --git a/source/slang/slang-lower-to-ir.cpp b/source/slang/slang-lower-to-ir.cpp
index f040fb003..e19ab2223 100644
--- a/source/slang/slang-lower-to-ir.cpp
+++ b/source/slang/slang-lower-to-ir.cpp
@@ -6473,6 +6473,8 @@ static void lowerFrontEndEntryPointToIR(
     }
 
     // Go through the entry point parameters creating decorations from layout as appropriate
+    // But only if this is a definition not a declaration
+    if (isDefinition(instToDecorate))
     {
         FilteredMemberList<ParamDecl> params = entryPointFuncDecl->GetParameters();
 
diff --git a/source/slang/slang-options.cpp b/source/slang/slang-options.cpp
index 76ebcca52..7429db2fc 100644
--- a/source/slang/slang-options.cpp
+++ b/source/slang/slang-options.cpp
@@ -497,7 +497,7 @@ struct OptionsParser
                 else if (argStr == "-dump-repro")
                 {
                     SLANG_RETURN_ON_FAIL(tryReadCommandLineArgument(sink, arg, &argCursor, argEnd, requestImpl->dumpRepro));
-                    spEnableReproCapture(asExternal(requestImpl));
+                    spEnableReproCapture(compileRequest);
                 }
                 else if (argStr == "-dump-repro-on-error")
                 {
@@ -510,6 +510,13 @@ struct OptionsParser
 
                     SLANG_RETURN_ON_FAIL(StateSerializeUtil::extractFilesToDirectory(reproName));
                 }
+                else if (argStr == "-module-name")
+                {
+                    String moduleName;
+                    SLANG_RETURN_ON_FAIL(tryReadCommandLineArgument(sink, arg, &argCursor, argEnd, moduleName));
+
+                    spSetDefaultModuleName(compileRequest, moduleName.getBuffer());
+                }
                 else if(argStr == "-load-repro")
                 {
                     String reproName;
@@ -860,16 +867,16 @@ struct OptionsParser
                 }
                 else if (argStr == "-r")
                 {
-                    String moduleName;
-                    SLANG_RETURN_ON_FAIL(tryReadCommandLineArgument(sink, arg, &argCursor, argEnd, moduleName));
+                    String referenceModuleName;
+                    SLANG_RETURN_ON_FAIL(tryReadCommandLineArgument(sink, arg, &argCursor, argEnd, referenceModuleName));
 
                     // We need to deserialize and add the modules
-                    FileStream fileStream(moduleName, FileMode::Open, FileAccess::Read, FileShare::ReadWrite);
+                    FileStream fileStream(referenceModuleName, FileMode::Open, FileAccess::Read, FileShare::ReadWrite);
 
                     List<RefPtr<IRModule>> irModules;
                     if (SLANG_FAILED(IRSerialReader::readStreamModules(&fileStream, asInternal(session), requestImpl->getFrontEndReq()->getSourceManager(), irModules)))
                     {
-                        sink->diagnose(SourceLoc(), Diagnostics::unableToReadModuleContainer, moduleName);
+                        sink->diagnose(SourceLoc(), Diagnostics::unableToReadModuleContainer, referenceModuleName);
                         return SLANG_FAIL;
                     }
 
diff --git a/source/slang/slang-state-serialize.cpp b/source/slang/slang-state-serialize.cpp
index e93495fdb..7bda7a377 100644
--- a/source/slang/slang-state-serialize.cpp
+++ b/source/slang/slang-state-serialize.cpp
@@ -878,7 +878,9 @@ struct LoadContext
         {
             const auto& srcTranslationUnit = base.asRaw(srcTranslationUnits[i]);
 
-            int index = frontEndReq->addTranslationUnit(srcTranslationUnit.language);
+            // TODO(JS): We should probably serialize off the module name
+            // Passing in nullptr will just generate the module name
+            int index = frontEndReq->addTranslationUnit(srcTranslationUnit.language, nullptr);
             SLANG_UNUSED(index);
             SLANG_ASSERT(index == i);
 
diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp
index ed0e87045..c7171d835 100644
--- a/source/slang/slang.cpp
+++ b/source/slang/slang.cpp
@@ -1348,6 +1348,18 @@ int FrontEndCompileRequest::addTranslationUnit(SourceLanguage language, Name* mo
 {
     Index result = translationUnits.getCount();
 
+    if (!moduleName)
+    {
+        // We want to ensure that symbols defined in different translation
+        // units get unique mangled names, so that we can, e.g., tell apart
+        // a `main()` function in `vertex.slang` and a `main()` in `fragment.slang`,
+        // even when they are being compiled together.
+        //
+        String generatedName = "tu";
+        generatedName.append(translationUnits.getCount());
+        moduleName = getNamePool()->getName(generatedName);
+    }
+
     RefPtr<TranslationUnitRequest> translationUnit = new TranslationUnitRequest(this);
     translationUnit->compileRequest = this;
     translationUnit->sourceLanguage = SourceLanguage(language);
@@ -1359,18 +1371,6 @@ int FrontEndCompileRequest::addTranslationUnit(SourceLanguage language, Name* mo
     return (int) result;
 }
 
-int FrontEndCompileRequest::addTranslationUnit(SourceLanguage language)
-{
-    // We want to ensure that symbols defined in different translation
-    // units get unique mangled names, so that we can, e.g., tell apart
-    // a `main()` function in `vertex.slang` and a `main()` in `fragment.slang`,
-    // even when they are being compiled together.
-    //
-    String generatedName = "tu";
-    generatedName.append(translationUnits.getCount());
-    return addTranslationUnit(language,  getNamePool()->getName(generatedName));
-}
-
 void FrontEndCompileRequest::addTranslationUnitSourceFile(
     int             translationUnitIndex,
     SourceFile*     sourceFile)
@@ -3009,17 +3009,36 @@ SLANG_API SlangResult spGetDiagnosticOutputBlob(
 SLANG_API int spAddTranslationUnit(
     SlangCompileRequest*    request,
     SlangSourceLanguage     language,
-    char const*             name)
+    char const*             inName)
 {
-    SLANG_UNUSED(name);
-
     auto req = Slang::asInternal(request);
     auto frontEndReq = req->getFrontEndReq();
 
+    Slang::NamePool* namePool = req->getFrontEndReq()->getNamePool();
+
+    // Work out a module name. Can be nullptr if so will generate a name
+    Slang::Name* moduleName = inName ? namePool->getName(inName) : frontEndReq->m_defaultModuleName;
+
+    // If moduleName is nullptr a name will be generated
+    
     return frontEndReq->addTranslationUnit(
-        Slang::SourceLanguage(language));
+        Slang::SourceLanguage(language),
+        moduleName);
+}
+
+SLANG_API void spSetDefaultModuleName(
+    SlangCompileRequest*    request,
+    const char* defaultModuleName)
+{
+    auto req = Slang::asInternal(request);
+    auto frontEndReq = req->getFrontEndReq();
+
+    Slang::NamePool* namePool = req->getFrontEndReq()->getNamePool();
+
+    frontEndReq->m_defaultModuleName = namePool->getName(defaultModuleName);
 }
 
+
 SLANG_API void spTranslationUnit_addPreprocessorDefine(
     SlangCompileRequest*    request,
     int                     translationUnitIndex,