Serialization fixes based on review of #1547 (#1551)

* Test if blob is returned.

* Rename serialize files so can be grouped.

* StringRepresentationCache -> SerialStringTable

* Split out SerialStringTable from slang-serialize-ir

* First pass at reorganizing serialization/containers. Remain some issues about debug info.

* Fix bug in calculating sourceloc.

* Improve calcFixSourceLoc

* Make allocations for payload RiffContainer align to at least 8 bytes. This is important for read, if the payload can contain 8 byte aligned data. Note this has no effect on Riff file format alignment rules.

* Improve comments around RiffContainer and alignment.

* Remove SerialStringTable, can just use StringSlicePool instead.

* Add flags to control what is output in SerialContainer.
Turn off AST output for obfuscated code.
Lazily create astClasses when doing write container serialization.

* Typo fix for Clang/Linux.

* Fixes that came out of review

* TranslationUnit -> Module
* TargetModule -> TargetComponent
* PAYLOAD_MIN_ALIGNMENT -> kPayloadMinAlignment

1 files changed, 30 insertions, 26 deletions

diff --git a/source/slang/slang-serialize-container.cpp b/source/slang/slang-serialize-container.cpp
index 23aacc5d3..2301da9d4 100644
--- a/source/slang/slang-serialize-container.cpp
+++ b/source/slang/slang-serialize-container.cpp
@@ -33,11 +33,15 @@ namespace Slang {
 
         SLANG_ASSERT(irModule || moduleDecl);
 
-        SerialContainerData::TranslationUnit dstTranslationUnit;
-        dstTranslationUnit.astRootNode = moduleDecl;
-        dstTranslationUnit.irModule = irModule;
+        SerialContainerData::Module dstModule;
 
-        out.translationUnits.add(dstTranslationUnit);
+        // NOTE: The astBuilder is not set here, as not needed to be scoped for serialization (it is assumed the
+        // TranslationUnitRequest stays in scope)
+
+        dstModule.astRootNode = moduleDecl;
+        dstModule.irModule = irModule;
+
+        out.modules.add(dstModule);
     }
 
     auto program = request->getSpecializedGlobalAndEntryPointsComponentType();
@@ -50,18 +54,18 @@ namespace Slang {
             auto targetProgram = program->getTargetProgram(target);
             auto irModule = targetProgram->getOrCreateIRModuleForLayout(sink);
 
-            SerialContainerData::TargetModule targetModule;
+            SerialContainerData::TargetComponent targetComponent;
 
-            targetModule.irModule = irModule;
+            targetComponent.irModule = irModule;
 
-            auto& dstTarget = targetModule.target;
+            auto& dstTarget = targetComponent.target;
 
             dstTarget.floatingPointMode = target->floatingPointMode;
             dstTarget.profile = target->targetProfile;
             dstTarget.flags = target->targetFlags;
             dstTarget.codeGenTarget = target->target;
 
-            out.targetModules.add(targetModule);
+            out.targetComponents.add(targetComponent);
         }
     }
 
@@ -105,10 +109,10 @@ namespace Slang {
         container->write(&containerHeader, sizeof(containerHeader));
     }
 
-    if (data.translationUnits.getCount() && (options.optionFlags & (SerialOptionFlag::IRModule | SerialOptionFlag::ASTModule)))
+    if (data.modules.getCount() && (options.optionFlags & (SerialOptionFlag::IRModule | SerialOptionFlag::ASTModule)))
     {
         // Module list
-        RiffContainer::ScopeChunk moduleListScope(container, RiffContainer::Chunk::Kind::List, SerialBinary::kTranslationUnitListFourCc);
+        RiffContainer::ScopeChunk moduleListScope(container, RiffContainer::Chunk::Kind::List, SerialBinary::kModuleListFourCc);
 
         if (options.optionFlags & SerialOptionFlag::DebugInfo)
         {
@@ -117,17 +121,17 @@ namespace Slang {
 
         RefPtr<ASTSerialClasses> astClasses;
 
-        for (const auto& translationUnit : data.translationUnits)
+        for (const auto& module : data.modules)
         {
             // Okay, we need to serialize this module to our container file.
             // We currently don't serialize it's name..., but support for that could be added.
 
             // Write the IR information
-            if ((options.optionFlags & SerialOptionFlag::IRModule) && translationUnit.irModule)
+            if ((options.optionFlags & SerialOptionFlag::IRModule) && module.irModule)
             {
                 IRSerialData serialData;
                 IRSerialWriter writer;
-                SLANG_RETURN_ON_FAIL(writer.write(translationUnit.irModule, debugWriter, options.optionFlags, &serialData));
+                SLANG_RETURN_ON_FAIL(writer.write(module.irModule, debugWriter, options.optionFlags, &serialData));
                 SLANG_RETURN_ON_FAIL(IRSerialWriter::writeContainer(serialData, options.compressionType, container));
             }
 
@@ -135,7 +139,7 @@ namespace Slang {
 
             if (options.optionFlags & SerialOptionFlag::ASTModule)
             {
-                if (ModuleDecl* moduleDecl = as<ModuleDecl>(translationUnit.astRootNode))
+                if (ModuleDecl* moduleDecl = as<ModuleDecl>(module.astRootNode))
                 {
                     if (!astClasses)
                     {
@@ -154,14 +158,14 @@ namespace Slang {
             }
         }
 
-        if (data.targetModules.getCount() && (options.optionFlags & SerialOptionFlag::IRModule))
+        if (data.targetComponents.getCount() && (options.optionFlags & SerialOptionFlag::IRModule))
         {
             // TODO: in the case where we have specialization, we might need
             // to serialize IR related to `program`...
 
-            for (const auto& targetModule : data.targetModules)
+            for (const auto& targetComponent : data.targetComponents)
             {
-                IRModule* irModule = targetModule.irModule;
+                IRModule* irModule = targetComponent.irModule;
 
                 // Okay, we need to serialize this target program and its IR too...
                 IRSerialData serialData;
@@ -255,10 +259,10 @@ namespace Slang {
         SLANG_RETURN_ON_FAIL(debugReader->read(&debugData, options.sourceManager));
     }
 
-    // Add translation units
-    if (RiffContainer::ListChunk* translationUnits = containerChunk->findContainedList(SerialBinary::kTranslationUnitListFourCc))
+    // Add modules
+    if (RiffContainer::ListChunk* moduleList = containerChunk->findContainedList(SerialBinary::kModuleListFourCc))
     {
-        RiffContainer::Chunk* chunk = translationUnits->getFirstContainedChunk();
+        RiffContainer::Chunk* chunk = moduleList->getFirstContainedChunk();
         while (chunk)
         {
             auto startChunk = chunk;
@@ -297,7 +301,7 @@ namespace Slang {
                     // For now we just generate a name.
 
                     StringBuilder buf;
-                    buf << "tu" << out.translationUnits.getCount();
+                    buf << "tu" << out.modules.getCount();
 
                     astBuilder = new ASTBuilder(options.sharedASTBuilder, buf.ProduceString());
 
@@ -315,13 +319,13 @@ namespace Slang {
 
             if (astBuilder || irModule)
             {
-                SerialContainerData::TranslationUnit translationUnit;
+                SerialContainerData::Module module;
 
-                translationUnit.astBuilder = astBuilder;
-                translationUnit.astRootNode = astRootNode;
-                translationUnit.irModule = irModule;
+                module.astBuilder = astBuilder;
+                module.astRootNode = astRootNode;
+                module.irModule = irModule;
 
-                out.translationUnits.add(translationUnit);
+                out.modules.add(module);
             }
 
             // If no progress, step to next chunk