Control container serialization with SerialOptionFlags (#1550)

* Test if blob is returned.

* Rename serialize files so can be grouped.

* StringRepresentationCache -> SerialStringTable

* Split out SerialStringTable from slang-serialize-ir

* First pass at reorganizing serialization/containers. Remain some issues about debug info.

* Fix bug in calculating sourceloc.

* Improve calcFixSourceLoc

* Make allocations for payload RiffContainer align to at least 8 bytes. This is important for read, if the payload can contain 8 byte aligned data. Note this has no effect on Riff file format alignment rules.

* Improve comments around RiffContainer and alignment.

* Remove SerialStringTable, can just use StringSlicePool instead.

* Add flags to control what is output in SerialContainer.
Turn off AST output for obfuscated code.
Lazily create astClasses when doing write container serialization.

* Typo fix for Clang/Linux.

5 files changed, 33 insertions, 20 deletions

diff --git a/source/slang/slang-compiler.cpp b/source/slang/slang-compiler.cpp
index d8da642f4..3855eda68 100755
--- a/source/slang/slang-compiler.cpp
+++ b/source/slang/slang-compiler.cpp
@@ -2357,6 +2357,13 @@ SlangResult dissassembleDXILUsingDXC(
         {
             options.optionFlags |= SerialOptionFlag::DebugInfo;
         }
+        if (linkage->m_obfuscateCode)
+        {
+            // If code is obfuscated, we *disable* AST output as it is not obfuscated and will reveal
+            // too much about IR.
+            // Also currently only IR is needed.
+            options.optionFlags &= ~SerialOptionFlag::ASTModule;
+        }
 
         {
             RiffContainer container;
diff --git a/source/slang/slang-serialize-container.cpp b/source/slang/slang-serialize-container.cpp
index 6f6438947..23aacc5d3 100644
--- a/source/slang/slang-serialize-container.cpp
+++ b/source/slang/slang-serialize-container.cpp
@@ -88,11 +88,6 @@ namespace Slang {
 
 /* static */SlangResult SerialContainerUtil::write(const SerialContainerData& data, const WriteOptions& options, RiffContainer* container)
 {
-    //auto linkage = request->getLinkage();
-    //auto sink = request->getSink();
-    //auto frontEndReq = request->getFrontEndReq();
-    //SourceManager* sourceManager = frontEndReq->getSourceManager();
-
     RefPtr<DebugSerialWriter> debugWriter;
 
     // The string pool used across the whole of the container
@@ -110,7 +105,7 @@ namespace Slang {
         container->write(&containerHeader, sizeof(containerHeader));
     }
 
-    if (data.translationUnits.getCount())
+    if (data.translationUnits.getCount() && (options.optionFlags & (SerialOptionFlag::IRModule | SerialOptionFlag::ASTModule)))
     {
         // Module list
         RiffContainer::ScopeChunk moduleListScope(container, RiffContainer::Chunk::Kind::List, SerialBinary::kTranslationUnitListFourCc);
@@ -120,7 +115,7 @@ namespace Slang {
             debugWriter = new DebugSerialWriter(options.sourceManager);
         }
 
-        RefPtr<ASTSerialClasses> astClasses = new ASTSerialClasses;
+        RefPtr<ASTSerialClasses> astClasses;
 
         for (const auto& translationUnit : data.translationUnits)
         {
@@ -128,7 +123,7 @@ namespace Slang {
             // We currently don't serialize it's name..., but support for that could be added.
 
             // Write the IR information
-            if (translationUnit.irModule)
+            if ((options.optionFlags & SerialOptionFlag::IRModule) && translationUnit.irModule)
             {
                 IRSerialData serialData;
                 IRSerialWriter writer;
@@ -138,20 +133,28 @@ namespace Slang {
 
             // Write the AST information
 
-            if (ModuleDecl* moduleDecl = as<ModuleDecl>(translationUnit.astRootNode))
+            if (options.optionFlags & SerialOptionFlag::ASTModule)
             {
-                ModuleASTSerialFilter filter(moduleDecl);
-                ASTSerialWriter writer(astClasses, &filter, debugWriter);
+                if (ModuleDecl* moduleDecl = as<ModuleDecl>(translationUnit.astRootNode))
+                {
+                    if (!astClasses)
+                    {
+                        astClasses = new ASTSerialClasses;
+                    }
 
-                // Add the module and everything that isn't filtered out in the filter.
-                writer.addPointer(moduleDecl);
+                    ModuleASTSerialFilter filter(moduleDecl);
+                    ASTSerialWriter writer(astClasses, &filter, debugWriter);
 
-                // We can now serialize it into the riff container.
-                SLANG_RETURN_ON_FAIL(writer.writeIntoContainer(container));
+                    // Add the module and everything that isn't filtered out in the filter.
+                    writer.addPointer(moduleDecl);
+
+                    // We can now serialize it into the riff container.
+                    SLANG_RETURN_ON_FAIL(writer.writeIntoContainer(container));
+                }
             }
         }
 
-        if (data.targetModules.getCount())
+        if (data.targetModules.getCount() && (options.optionFlags & SerialOptionFlag::IRModule))
         {
             // TODO: in the case where we have specialization, we might need
             // to serialize IR related to `program`...
diff --git a/source/slang/slang-serialize-container.h b/source/slang/slang-serialize-container.h
index 2f2f509b8..0bed80568 100644
--- a/source/slang/slang-serialize-container.h
+++ b/source/slang/slang-serialize-container.h
@@ -79,7 +79,7 @@ struct SerialContainerUtil
     struct WriteOptions
     {
         SerialCompressionType compressionType = SerialCompressionType::VariableByteLite;
-        SerialOptionFlags optionFlags = 0;
+        SerialOptionFlags optionFlags = SerialOptionFlag::ASTModule | SerialOptionFlag::IRModule;
         SourceManager* sourceManager = nullptr;
     };
 
diff --git a/source/slang/slang-serialize-types.h b/source/slang/slang-serialize-types.h
index f799dd607..405a7bfcd 100644
--- a/source/slang/slang-serialize-types.h
+++ b/source/slang/slang-serialize-types.h
@@ -18,8 +18,10 @@ struct SerialOptionFlag
     typedef uint32_t Type;
     enum Enum : Type
     {
-        RawSourceLocation = 0x01,
-        DebugInfo = 0x02,
+        RawSourceLocation   = 0x01,     ///< If set will store directly SourceLoc - only useful if current source locs will be identical when read in (typically this is *NOT* the case)
+        DebugInfo           = 0x02,     ///< If set will output debug information, that can be reconstructed when read after being stored.
+        ASTModule           = 0x04,     ///< If set will output AST modules - typically required, but potentially not desired (for example with obsfucation)
+        IRModule            = 0x08,     ///< If set will output IR modules - typically required
     };
 };
 typedef SerialOptionFlag::Type SerialOptionFlags;
diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp
index c9dccbdbf..43d1ede7e 100644
--- a/source/slang/slang.cpp
+++ b/source/slang/slang.cpp
@@ -1076,9 +1076,10 @@ void FrontEndCompileRequest::generateIR()
         if (verifyDebugSerialization)
         {
             SerialContainerUtil::WriteOptions options;
+
             options.compressionType = SerialCompressionType::None;
             options.sourceManager = getSourceManager();
-            options.optionFlags = SerialOptionFlag::DebugInfo;
+            options.optionFlags |= SerialOptionFlag::DebugInfo;
 
             // Verify debug information
             if (SLANG_FAILED(SerialContainerUtil::verifyIRSerialize(irModule, getSession(), options)))