Standard library save/loadable (#1592)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Fix handling of access modifiers inside type definition.

* Fix access problem for AST node.
Make dumping produce a single function with switch, to potentially make available without Dump specific access.

* WIP on serialization design doc.

* Remove project references to previously generated files.

* More docs on serialization design.

* Improve serialization documentation.
Remove unused function from IRSerialReader.

* Small fixes around naming. Remove long comment from slang-serialize.h - as covered in serialization.md

* Remove long comment in slang-serialize.h as covered in serialization.md

* More information about doing replacements on read for AST and problems surrounding.

* Typo fix.

* Spelling fixes.

* Value serialize.

* Value types with inheritence.

* Use value reflection serial conversion for more AST types

* Use automatic serialization on more of AST.

* Get the types via decltype, simplifies what the extractor has to do.

* Update the serialization.md for the value serialization.

* Small doc improvements.

* Update project.

* Remove ImportExternalDecl type
Added addImportSymbol and ImportSymbol type
Fixed bug in container which meant it wouldn't read back AST module

* Because of change of how imports and handled, store objects as SerialPointers.

* First pass symbol lookup from mangled names.

* Cache current module looked up from mangled name.

* Fix SourceLoc bug.
Improve comments.

* Added diagnostic on mangled symbol not being found

* Fix typo.

* WIP serializing stdlib.

* WIP serializing stdlib in.

* Fix problem serializing arrays that hold data that is already serialized.

* Remove clash of names in MagicTypeModifier.

* Make conversion from char to String explicit.
Fix reference count issue with SerialReader.

* Add code to save/load stdlib.

* Use return code to avoid warning -                 SerialContainerUtil::write(module, options, &stream))

* Make all String numeric ctors explicit.
Added isChar to UnownedStringSlice.
Added operator== for UnownedStringSlice to String to avoid need to convert to String and allocate.

* Add error check to readAllText.

* tabs -> spaces on String.h

* tab -> spaces String.cpp

* Remove msg for StringBuilder, just build inplace for exceptions.

* Check SerialClasses - for name clashes.
Renamed Modifier::name as Modifier::keywordName

* Handling of extensions when deserializing AST - updating the moduleDecl->mapTypeToCandidateExtensions

Co-authored-by: Tim Foley <tim.foley.is@gmail.com>

1 files changed, 127 insertions, 23 deletions

diff --git a/source/slang/slang-serialize-container.cpp b/source/slang/slang-serialize-container.cpp
index 8c4edb9a0..067e6637a 100644
--- a/source/slang/slang-serialize-container.cpp
+++ b/source/slang/slang-serialize-container.cpp
@@ -16,42 +16,100 @@
 
 namespace Slang {
 
-/* static */SlangResult SerialContainerUtil::requestToData(EndToEndCompileRequest* request, const WriteOptions& options, SerialContainerData& out)
+/* static */SlangResult SerialContainerUtil::write(Module* module, const WriteOptions& options, Stream* stream)
 {
-    SLANG_UNUSED(options);
-
-    out.clear();
-
-    auto linkage = request->getLinkage();
-    auto sink = request->getSink();
-    auto frontEndReq = request->getFrontEndReq();
-
-    for (TranslationUnitRequest* translationUnit : frontEndReq->translationUnits)
+    RiffContainer container;
     {
-        auto module = translationUnit->module;
-        auto irModule = module->getIRModule();
+        SerialContainerData data;
+        SLANG_RETURN_ON_FAIL(SerialContainerUtil::addModuleToData(module, options, data));
+        SLANG_RETURN_ON_FAIL(SerialContainerUtil::write(data, options, &container));
+    }
+    // We now write the RiffContainer to the stream
+    SLANG_RETURN_ON_FAIL(RiffUtil::write(container.getRoot(), true, stream));
+    return SLANG_OK;
+}
 
-        // Root AST node
-        auto moduleDecl = translationUnit->getModuleDecl();
+/* static */SlangResult SerialContainerUtil::write(FrontEndCompileRequest* frontEndReq, const WriteOptions& options, Stream* stream)
+{
+    RiffContainer container;
+    {
+        SerialContainerData data;
+        SLANG_RETURN_ON_FAIL(SerialContainerUtil::addFrontEndRequestToData(frontEndReq, options, data));
+        SLANG_RETURN_ON_FAIL(SerialContainerUtil::write(data, options, &container));
+    }
+    // We now write the RiffContainer to the stream
+    SLANG_RETURN_ON_FAIL(RiffUtil::write(container.getRoot(), true, stream));
+    return SLANG_OK;
+}
 
-        SLANG_ASSERT(irModule || moduleDecl);
+/* static */SlangResult SerialContainerUtil::write(EndToEndCompileRequest* request, const WriteOptions& options, Stream* stream)
+{
+    RiffContainer container;
+    {
+        SerialContainerData data;
+        SLANG_RETURN_ON_FAIL(SerialContainerUtil::addEndToEndRequestToData(request, options, data));
+        SLANG_RETURN_ON_FAIL(SerialContainerUtil::write(data, options, &container));
+    }
+    // We now write the RiffContainer to the stream
+    SLANG_RETURN_ON_FAIL(RiffUtil::write(container.getRoot(), true, stream));
+    return SLANG_OK;
+}
 
+/* static */SlangResult SerialContainerUtil::addModuleToData(Module* module, const WriteOptions& options, SerialContainerData& outData)
+{
+    if (options.optionFlags & (SerialOptionFlag::ASTModule | SerialOptionFlag::IRModule))
+    {
         SerialContainerData::Module dstModule;
 
         // NOTE: The astBuilder is not set here, as not needed to be scoped for serialization (it is assumed the
         // TranslationUnitRequest stays in scope)
 
-        dstModule.astRootNode = moduleDecl;
-        dstModule.irModule = irModule;
+        if (options.optionFlags & SerialOptionFlag::ASTModule)
+        {
+            // Root AST node
+            auto moduleDecl = module->getModuleDecl();
+            SLANG_ASSERT(moduleDecl);
 
-        out.modules.add(dstModule);
+            dstModule.astRootNode = moduleDecl;
+        }
+        if (options.optionFlags & SerialOptionFlag::IRModule)
+        {
+            // IR module
+            dstModule.irModule = module->getIRModule();
+            SLANG_ASSERT(dstModule.irModule);
+        }
+
+        outData.modules.add(dstModule);
+    }
+
+    return SLANG_OK;
+}
+
+
+/* static */SlangResult SerialContainerUtil::addFrontEndRequestToData(FrontEndCompileRequest* frontEndReq, const WriteOptions& options, SerialContainerData& outData)
+{
+    // Go through translation units, adding modules
+    for (TranslationUnitRequest* translationUnit : frontEndReq->translationUnits)
+    {
+        SLANG_RETURN_ON_FAIL(addModuleToData(translationUnit->module, options, outData));        
     }
 
+    return SLANG_OK;
+}
+
+/* static */SlangResult SerialContainerUtil::addEndToEndRequestToData(EndToEndCompileRequest* request, const WriteOptions& options, SerialContainerData& out)
+{    
+    auto linkage = request->getLinkage();
+    auto sink = request->getSink();
+
+    // Output the front end request data
+    SLANG_RETURN_ON_FAIL(addFrontEndRequestToData(request->getFrontEndReq(), options, out));
+
+    //
     auto program = request->getSpecializedGlobalAndEntryPointsComponentType();
 
     // Add all the target modules
     {
-        
         for (auto target : linkage->targets)
         {
             auto targetProgram = program->getTargetProgram(target);
@@ -117,7 +175,7 @@ namespace Slang {
         // Module list
         RiffContainer::ScopeChunk moduleListScope(container, RiffContainer::Chunk::Kind::List, SerialBinary::kModuleListFourCc);
 
-        if (options.optionFlags & SerialOptionFlag::DebugInfo)
+        if (options.optionFlags & SerialOptionFlag::SourceLocation)
         {
             sourceLocWriter = new SerialSourceLocWriter(options.sourceManager);
         }
@@ -226,6 +284,20 @@ namespace Slang {
     return SLANG_OK;
 }
 
+
+static List<ExtensionDecl*>& _getCandidateExtensionList(
+    AggTypeDecl* typeDecl,
+    Dictionary<AggTypeDecl*, RefPtr<CandidateExtensionList>>& mapTypeToCandidateExtensions)
+{
+    RefPtr<CandidateExtensionList> entry;
+    if (!mapTypeToCandidateExtensions.TryGetValue(typeDecl, entry))
+    {
+        entry = new CandidateExtensionList();
+        mapTypeToCandidateExtensions.Add(typeDecl, entry);
+    }
+    return entry->candidateExtensions;
+}
+
 /* static */Result SerialContainerUtil::read(RiffContainer* container, const ReadOptions& options, SerialContainerData& out)
 {
     out.clear();
@@ -396,6 +468,10 @@ namespace Slang {
                     // Set the sourceLocReader before doing de-serialize, such can lookup the remapped sourceLocs
                     reader.getExtraObjects().set(sourceLocReader);
 
+                    // Go through all of the AST nodes
+                    // 1) Set the ASTBuilder on Type nodes
+                    
+
                     // TODO(JS):
                     // If modules can have more complicated relationships (like a two modules can refer to symbols
                     // from each other), then we can make this work by
@@ -412,6 +488,34 @@ namespace Slang {
 
                     // Get the root node. It's at index 1 (0 is the null value).
                     astRootNode = reader.getPointer(SerialIndex(1)).dynamicCast<NodeBase>();
+
+                    // 2) Add the extensions to the module mapTypeToCandidateExtensions cache
+
+                    {
+                        ModuleDecl* moduleDecl = as<ModuleDecl>(astRootNode);
+                        
+                        for (auto& obj : reader.getObjects())
+                        {
+                            if (Type* type = obj.dynamicCast<Type>())
+                            {
+                                type->_setASTBuilder(astBuilder);
+                            }
+
+                            if (ExtensionDecl* extensionDecl = obj.dynamicCast<ExtensionDecl>())
+                            {
+                                if (auto targetDeclRefType = as<DeclRefType>(extensionDecl->targetType))
+                                {
+                                    // Attach our extension to that type as a candidate...
+                                    if (auto aggTypeDeclRef = targetDeclRefType->declRef.as<AggTypeDecl>())
+                                    {
+                                        auto aggTypeDecl = aggTypeDeclRef.getDecl();
+
+                                        _getCandidateExtensionList(aggTypeDecl, moduleDecl->mapTypeToCandidateExtensions).add(extensionDecl);
+                                    }
+                                }
+                            }
+                        }
+                    }
                 }
 
                 // Onto next chunk
@@ -489,7 +593,7 @@ namespace Slang {
 
         RefPtr<SerialSourceLocWriter> sourceLocWriter;
 
-        if (options.optionFlags & SerialOptionFlag::DebugInfo)
+        if (options.optionFlags & SerialOptionFlag::SourceLocation)
         {
             sourceLocWriter = new SerialSourceLocWriter(options.sourceManager);
         }
@@ -530,7 +634,7 @@ namespace Slang {
         RefPtr<SerialSourceLocReader> sourceLocReader;
 
         // If we have debug info then find and read it
-        if (options.optionFlags & SerialOptionFlag::DebugInfo)
+        if (options.optionFlags & SerialOptionFlag::SourceLocation)
         {
             RiffContainer::ListChunk* debugList = rootList->findContainedList(SerialSourceLocData::kDebugFourCc);
             if (!debugList)
@@ -593,7 +697,7 @@ namespace Slang {
             }
         }
     }
-    else if (options.optionFlags & SerialOptionFlag::DebugInfo)
+    else if (options.optionFlags & SerialOptionFlag::SourceLocation)
     {
         // They should be on the same line nos
         for (Index i = 1; i < readInsts.getCount(); ++i)