Standard library save/loadable (#1592)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Fix handling of access modifiers inside type definition.

* Fix access problem for AST node.
Make dumping produce a single function with switch, to potentially make available without Dump specific access.

* WIP on serialization design doc.

* Remove project references to previously generated files.

* More docs on serialization design.

* Improve serialization documentation.
Remove unused function from IRSerialReader.

* Small fixes around naming. Remove long comment from slang-serialize.h - as covered in serialization.md

* Remove long comment in slang-serialize.h as covered in serialization.md

* More information about doing replacements on read for AST and problems surrounding.

* Typo fix.

* Spelling fixes.

* Value serialize.

* Value types with inheritence.

* Use value reflection serial conversion for more AST types

* Use automatic serialization on more of AST.

* Get the types via decltype, simplifies what the extractor has to do.

* Update the serialization.md for the value serialization.

* Small doc improvements.

* Update project.

* Remove ImportExternalDecl type
Added addImportSymbol and ImportSymbol type
Fixed bug in container which meant it wouldn't read back AST module

* Because of change of how imports and handled, store objects as SerialPointers.

* First pass symbol lookup from mangled names.

* Cache current module looked up from mangled name.

* Fix SourceLoc bug.
Improve comments.

* Added diagnostic on mangled symbol not being found

* Fix typo.

* WIP serializing stdlib.

* WIP serializing stdlib in.

* Fix problem serializing arrays that hold data that is already serialized.

* Remove clash of names in MagicTypeModifier.

* Make conversion from char to String explicit.
Fix reference count issue with SerialReader.

* Add code to save/load stdlib.

* Use return code to avoid warning -                 SerialContainerUtil::write(module, options, &stream))

* Make all String numeric ctors explicit.
Added isChar to UnownedStringSlice.
Added operator== for UnownedStringSlice to String to avoid need to convert to String and allocate.

* Add error check to readAllText.

* tabs -> spaces on String.h

* tab -> spaces String.cpp

* Remove msg for StringBuilder, just build inplace for exceptions.

* Check SerialClasses - for name clashes.
Renamed Modifier::name as Modifier::keywordName

* Handling of extensions when deserializing AST - updating the moduleDecl->mapTypeToCandidateExtensions

Co-authored-by: Tim Foley <tim.foley.is@gmail.com>

1 files changed, 132 insertions, 8 deletions

diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp
index 90ddf030c..a66441948 100644
--- a/source/slang/slang.cpp
+++ b/source/slang/slang.cpp
@@ -136,7 +136,6 @@ void Session::init()
     // And the global ASTBuilder
     globalAstBuilder = new ASTBuilder(m_sharedASTBuilder, "globalAstBuilder");
 
-
     // Make sure our source manager is initialized
     builtinSourceManager.initialize(nullptr, nullptr);
 
@@ -175,8 +174,41 @@ void Session::init()
     slangLanguageScope = new Scope();
     slangLanguageScope->nextSibling = hlslLanguageScope;
 
-    addBuiltinSource(coreLanguageScope, "core", getCoreLibraryCode());
-    addBuiltinSource(hlslLanguageScope, "hlsl", getHLSLLibraryCode());
+    if (false)
+    {
+        // Let's try loading serialized modules and adding them
+        _readBuiltinModule(coreLanguageScope, "core");
+        _readBuiltinModule(hlslLanguageScope, "hlsl");
+    }
+    else
+    {
+        addBuiltinSource(coreLanguageScope, "core", getCoreLibraryCode());
+        addBuiltinSource(hlslLanguageScope, "hlsl", getHLSLLibraryCode());
+
+        // Write out
+        if (false)
+        {
+            for (auto& pair : m_builtinLinkage->mapNameToLoadedModules)
+            {
+                const Name* moduleName = pair.Key;
+                Module* module = pair.Value;
+
+                // Set up options
+                SerialContainerUtil::WriteOptions options;
+                //options.optionFlags |= SerialOptionFlag::SourceLocation;
+                options.sourceManager = m_builtinLinkage->getSourceManager();
+
+                StringBuilder builder;
+                builder << moduleName->text << ".slang-module";
+
+                FileStream stream(builder.ProduceString(), FileMode::Create, FileAccess::Write, FileShare::ReadWrite);
+                if (SLANG_FAILED(SerialContainerUtil::write(module, options, &stream)))
+                {
+                    SLANG_UNEXPECTED("Unable to load stdlib");
+                }
+            }
+        }
+    }
 
     {
         for (Index i = 0; i < Index(SourceLanguage::CountOf); ++i)
@@ -194,6 +226,88 @@ void Session::init()
     m_languagePreludes[Index(SourceLanguage::HLSL)] = get_slang_hlsl_prelude();
 }
 
+SlangResult Session::_readBuiltinModule(Scope* scope, String moduleName)
+{
+    StringBuilder moduleFilename;
+    moduleFilename << moduleName << ".slang-module";
+
+    RiffContainer riffContainer;
+    try
+    {
+        FileStream stream(moduleFilename.ProduceString(), FileMode::Open, FileAccess::Read, FileShare::ReadOnly);
+        // Load the riff container
+        SLANG_RETURN_ON_FAIL(RiffUtil::read(&stream, riffContainer));
+    }
+    catch (const IOException&)
+    {
+        return SLANG_FAIL;
+    }
+
+    // Load up the module
+    
+    SerialContainerData containerData;
+
+    Linkage* linkage = getBuiltinLinkage();
+
+    NamePool* sessionNamePool = &namePool;
+    NamePool* linkageNamePool = linkage->getNamePool();
+
+    SerialContainerUtil::ReadOptions options;
+    options.namePool = linkageNamePool;
+    options.session = this;
+    options.sharedASTBuilder = linkage->getASTBuilder()->getSharedASTBuilder();
+    options.sourceManager = linkage->getSourceManager();
+    options.linkage = linkage;
+
+    // Hmm - don't have a suitable sink yet, so attempt to just not have one
+    options.sink = nullptr;
+
+    SLANG_RETURN_ON_FAIL(SerialContainerUtil::read(&riffContainer, options, containerData));
+
+    for (auto& srcModule : containerData.modules)
+    {
+        RefPtr<Module> module(new Module(linkage, srcModule.astBuilder));
+
+        ModuleDecl* moduleDecl = as<ModuleDecl>(srcModule.astRootNode);
+
+        if (moduleDecl)
+        {
+            if (isFromStdLib(moduleDecl))
+            {
+                registerBuiltinDecls(this, moduleDecl);
+            }
+            
+            module->setModuleDecl(moduleDecl);
+        }
+
+        module->setIRModule(srcModule.irModule);
+
+        // Put in the loaded module map
+        linkage->mapNameToLoadedModules.Add(sessionNamePool->getName(moduleName), module);
+
+        // Add the resulting code to the appropriate scope
+        if (!scope->containerDecl)
+        {
+            // We are the first chunk of code to be loaded for this scope
+            scope->containerDecl = moduleDecl;
+        }
+        else
+        {
+            // We need to create a new scope to link into the whole thing
+            auto subScope = new Scope();
+            subScope->containerDecl = moduleDecl;
+            subScope->nextSibling = scope->nextSibling;
+            scope->nextSibling = subScope;
+        }
+
+        // We need to retain this AST so that we can use it in other code
+        // (Note that the `Scope` type does not retain the AST it points to)
+        stdlibModules.add(module);
+    }
+
+    return SLANG_OK;
+}
+
 ISlangUnknown* Session::getInterface(const Guid& guid)
 {
     if(guid == IID_ISlangUnknown || guid == IID_IGlobalSession)
@@ -1240,7 +1354,7 @@ void FrontEndCompileRequest::generateIR()
 
             options.compressionType = SerialCompressionType::None;
             options.sourceManager = getSourceManager();
-            options.optionFlags |= SerialOptionFlag::DebugInfo;
+            options.optionFlags |= SerialOptionFlag::SourceLocation;
 
             // Verify debug information
             if (SLANG_FAILED(SerialContainerUtil::verifyIRSerialize(irModule, getSession(), options)))
@@ -1950,11 +2064,19 @@ void FilePathDependencyList::addDependency(Module* module)
 // Module
 //
 
-Module::Module(Linkage* linkage)
+Module::Module(Linkage* linkage, ASTBuilder* astBuilder)
     : ComponentType(linkage)
-    , m_astBuilder(linkage->getASTBuilder()->getSharedASTBuilder(), "Module")
     , m_mangledExportPool(StringSlicePool::Style::Empty)
 {
+    if (astBuilder)
+    {
+        m_astBuilder = astBuilder;
+    }
+    else
+    {
+        m_astBuilder = new ASTBuilder(linkage->getASTBuilder()->getSharedASTBuilder(), "Module");
+    }
+
     addModuleDependency(this);
 }
 
@@ -2053,7 +2175,7 @@ void Module::_processFindDeclsExportSymbolsRec(Decl* decl)
         return;
     }
 
-    // process `decl` itself
+    // If it's a container process it's children
     if(auto containerDecl = as<ContainerDecl>(decl))
     {
         for (auto child : containerDecl->members)
@@ -2061,7 +2183,9 @@ void Module::_processFindDeclsExportSymbolsRec(Decl* decl)
             _processFindDeclsExportSymbolsRec(child);
         }
     }
-    else if (auto genericDecl = as<GenericDecl>(decl))
+
+    // GenericDecl is also a container, so do subsequent test
+    if (auto genericDecl = as<GenericDecl>(decl))
     {
         _processFindDeclsExportSymbolsRec(genericDecl->inner);
     }