From c985f5f2f95dc95998fdfb8400baa0a04760ada2 Mon Sep 17 00:00:00 2001 From: jsmall-nvidia Date: Thu, 5 Nov 2020 13:43:00 -0500 Subject: Standard library save/loadable (#1592) * #include an absolute path didn't work - because paths were taken to always be relative. * Fix handling of access modifiers inside type definition. * Fix access problem for AST node. Make dumping produce a single function with switch, to potentially make available without Dump specific access. * WIP on serialization design doc. * Remove project references to previously generated files. * More docs on serialization design. * Improve serialization documentation. Remove unused function from IRSerialReader. * Small fixes around naming. Remove long comment from slang-serialize.h - as covered in serialization.md * Remove long comment in slang-serialize.h as covered in serialization.md * More information about doing replacements on read for AST and problems surrounding. * Typo fix. * Spelling fixes. * Value serialize. * Value types with inheritence. * Use value reflection serial conversion for more AST types * Use automatic serialization on more of AST. * Get the types via decltype, simplifies what the extractor has to do. * Update the serialization.md for the value serialization. * Small doc improvements. * Update project. * Remove ImportExternalDecl type Added addImportSymbol and ImportSymbol type Fixed bug in container which meant it wouldn't read back AST module * Because of change of how imports and handled, store objects as SerialPointers. * First pass symbol lookup from mangled names. * Cache current module looked up from mangled name. * Fix SourceLoc bug. Improve comments. * Added diagnostic on mangled symbol not being found * Fix typo. * WIP serializing stdlib. * WIP serializing stdlib in. * Fix problem serializing arrays that hold data that is already serialized. * Remove clash of names in MagicTypeModifier. * Make conversion from char to String explicit. Fix reference count issue with SerialReader. * Add code to save/load stdlib. * Use return code to avoid warning - SerialContainerUtil::write(module, options, &stream)) * Make all String numeric ctors explicit. Added isChar to UnownedStringSlice. Added operator== for UnownedStringSlice to String to avoid need to convert to String and allocate. * Add error check to readAllText. * tabs -> spaces on String.h * tab -> spaces String.cpp * Remove msg for StringBuilder, just build inplace for exceptions. * Check SerialClasses - for name clashes. Renamed Modifier::name as Modifier::keywordName * Handling of extensions when deserializing AST - updating the moduleDecl->mapTypeToCandidateExtensions Co-authored-by: Tim Foley --- source/slang/slang.cpp | 140 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 132 insertions(+), 8 deletions(-) (limited to 'source/slang/slang.cpp') diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp index 90ddf030c..a66441948 100644 --- a/source/slang/slang.cpp +++ b/source/slang/slang.cpp @@ -136,7 +136,6 @@ void Session::init() // And the global ASTBuilder globalAstBuilder = new ASTBuilder(m_sharedASTBuilder, "globalAstBuilder"); - // Make sure our source manager is initialized builtinSourceManager.initialize(nullptr, nullptr); @@ -175,8 +174,41 @@ void Session::init() slangLanguageScope = new Scope(); slangLanguageScope->nextSibling = hlslLanguageScope; - addBuiltinSource(coreLanguageScope, "core", getCoreLibraryCode()); - addBuiltinSource(hlslLanguageScope, "hlsl", getHLSLLibraryCode()); + if (false) + { + // Let's try loading serialized modules and adding them + _readBuiltinModule(coreLanguageScope, "core"); + _readBuiltinModule(hlslLanguageScope, "hlsl"); + } + else + { + addBuiltinSource(coreLanguageScope, "core", getCoreLibraryCode()); + addBuiltinSource(hlslLanguageScope, "hlsl", getHLSLLibraryCode()); + + // Write out + if (false) + { + for (auto& pair : m_builtinLinkage->mapNameToLoadedModules) + { + const Name* moduleName = pair.Key; + Module* module = pair.Value; + + // Set up options + SerialContainerUtil::WriteOptions options; + //options.optionFlags |= SerialOptionFlag::SourceLocation; + options.sourceManager = m_builtinLinkage->getSourceManager(); + + StringBuilder builder; + builder << moduleName->text << ".slang-module"; + + FileStream stream(builder.ProduceString(), FileMode::Create, FileAccess::Write, FileShare::ReadWrite); + if (SLANG_FAILED(SerialContainerUtil::write(module, options, &stream))) + { + SLANG_UNEXPECTED("Unable to load stdlib"); + } + } + } + } { for (Index i = 0; i < Index(SourceLanguage::CountOf); ++i) @@ -194,6 +226,88 @@ void Session::init() m_languagePreludes[Index(SourceLanguage::HLSL)] = get_slang_hlsl_prelude(); } +SlangResult Session::_readBuiltinModule(Scope* scope, String moduleName) +{ + StringBuilder moduleFilename; + moduleFilename << moduleName << ".slang-module"; + + RiffContainer riffContainer; + try + { + FileStream stream(moduleFilename.ProduceString(), FileMode::Open, FileAccess::Read, FileShare::ReadOnly); + // Load the riff container + SLANG_RETURN_ON_FAIL(RiffUtil::read(&stream, riffContainer)); + } + catch (const IOException&) + { + return SLANG_FAIL; + } + + // Load up the module + + SerialContainerData containerData; + + Linkage* linkage = getBuiltinLinkage(); + + NamePool* sessionNamePool = &namePool; + NamePool* linkageNamePool = linkage->getNamePool(); + + SerialContainerUtil::ReadOptions options; + options.namePool = linkageNamePool; + options.session = this; + options.sharedASTBuilder = linkage->getASTBuilder()->getSharedASTBuilder(); + options.sourceManager = linkage->getSourceManager(); + options.linkage = linkage; + + // Hmm - don't have a suitable sink yet, so attempt to just not have one + options.sink = nullptr; + + SLANG_RETURN_ON_FAIL(SerialContainerUtil::read(&riffContainer, options, containerData)); + + for (auto& srcModule : containerData.modules) + { + RefPtr module(new Module(linkage, srcModule.astBuilder)); + + ModuleDecl* moduleDecl = as(srcModule.astRootNode); + + if (moduleDecl) + { + if (isFromStdLib(moduleDecl)) + { + registerBuiltinDecls(this, moduleDecl); + } + + module->setModuleDecl(moduleDecl); + } + + module->setIRModule(srcModule.irModule); + + // Put in the loaded module map + linkage->mapNameToLoadedModules.Add(sessionNamePool->getName(moduleName), module); + + // Add the resulting code to the appropriate scope + if (!scope->containerDecl) + { + // We are the first chunk of code to be loaded for this scope + scope->containerDecl = moduleDecl; + } + else + { + // We need to create a new scope to link into the whole thing + auto subScope = new Scope(); + subScope->containerDecl = moduleDecl; + subScope->nextSibling = scope->nextSibling; + scope->nextSibling = subScope; + } + + // We need to retain this AST so that we can use it in other code + // (Note that the `Scope` type does not retain the AST it points to) + stdlibModules.add(module); + } + + return SLANG_OK; +} + ISlangUnknown* Session::getInterface(const Guid& guid) { if(guid == IID_ISlangUnknown || guid == IID_IGlobalSession) @@ -1240,7 +1354,7 @@ void FrontEndCompileRequest::generateIR() options.compressionType = SerialCompressionType::None; options.sourceManager = getSourceManager(); - options.optionFlags |= SerialOptionFlag::DebugInfo; + options.optionFlags |= SerialOptionFlag::SourceLocation; // Verify debug information if (SLANG_FAILED(SerialContainerUtil::verifyIRSerialize(irModule, getSession(), options))) @@ -1950,11 +2064,19 @@ void FilePathDependencyList::addDependency(Module* module) // Module // -Module::Module(Linkage* linkage) +Module::Module(Linkage* linkage, ASTBuilder* astBuilder) : ComponentType(linkage) - , m_astBuilder(linkage->getASTBuilder()->getSharedASTBuilder(), "Module") , m_mangledExportPool(StringSlicePool::Style::Empty) { + if (astBuilder) + { + m_astBuilder = astBuilder; + } + else + { + m_astBuilder = new ASTBuilder(linkage->getASTBuilder()->getSharedASTBuilder(), "Module"); + } + addModuleDependency(this); } @@ -2053,7 +2175,7 @@ void Module::_processFindDeclsExportSymbolsRec(Decl* decl) return; } - // process `decl` itself + // If it's a container process it's children if(auto containerDecl = as(decl)) { for (auto child : containerDecl->members) @@ -2061,7 +2183,9 @@ void Module::_processFindDeclsExportSymbolsRec(Decl* decl) _processFindDeclsExportSymbolsRec(child); } } - else if (auto genericDecl = as(decl)) + + // GenericDecl is also a container, so do subsequent test + if (auto genericDecl = as(decl)) { _processFindDeclsExportSymbolsRec(genericDecl->inner); } -- cgit v1.2.3