From c985f5f2f95dc95998fdfb8400baa0a04760ada2 Mon Sep 17 00:00:00 2001 From: jsmall-nvidia Date: Thu, 5 Nov 2020 13:43:00 -0500 Subject: Standard library save/loadable (#1592) * #include an absolute path didn't work - because paths were taken to always be relative. * Fix handling of access modifiers inside type definition. * Fix access problem for AST node. Make dumping produce a single function with switch, to potentially make available without Dump specific access. * WIP on serialization design doc. * Remove project references to previously generated files. * More docs on serialization design. * Improve serialization documentation. Remove unused function from IRSerialReader. * Small fixes around naming. Remove long comment from slang-serialize.h - as covered in serialization.md * Remove long comment in slang-serialize.h as covered in serialization.md * More information about doing replacements on read for AST and problems surrounding. * Typo fix. * Spelling fixes. * Value serialize. * Value types with inheritence. * Use value reflection serial conversion for more AST types * Use automatic serialization on more of AST. * Get the types via decltype, simplifies what the extractor has to do. * Update the serialization.md for the value serialization. * Small doc improvements. * Update project. * Remove ImportExternalDecl type Added addImportSymbol and ImportSymbol type Fixed bug in container which meant it wouldn't read back AST module * Because of change of how imports and handled, store objects as SerialPointers. * First pass symbol lookup from mangled names. * Cache current module looked up from mangled name. * Fix SourceLoc bug. Improve comments. * Added diagnostic on mangled symbol not being found * Fix typo. * WIP serializing stdlib. * WIP serializing stdlib in. * Fix problem serializing arrays that hold data that is already serialized. * Remove clash of names in MagicTypeModifier. * Make conversion from char to String explicit. Fix reference count issue with SerialReader. * Add code to save/load stdlib. * Use return code to avoid warning - SerialContainerUtil::write(module, options, &stream)) * Make all String numeric ctors explicit. Added isChar to UnownedStringSlice. Added operator== for UnownedStringSlice to String to avoid need to convert to String and allocate. * Add error check to readAllText. * tabs -> spaces on String.h * tab -> spaces String.cpp * Remove msg for StringBuilder, just build inplace for exceptions. * Check SerialClasses - for name clashes. Renamed Modifier::name as Modifier::keywordName * Handling of extensions when deserializing AST - updating the moduleDecl->mapTypeToCandidateExtensions Co-authored-by: Tim Foley --- source/slang/slang-serialize-container.cpp | 150 ++++++++++++++++++++++++----- 1 file changed, 127 insertions(+), 23 deletions(-) (limited to 'source/slang/slang-serialize-container.cpp') diff --git a/source/slang/slang-serialize-container.cpp b/source/slang/slang-serialize-container.cpp index 8c4edb9a0..067e6637a 100644 --- a/source/slang/slang-serialize-container.cpp +++ b/source/slang/slang-serialize-container.cpp @@ -16,42 +16,100 @@ namespace Slang { -/* static */SlangResult SerialContainerUtil::requestToData(EndToEndCompileRequest* request, const WriteOptions& options, SerialContainerData& out) +/* static */SlangResult SerialContainerUtil::write(Module* module, const WriteOptions& options, Stream* stream) { - SLANG_UNUSED(options); - - out.clear(); - - auto linkage = request->getLinkage(); - auto sink = request->getSink(); - auto frontEndReq = request->getFrontEndReq(); - - for (TranslationUnitRequest* translationUnit : frontEndReq->translationUnits) + RiffContainer container; { - auto module = translationUnit->module; - auto irModule = module->getIRModule(); + SerialContainerData data; + SLANG_RETURN_ON_FAIL(SerialContainerUtil::addModuleToData(module, options, data)); + SLANG_RETURN_ON_FAIL(SerialContainerUtil::write(data, options, &container)); + } + // We now write the RiffContainer to the stream + SLANG_RETURN_ON_FAIL(RiffUtil::write(container.getRoot(), true, stream)); + return SLANG_OK; +} - // Root AST node - auto moduleDecl = translationUnit->getModuleDecl(); +/* static */SlangResult SerialContainerUtil::write(FrontEndCompileRequest* frontEndReq, const WriteOptions& options, Stream* stream) +{ + RiffContainer container; + { + SerialContainerData data; + SLANG_RETURN_ON_FAIL(SerialContainerUtil::addFrontEndRequestToData(frontEndReq, options, data)); + SLANG_RETURN_ON_FAIL(SerialContainerUtil::write(data, options, &container)); + } + // We now write the RiffContainer to the stream + SLANG_RETURN_ON_FAIL(RiffUtil::write(container.getRoot(), true, stream)); + return SLANG_OK; +} - SLANG_ASSERT(irModule || moduleDecl); +/* static */SlangResult SerialContainerUtil::write(EndToEndCompileRequest* request, const WriteOptions& options, Stream* stream) +{ + RiffContainer container; + { + SerialContainerData data; + SLANG_RETURN_ON_FAIL(SerialContainerUtil::addEndToEndRequestToData(request, options, data)); + SLANG_RETURN_ON_FAIL(SerialContainerUtil::write(data, options, &container)); + } + // We now write the RiffContainer to the stream + SLANG_RETURN_ON_FAIL(RiffUtil::write(container.getRoot(), true, stream)); + return SLANG_OK; +} +/* static */SlangResult SerialContainerUtil::addModuleToData(Module* module, const WriteOptions& options, SerialContainerData& outData) +{ + if (options.optionFlags & (SerialOptionFlag::ASTModule | SerialOptionFlag::IRModule)) + { SerialContainerData::Module dstModule; // NOTE: The astBuilder is not set here, as not needed to be scoped for serialization (it is assumed the // TranslationUnitRequest stays in scope) - dstModule.astRootNode = moduleDecl; - dstModule.irModule = irModule; + if (options.optionFlags & SerialOptionFlag::ASTModule) + { + // Root AST node + auto moduleDecl = module->getModuleDecl(); + SLANG_ASSERT(moduleDecl); - out.modules.add(dstModule); + dstModule.astRootNode = moduleDecl; + } + if (options.optionFlags & SerialOptionFlag::IRModule) + { + // IR module + dstModule.irModule = module->getIRModule(); + SLANG_ASSERT(dstModule.irModule); + } + + outData.modules.add(dstModule); + } + + return SLANG_OK; +} + + +/* static */SlangResult SerialContainerUtil::addFrontEndRequestToData(FrontEndCompileRequest* frontEndReq, const WriteOptions& options, SerialContainerData& outData) +{ + // Go through translation units, adding modules + for (TranslationUnitRequest* translationUnit : frontEndReq->translationUnits) + { + SLANG_RETURN_ON_FAIL(addModuleToData(translationUnit->module, options, outData)); } + return SLANG_OK; +} + +/* static */SlangResult SerialContainerUtil::addEndToEndRequestToData(EndToEndCompileRequest* request, const WriteOptions& options, SerialContainerData& out) +{ + auto linkage = request->getLinkage(); + auto sink = request->getSink(); + + // Output the front end request data + SLANG_RETURN_ON_FAIL(addFrontEndRequestToData(request->getFrontEndReq(), options, out)); + + // auto program = request->getSpecializedGlobalAndEntryPointsComponentType(); // Add all the target modules { - for (auto target : linkage->targets) { auto targetProgram = program->getTargetProgram(target); @@ -117,7 +175,7 @@ namespace Slang { // Module list RiffContainer::ScopeChunk moduleListScope(container, RiffContainer::Chunk::Kind::List, SerialBinary::kModuleListFourCc); - if (options.optionFlags & SerialOptionFlag::DebugInfo) + if (options.optionFlags & SerialOptionFlag::SourceLocation) { sourceLocWriter = new SerialSourceLocWriter(options.sourceManager); } @@ -226,6 +284,20 @@ namespace Slang { return SLANG_OK; } + +static List& _getCandidateExtensionList( + AggTypeDecl* typeDecl, + Dictionary>& mapTypeToCandidateExtensions) +{ + RefPtr entry; + if (!mapTypeToCandidateExtensions.TryGetValue(typeDecl, entry)) + { + entry = new CandidateExtensionList(); + mapTypeToCandidateExtensions.Add(typeDecl, entry); + } + return entry->candidateExtensions; +} + /* static */Result SerialContainerUtil::read(RiffContainer* container, const ReadOptions& options, SerialContainerData& out) { out.clear(); @@ -396,6 +468,10 @@ namespace Slang { // Set the sourceLocReader before doing de-serialize, such can lookup the remapped sourceLocs reader.getExtraObjects().set(sourceLocReader); + // Go through all of the AST nodes + // 1) Set the ASTBuilder on Type nodes + + // TODO(JS): // If modules can have more complicated relationships (like a two modules can refer to symbols // from each other), then we can make this work by @@ -412,6 +488,34 @@ namespace Slang { // Get the root node. It's at index 1 (0 is the null value). astRootNode = reader.getPointer(SerialIndex(1)).dynamicCast(); + + // 2) Add the extensions to the module mapTypeToCandidateExtensions cache + + { + ModuleDecl* moduleDecl = as(astRootNode); + + for (auto& obj : reader.getObjects()) + { + if (Type* type = obj.dynamicCast()) + { + type->_setASTBuilder(astBuilder); + } + + if (ExtensionDecl* extensionDecl = obj.dynamicCast()) + { + if (auto targetDeclRefType = as(extensionDecl->targetType)) + { + // Attach our extension to that type as a candidate... + if (auto aggTypeDeclRef = targetDeclRefType->declRef.as()) + { + auto aggTypeDecl = aggTypeDeclRef.getDecl(); + + _getCandidateExtensionList(aggTypeDecl, moduleDecl->mapTypeToCandidateExtensions).add(extensionDecl); + } + } + } + } + } } // Onto next chunk @@ -489,7 +593,7 @@ namespace Slang { RefPtr sourceLocWriter; - if (options.optionFlags & SerialOptionFlag::DebugInfo) + if (options.optionFlags & SerialOptionFlag::SourceLocation) { sourceLocWriter = new SerialSourceLocWriter(options.sourceManager); } @@ -530,7 +634,7 @@ namespace Slang { RefPtr sourceLocReader; // If we have debug info then find and read it - if (options.optionFlags & SerialOptionFlag::DebugInfo) + if (options.optionFlags & SerialOptionFlag::SourceLocation) { RiffContainer::ListChunk* debugList = rootList->findContainedList(SerialSourceLocData::kDebugFourCc); if (!debugList) @@ -593,7 +697,7 @@ namespace Slang { } } } - else if (options.optionFlags & SerialOptionFlag::DebugInfo) + else if (options.optionFlags & SerialOptionFlag::SourceLocation) { // They should be on the same line nos for (Index i = 1; i < readInsts.getCount(); ++i) -- cgit v1.2.3