From c985f5f2f95dc95998fdfb8400baa0a04760ada2 Mon Sep 17 00:00:00 2001 From: jsmall-nvidia Date: Thu, 5 Nov 2020 13:43:00 -0500 Subject: Standard library save/loadable (#1592) * #include an absolute path didn't work - because paths were taken to always be relative. * Fix handling of access modifiers inside type definition. * Fix access problem for AST node. Make dumping produce a single function with switch, to potentially make available without Dump specific access. * WIP on serialization design doc. * Remove project references to previously generated files. * More docs on serialization design. * Improve serialization documentation. Remove unused function from IRSerialReader. * Small fixes around naming. Remove long comment from slang-serialize.h - as covered in serialization.md * Remove long comment in slang-serialize.h as covered in serialization.md * More information about doing replacements on read for AST and problems surrounding. * Typo fix. * Spelling fixes. * Value serialize. * Value types with inheritence. * Use value reflection serial conversion for more AST types * Use automatic serialization on more of AST. * Get the types via decltype, simplifies what the extractor has to do. * Update the serialization.md for the value serialization. * Small doc improvements. * Update project. * Remove ImportExternalDecl type Added addImportSymbol and ImportSymbol type Fixed bug in container which meant it wouldn't read back AST module * Because of change of how imports and handled, store objects as SerialPointers. * First pass symbol lookup from mangled names. * Cache current module looked up from mangled name. * Fix SourceLoc bug. Improve comments. * Added diagnostic on mangled symbol not being found * Fix typo. * WIP serializing stdlib. * WIP serializing stdlib in. * Fix problem serializing arrays that hold data that is already serialized. * Remove clash of names in MagicTypeModifier. * Make conversion from char to String explicit. Fix reference count issue with SerialReader. * Add code to save/load stdlib. * Use return code to avoid warning - SerialContainerUtil::write(module, options, &stream)) * Make all String numeric ctors explicit. Added isChar to UnownedStringSlice. Added operator== for UnownedStringSlice to String to avoid need to convert to String and allocate. * Add error check to readAllText. * tabs -> spaces on String.h * tab -> spaces String.cpp * Remove msg for StringBuilder, just build inplace for exceptions. * Check SerialClasses - for name clashes. Renamed Modifier::name as Modifier::keywordName * Handling of extensions when deserializing AST - updating the moduleDecl->mapTypeToCandidateExtensions Co-authored-by: Tim Foley --- source/slang/slang-ast-base.h | 9 +- source/slang/slang-ast-builder.cpp | 2 +- source/slang/slang-ast-modifier.h | 7 +- source/slang/slang-ast-support-types.h | 6 +- source/slang/slang-check-decl.cpp | 5 + source/slang/slang-check-modifier.cpp | 10 +- source/slang/slang-check.h | 2 + source/slang/slang-compiler.cpp | 4 +- source/slang/slang-compiler.h | 8 +- source/slang/slang-parser.cpp | 10 +- source/slang/slang-reflection.cpp | 4 +- source/slang/slang-serialize-ast-type-info.h | 8 ++ source/slang/slang-serialize-container.cpp | 150 +++++++++++++++++++++++---- source/slang/slang-serialize-container.h | 14 ++- source/slang/slang-serialize-factory.cpp | 3 + source/slang/slang-serialize-ir.cpp | 2 +- source/slang/slang-serialize-type-info.h | 96 +++++++++++------ source/slang/slang-serialize-types.h | 2 +- source/slang/slang-serialize.cpp | 69 +++++++++++- source/slang/slang-serialize.h | 26 +++-- source/slang/slang-syntax.cpp | 22 ++-- source/slang/slang.cpp | 140 +++++++++++++++++++++++-- 22 files changed, 484 insertions(+), 115 deletions(-) (limited to 'source/slang') diff --git a/source/slang/slang-ast-base.h b/source/slang/slang-ast-base.h index e9005212c..2bfa7940b 100644 --- a/source/slang/slang-ast-base.h +++ b/source/slang/slang-ast-base.h @@ -155,6 +155,8 @@ class Type: public Val bool _equalsImplOverride(Type* type); Type* _createCanonicalTypeOverride(); + void _setASTBuilder(ASTBuilder* astBuilder) { m_astBuilder = astBuilder; } + protected: bool equalsImpl(Type* type); Type* createCanonicalType(); @@ -276,10 +278,10 @@ class Modifier : public SyntaxNode Modifier* next = nullptr; // The keyword that was used to introduce t that was used to name this modifier. - Name* name = nullptr; + Name* keywordName = nullptr; - Name* getName() { return name; } - NameLoc getNameAndLoc() { return NameLoc(name, loc); } + Name* getKeywordName() { return keywordName; } + NameLoc getKeywordNameAndLoc() { return NameLoc(keywordName, loc); } }; // A syntax node which can have modifiers applied @@ -327,7 +329,6 @@ public: SourceLoc getNameLoc() { return nameAndLoc.loc ; } NameLoc getNameAndLoc() { return nameAndLoc ; } - DeclCheckStateExt checkState = DeclCheckState::Unchecked; // The next declaration defined in the same container with the same name diff --git a/source/slang/slang-ast-builder.cpp b/source/slang/slang-ast-builder.cpp index 54fe55478..3b97bfbb7 100644 --- a/source/slang/slang-ast-builder.cpp +++ b/source/slang/slang-ast-builder.cpp @@ -140,7 +140,7 @@ void SharedASTBuilder::registerMagicDecl(Decl* decl, MagicTypeModifier* modifier if (auto genericDecl = as(decl->parentDecl)) declToRegister = genericDecl; - m_magicDecls[modifier->name] = declToRegister; + m_magicDecls[modifier->magicName] = declToRegister; } Decl* SharedASTBuilder::findMagicDecl(const String& name) diff --git a/source/slang/slang-ast-modifier.h b/source/slang/slang-ast-modifier.h index d707e9771..55b62483b 100644 --- a/source/slang/slang-ast-modifier.h +++ b/source/slang/slang-ast-modifier.h @@ -350,9 +350,10 @@ class BuiltinTypeModifier : public Modifier class MagicTypeModifier : public Modifier { SLANG_AST_CLASS(MagicTypeModifier) - - String name; - uint32_t tag; + + /// Modifier has a name so call this magicModifier to disambiguate + String magicName; + uint32_t tag = uint32_t(0); }; // A modifier applied to declarations of builtin types to indicate how they diff --git a/source/slang/slang-ast-support-types.h b/source/slang/slang-ast-support-types.h index 10bf46b63..8338f0b21 100644 --- a/source/slang/slang-ast-support-types.h +++ b/source/slang/slang-ast-support-types.h @@ -1255,6 +1255,8 @@ namespace Slang // requirement by a particular declaration or value. struct RequirementWitness { + SLANG_VALUE_CLASS(RequirementWitness) + RequirementWitness() : m_flavor(Flavor::none) {} @@ -1305,8 +1307,10 @@ namespace Slang typedef Dictionary RequirementDictionary; - struct WitnessTable : RefObject + struct WitnessTable : SerialRefObject { + SLANG_OBJ_CLASS(WitnessTable) + List> requirementList; RequirementDictionary requirementDictionary; diff --git a/source/slang/slang-check-decl.cpp b/source/slang/slang-check-decl.cpp index d771e689c..e7c2a5853 100644 --- a/source/slang/slang-check-decl.cpp +++ b/source/slang/slang-check-decl.cpp @@ -1231,6 +1231,11 @@ namespace Slang } } + void registerBuiltinDecls(Session* session, Decl* decl) + { + _registerBuiltinDeclsRec(session, decl); + } + void SemanticsDeclVisitorBase::checkModule(ModuleDecl* moduleDecl) { // When we are dealing with code from the standard library, diff --git a/source/slang/slang-check-modifier.cpp b/source/slang/slang-check-modifier.cpp index b38018354..4594af28d 100644 --- a/source/slang/slang-check-modifier.cpp +++ b/source/slang/slang-check-modifier.cpp @@ -395,7 +395,7 @@ namespace Slang // Let it go thru iff single string attribute if (!hasStringArgs(attr, 1)) { - getSink()->diagnose(attr, Diagnostics::expectedSingleStringArg, attr->name); + getSink()->diagnose(attr, Diagnostics::expectedSingleStringArg, attr->keywordName); } } else if (as(attr)) @@ -403,7 +403,7 @@ namespace Slang // Let it go thru iff single integral attribute if (!hasIntArgs(attr, 1)) { - getSink()->diagnose(attr, Diagnostics::expectedSingleIntArg, attr->name); + getSink()->diagnose(attr, Diagnostics::expectedSingleIntArg, attr->keywordName); } } else if (as(attr)) @@ -433,7 +433,7 @@ namespace Slang } else { - getSink()->diagnose(attr, Diagnostics::expectedSingleIntArg, attr->name); + getSink()->diagnose(attr, Diagnostics::expectedSingleIntArg, attr->keywordName); return false; } } @@ -551,7 +551,7 @@ namespace Slang UncheckedAttribute* uncheckedAttr, ModifiableSyntaxNode* attrTarget) { - auto attrName = uncheckedAttr->getName(); + auto attrName = uncheckedAttr->getKeywordName(); auto attrDecl = lookUpAttributeDecl( attrName, uncheckedAttr->scope); @@ -580,7 +580,7 @@ namespace Slang // We are going to replace the unchecked attribute with the checked one. // First copy all of the state over from the original attribute. - attr->name = uncheckedAttr->name; + attr->keywordName = uncheckedAttr->keywordName; attr->args = uncheckedAttr->args; attr->loc = uncheckedAttr->loc; diff --git a/source/slang/slang-check.h b/source/slang/slang-check.h index 94deba1aa..0309e5393 100644 --- a/source/slang/slang-check.h +++ b/source/slang/slang-check.h @@ -36,4 +36,6 @@ namespace Slang bool isGlobalShaderParameter(VarDeclBase* decl); bool isFromStdLib(Decl* decl); + + void registerBuiltinDecls(Session* session, Decl* decl); } diff --git a/source/slang/slang-compiler.cpp b/source/slang/slang-compiler.cpp index ea8540800..306119b5f 100755 --- a/source/slang/slang-compiler.cpp +++ b/source/slang/slang-compiler.cpp @@ -2346,7 +2346,7 @@ SlangResult dissassembleDXILUsingDXC( options.compressionType = linkage->serialCompressionType; if (linkage->debugInfoLevel != DebugInfoLevel::None) { - options.optionFlags |= SerialOptionFlag::DebugInfo; + options.optionFlags |= SerialOptionFlag::SourceLocation; } if (linkage->m_obfuscateCode) { @@ -2360,7 +2360,7 @@ SlangResult dissassembleDXILUsingDXC( RiffContainer container; { SerialContainerData data; - SLANG_RETURN_ON_FAIL(SerialContainerUtil::requestToData(this, options, data)); + SLANG_RETURN_ON_FAIL(SerialContainerUtil::addEndToEndRequestToData(this, options, data)); SLANG_RETURN_ON_FAIL(SerialContainerUtil::write(data, options, &container)); } // We now write the RiffContainer to the stream diff --git a/source/slang/slang-compiler.h b/source/slang/slang-compiler.h index 56885ab46..1710baa6e 100755 --- a/source/slang/slang-compiler.h +++ b/source/slang/slang-compiler.h @@ -925,7 +925,7 @@ namespace Slang // /// Create a module (initially empty). - Module(Linkage* linkage); + Module(Linkage* linkage, ASTBuilder* astBuilder = nullptr); /// Get the AST for the module (if it has been parsed) ModuleDecl* getModuleDecl() { return m_moduleDecl; } @@ -978,7 +978,7 @@ namespace Slang NodeBase* findExportFromMangledName(const UnownedStringSlice& slice); /// Get the ASTBuilder - ASTBuilder* getASTBuilder() { return &m_astBuilder; } + ASTBuilder* getASTBuilder() { return m_astBuilder; } /// Collect information on the shader parameters of the module. /// @@ -1058,7 +1058,7 @@ namespace Slang // The builder that owns all of the AST nodes from parsing the source of // this module. - ASTBuilder m_astBuilder; + RefPtr m_astBuilder; // Holds map of exported mangled names to symbols. m_mangledExportPool maps names to indices, // and m_mangledExportSymbols holds the NodeBase* values for each index. @@ -2170,6 +2170,8 @@ namespace Slang private: + SlangResult _readBuiltinModule(Scope* scope, String moduleName); + SlangResult _loadRequest(EndToEndCompileRequest* request, const void* data, size_t size); /// Linkage used for all built-in (stdlib) code. diff --git a/source/slang/slang-parser.cpp b/source/slang/slang-parser.cpp index e02ccf245..4d93c35a8 100644 --- a/source/slang/slang-parser.cpp +++ b/source/slang/slang-parser.cpp @@ -730,7 +730,7 @@ namespace Slang Token nameToken = parseAttributeName(parser); UncheckedAttribute* modifier = parser->astBuilder->create(); - modifier->name = nameToken.getName(); + modifier->keywordName = nameToken.getName(); modifier->loc = nameToken.getLoc(); modifier->scope = parser->currentScope; @@ -903,7 +903,7 @@ namespace Slang Modifier* parsedModifier = nullptr; if (tryParseUsingSyntaxDecl(parser, &parsedModifier)) { - parsedModifier->name = nameToken.getName(); + parsedModifier->keywordName = nameToken.getName(); if (!parsedModifier->loc.isValid()) { parsedModifier->loc = nameToken.loc; @@ -5504,7 +5504,7 @@ namespace Slang numThreadsAttrib->args.setCount(3); // Just mark the loc and name from the first in the list - numThreadsAttrib->name = getName(parser, "numthreads"); + numThreadsAttrib->keywordName = getName(parser, "numthreads"); numThreadsAttrib->loc = nameAndLoc.loc; numThreadsAttrib->scope = parser->currentScope; } @@ -5566,7 +5566,7 @@ namespace Slang SLANG_ASSERT(modifier); #undef CASE - modifier->name = nameAndLoc.name; + modifier->keywordName = nameAndLoc.name; modifier->loc = nameAndLoc.loc; // Special handling for GLSLLayoutModifier @@ -5610,7 +5610,7 @@ namespace Slang { MagicTypeModifier* modifier = parser->astBuilder->create(); parser->ReadToken(TokenType::LParent); - modifier->name = parser->ReadToken(TokenType::Identifier).getContent(); + modifier->magicName = parser->ReadToken(TokenType::Identifier).getContent(); if (AdvanceIf(parser, TokenType::Comma)) { modifier->tag = uint32_t(StringToInt(parser->ReadToken(TokenType::IntegerLiteral).getContent())); diff --git a/source/slang/slang-reflection.cpp b/source/slang/slang-reflection.cpp index 1b0a590cc..fd76c5e0b 100644 --- a/source/slang/slang-reflection.cpp +++ b/source/slang/slang-reflection.cpp @@ -114,7 +114,7 @@ SlangReflectionUserAttribute* findUserAttributeByName(Session* session, Decl* de auto nameObj = session->tryGetNameObj(name); for (auto x : decl->getModifiersOfType()) { - if (x->name == nameObj) + if (x->keywordName == nameObj) return (SlangReflectionUserAttribute*)(x); } return nullptr; @@ -136,7 +136,7 @@ SLANG_API char const* spReflectionUserAttribute_GetName(SlangReflectionUserAttri { auto userAttr = convert(attrib); if (!userAttr) return nullptr; - return userAttr->getName()->text.getBuffer(); + return userAttr->getKeywordName()->text.getBuffer(); } SLANG_API unsigned int spReflectionUserAttribute_GetArgumentCount(SlangReflectionUserAttribute* attrib) { diff --git a/source/slang/slang-serialize-ast-type-info.h b/source/slang/slang-serialize-ast-type-info.h index 733694e8a..400474046 100644 --- a/source/slang/slang-serialize-ast-type-info.h +++ b/source/slang/slang-serialize-ast-type-info.h @@ -92,6 +92,7 @@ SLANG_VALUE_TYPE_INFO(LookupResultItem) // QualType SLANG_VALUE_TYPE_INFO(QualType) + // LookupResult template <> struct SerialTypeInfo @@ -205,6 +206,13 @@ struct SerialTypeInfo : public S template <> struct SerialTypeInfo : public SerialConvertTypeInfo {}; +// RequirementWitness::Flavor +template <> +struct SerialTypeInfo : public SerialConvertTypeInfo {}; + +// RequirementWitness +SLANG_VALUE_TYPE_INFO(RequirementWitness) + } // namespace Slang #endif diff --git a/source/slang/slang-serialize-container.cpp b/source/slang/slang-serialize-container.cpp index 8c4edb9a0..067e6637a 100644 --- a/source/slang/slang-serialize-container.cpp +++ b/source/slang/slang-serialize-container.cpp @@ -16,42 +16,100 @@ namespace Slang { -/* static */SlangResult SerialContainerUtil::requestToData(EndToEndCompileRequest* request, const WriteOptions& options, SerialContainerData& out) +/* static */SlangResult SerialContainerUtil::write(Module* module, const WriteOptions& options, Stream* stream) { - SLANG_UNUSED(options); - - out.clear(); - - auto linkage = request->getLinkage(); - auto sink = request->getSink(); - auto frontEndReq = request->getFrontEndReq(); - - for (TranslationUnitRequest* translationUnit : frontEndReq->translationUnits) + RiffContainer container; { - auto module = translationUnit->module; - auto irModule = module->getIRModule(); + SerialContainerData data; + SLANG_RETURN_ON_FAIL(SerialContainerUtil::addModuleToData(module, options, data)); + SLANG_RETURN_ON_FAIL(SerialContainerUtil::write(data, options, &container)); + } + // We now write the RiffContainer to the stream + SLANG_RETURN_ON_FAIL(RiffUtil::write(container.getRoot(), true, stream)); + return SLANG_OK; +} - // Root AST node - auto moduleDecl = translationUnit->getModuleDecl(); +/* static */SlangResult SerialContainerUtil::write(FrontEndCompileRequest* frontEndReq, const WriteOptions& options, Stream* stream) +{ + RiffContainer container; + { + SerialContainerData data; + SLANG_RETURN_ON_FAIL(SerialContainerUtil::addFrontEndRequestToData(frontEndReq, options, data)); + SLANG_RETURN_ON_FAIL(SerialContainerUtil::write(data, options, &container)); + } + // We now write the RiffContainer to the stream + SLANG_RETURN_ON_FAIL(RiffUtil::write(container.getRoot(), true, stream)); + return SLANG_OK; +} - SLANG_ASSERT(irModule || moduleDecl); +/* static */SlangResult SerialContainerUtil::write(EndToEndCompileRequest* request, const WriteOptions& options, Stream* stream) +{ + RiffContainer container; + { + SerialContainerData data; + SLANG_RETURN_ON_FAIL(SerialContainerUtil::addEndToEndRequestToData(request, options, data)); + SLANG_RETURN_ON_FAIL(SerialContainerUtil::write(data, options, &container)); + } + // We now write the RiffContainer to the stream + SLANG_RETURN_ON_FAIL(RiffUtil::write(container.getRoot(), true, stream)); + return SLANG_OK; +} +/* static */SlangResult SerialContainerUtil::addModuleToData(Module* module, const WriteOptions& options, SerialContainerData& outData) +{ + if (options.optionFlags & (SerialOptionFlag::ASTModule | SerialOptionFlag::IRModule)) + { SerialContainerData::Module dstModule; // NOTE: The astBuilder is not set here, as not needed to be scoped for serialization (it is assumed the // TranslationUnitRequest stays in scope) - dstModule.astRootNode = moduleDecl; - dstModule.irModule = irModule; + if (options.optionFlags & SerialOptionFlag::ASTModule) + { + // Root AST node + auto moduleDecl = module->getModuleDecl(); + SLANG_ASSERT(moduleDecl); - out.modules.add(dstModule); + dstModule.astRootNode = moduleDecl; + } + if (options.optionFlags & SerialOptionFlag::IRModule) + { + // IR module + dstModule.irModule = module->getIRModule(); + SLANG_ASSERT(dstModule.irModule); + } + + outData.modules.add(dstModule); + } + + return SLANG_OK; +} + + +/* static */SlangResult SerialContainerUtil::addFrontEndRequestToData(FrontEndCompileRequest* frontEndReq, const WriteOptions& options, SerialContainerData& outData) +{ + // Go through translation units, adding modules + for (TranslationUnitRequest* translationUnit : frontEndReq->translationUnits) + { + SLANG_RETURN_ON_FAIL(addModuleToData(translationUnit->module, options, outData)); } + return SLANG_OK; +} + +/* static */SlangResult SerialContainerUtil::addEndToEndRequestToData(EndToEndCompileRequest* request, const WriteOptions& options, SerialContainerData& out) +{ + auto linkage = request->getLinkage(); + auto sink = request->getSink(); + + // Output the front end request data + SLANG_RETURN_ON_FAIL(addFrontEndRequestToData(request->getFrontEndReq(), options, out)); + + // auto program = request->getSpecializedGlobalAndEntryPointsComponentType(); // Add all the target modules { - for (auto target : linkage->targets) { auto targetProgram = program->getTargetProgram(target); @@ -117,7 +175,7 @@ namespace Slang { // Module list RiffContainer::ScopeChunk moduleListScope(container, RiffContainer::Chunk::Kind::List, SerialBinary::kModuleListFourCc); - if (options.optionFlags & SerialOptionFlag::DebugInfo) + if (options.optionFlags & SerialOptionFlag::SourceLocation) { sourceLocWriter = new SerialSourceLocWriter(options.sourceManager); } @@ -226,6 +284,20 @@ namespace Slang { return SLANG_OK; } + +static List& _getCandidateExtensionList( + AggTypeDecl* typeDecl, + Dictionary>& mapTypeToCandidateExtensions) +{ + RefPtr entry; + if (!mapTypeToCandidateExtensions.TryGetValue(typeDecl, entry)) + { + entry = new CandidateExtensionList(); + mapTypeToCandidateExtensions.Add(typeDecl, entry); + } + return entry->candidateExtensions; +} + /* static */Result SerialContainerUtil::read(RiffContainer* container, const ReadOptions& options, SerialContainerData& out) { out.clear(); @@ -396,6 +468,10 @@ namespace Slang { // Set the sourceLocReader before doing de-serialize, such can lookup the remapped sourceLocs reader.getExtraObjects().set(sourceLocReader); + // Go through all of the AST nodes + // 1) Set the ASTBuilder on Type nodes + + // TODO(JS): // If modules can have more complicated relationships (like a two modules can refer to symbols // from each other), then we can make this work by @@ -412,6 +488,34 @@ namespace Slang { // Get the root node. It's at index 1 (0 is the null value). astRootNode = reader.getPointer(SerialIndex(1)).dynamicCast(); + + // 2) Add the extensions to the module mapTypeToCandidateExtensions cache + + { + ModuleDecl* moduleDecl = as(astRootNode); + + for (auto& obj : reader.getObjects()) + { + if (Type* type = obj.dynamicCast()) + { + type->_setASTBuilder(astBuilder); + } + + if (ExtensionDecl* extensionDecl = obj.dynamicCast()) + { + if (auto targetDeclRefType = as(extensionDecl->targetType)) + { + // Attach our extension to that type as a candidate... + if (auto aggTypeDeclRef = targetDeclRefType->declRef.as()) + { + auto aggTypeDecl = aggTypeDeclRef.getDecl(); + + _getCandidateExtensionList(aggTypeDecl, moduleDecl->mapTypeToCandidateExtensions).add(extensionDecl); + } + } + } + } + } } // Onto next chunk @@ -489,7 +593,7 @@ namespace Slang { RefPtr sourceLocWriter; - if (options.optionFlags & SerialOptionFlag::DebugInfo) + if (options.optionFlags & SerialOptionFlag::SourceLocation) { sourceLocWriter = new SerialSourceLocWriter(options.sourceManager); } @@ -530,7 +634,7 @@ namespace Slang { RefPtr sourceLocReader; // If we have debug info then find and read it - if (options.optionFlags & SerialOptionFlag::DebugInfo) + if (options.optionFlags & SerialOptionFlag::SourceLocation) { RiffContainer::ListChunk* debugList = rootList->findContainedList(SerialSourceLocData::kDebugFourCc); if (!debugList) @@ -593,7 +697,7 @@ namespace Slang { } } } - else if (options.optionFlags & SerialOptionFlag::DebugInfo) + else if (options.optionFlags & SerialOptionFlag::SourceLocation) { // They should be on the same line nos for (Index i = 1; i < readInsts.getCount(); ++i) diff --git a/source/slang/slang-serialize-container.h b/source/slang/slang-serialize-container.h index bc34982e7..5368f62e6 100644 --- a/source/slang/slang-serialize-container.h +++ b/source/slang/slang-serialize-container.h @@ -93,8 +93,14 @@ struct SerialContainerUtil DiagnosticSink* sink = nullptr; }; + /// Add module to outData + static SlangResult addModuleToData(Module* module, const WriteOptions& options, SerialContainerData& outData); + /// Get the serializable contents of the request as data - static SlangResult requestToData(EndToEndCompileRequest* request, const WriteOptions& options, SerialContainerData& outData); + static SlangResult addEndToEndRequestToData(EndToEndCompileRequest* request, const WriteOptions& options, SerialContainerData& outData); + + /// Convert front end request into something serializable + static SlangResult addFrontEndRequestToData(FrontEndCompileRequest* request, const WriteOptions& options, SerialContainerData& outData); /// Write the data into the container static SlangResult write(const SerialContainerData& data, const WriteOptions& options, RiffContainer* container); @@ -104,6 +110,12 @@ struct SerialContainerUtil /// Verify IR serialization static SlangResult verifyIRSerialize(IRModule* module, Session* session, const WriteOptions& options); + + /// Write the request to the stream + static SlangResult write(FrontEndCompileRequest* frontEndReq, const WriteOptions& options, Stream* stream); + static SlangResult write(EndToEndCompileRequest* request, const WriteOptions& options, Stream* stream); + static SlangResult write(Module* module, const WriteOptions& options, Stream* stream); + }; } // namespace Slang diff --git a/source/slang/slang-serialize-factory.cpp b/source/slang/slang-serialize-factory.cpp index f93fbba69..3ff536137 100644 --- a/source/slang/slang-serialize-factory.cpp +++ b/source/slang/slang-serialize-factory.cpp @@ -111,6 +111,9 @@ SerialIndex ModuleSerialFilter::writePointer(SerialWriter* writer, const NodeBas ASTSerialUtil::addSerialClasses(serialClasses); SerialRefObjects::addSerialClasses(serialClasses); + // Check if it seems ok + SLANG_ASSERT(serialClasses->isOk()); + return SLANG_OK; } diff --git a/source/slang/slang-serialize-ir.cpp b/source/slang/slang-serialize-ir.cpp index d923577f4..50e4467e3 100644 --- a/source/slang/slang-serialize-ir.cpp +++ b/source/slang/slang-serialize-ir.cpp @@ -307,7 +307,7 @@ Result IRSerialWriter::write(IRModule* module, SerialSourceLocWriter* sourceLocW } } - if ((options & SerialOptionFlag::DebugInfo) && sourceLocWriter) + if ((options & SerialOptionFlag::SourceLocation) && sourceLocWriter) { _calcDebugInfo(sourceLocWriter); } diff --git a/source/slang/slang-serialize-type-info.h b/source/slang/slang-serialize-type-info.h index 89097fc32..7ed45bb0b 100644 --- a/source/slang/slang-serialize-type-info.h +++ b/source/slang/slang-serialize-type-info.h @@ -150,11 +150,7 @@ struct SerialTypeInfo { typedef T* NativeType; typedef SerialIndex SerialType; - - enum - { - SerialAlignment = SLANG_ALIGN_OF(SerialType) - }; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; static void toSerial(SerialWriter* writer, const void* inNative, void* outSerial) { @@ -166,6 +162,25 @@ struct SerialTypeInfo } }; +// RefPtr (pretty much the same as T* - except for native rep) +template +struct SerialTypeInfo> +{ + typedef RefPtr NativeType; + typedef SerialIndex SerialType; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(SerialWriter* writer, const void* native, void* serial) + { + auto& src = *(const NativeType*)native; + *(SerialType*)serial = writer->addPointer(src); + } + static void toNative(SerialReader* reader, const void* serial, void* native) + { + *(NativeType*)native = reader->getPointer(*(const SerialType*)serial).dynamicCast(); + } +}; + // Special case Name template <> struct SerialTypeInfo : public SerialTypeInfo @@ -208,6 +223,27 @@ struct SerialTypeInfo> } }; +// String +template <> +struct SerialTypeInfo +{ + typedef String NativeType; + typedef SerialIndex SerialType; + enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + + static void toSerial(SerialWriter* writer, const void* native, void* serial) + { + auto& src = *(const NativeType*)native; + *(SerialType*)serial = writer->addString(src); + } + static void toNative(SerialReader* reader, const void* serial, void* native) + { + auto& src = *(const SerialType*)serial; + auto& dst = *(NativeType*)native; + dst = reader->getString(src); + } +}; + // Dictionary template struct SerialTypeInfo> @@ -244,8 +280,9 @@ struct SerialTypeInfo> i++; } - dst.keys = writer->addArray(keys.getBuffer(), count); - dst.values = writer->addArray(values.getBuffer(), count); + // When we add the array it is already converted to a serializable type, so add as SerialArray + dst.keys = writer->addSerialArray(keys.getBuffer(), count); + dst.values = writer->addSerialArray(values.getBuffer(), count); } static void toNative(SerialReader* reader, const void* serial, void* native) { @@ -271,49 +308,42 @@ struct SerialTypeInfo> } }; -// Handle RefPtr - just convert into * to do the conversion -template -struct SerialTypeInfo> +// KeyValuePair +template +struct SerialTypeInfo> { - typedef RefPtr NativeType; - typedef typename SerialTypeInfo::SerialType SerialType; - enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; + typedef KeyValuePair NativeType; - static void toSerial(SerialWriter* writer, const void* native, void* serial) - { - auto& src = *(const NativeType*)native; - T* obj = src; - SerialTypeInfo::toSerial(writer, &obj, serial); - } - static void toNative(SerialReader* reader, const void* serial, void* native) + typedef typename SerialTypeInfo::SerialType KeySerialType; + typedef typename SerialTypeInfo::SerialType ValueSerialType; + + struct SerialType { - T* obj = nullptr; - SerialTypeInfo::toNative(reader, serial, &obj); - *(NativeType*)native = obj; - } -}; + KeySerialType key; + ValueSerialType value; + }; -// String -template <> -struct SerialTypeInfo -{ - typedef String NativeType; - typedef SerialIndex SerialType; enum { SerialAlignment = SLANG_ALIGN_OF(SerialType) }; static void toSerial(SerialWriter* writer, const void* native, void* serial) { auto& src = *(const NativeType*)native; - *(SerialType*)serial = writer->addString(src); + auto& dst = *(SerialType*)serial; + + SerialTypeInfo::toSerial(writer, &src.Key, &dst.key); + SerialTypeInfo::toSerial(writer, &src.Value, &dst.value); } static void toNative(SerialReader* reader, const void* serial, void* native) { auto& src = *(const SerialType*)serial; auto& dst = *(NativeType*)native; - dst = reader->getString(src); + + SerialTypeInfo::toNative(reader, &src.key, &dst.Key); + SerialTypeInfo::toNative(reader, &src.value, &dst.Value); } }; + } // namespace Slang #endif diff --git a/source/slang/slang-serialize-types.h b/source/slang/slang-serialize-types.h index 9bb84e290..fb0dd2f9d 100644 --- a/source/slang/slang-serialize-types.h +++ b/source/slang/slang-serialize-types.h @@ -27,7 +27,7 @@ struct SerialOptionFlag enum Enum : Type { RawSourceLocation = 0x01, ///< If set will store directly SourceLoc - only useful if current source locs will be identical when read in (typically this is *NOT* the case) - DebugInfo = 0x02, ///< If set will output debug information, that can be reconstructed when read after being stored. + SourceLocation = 0x02, ///< If set will output SourceLoc information, that can be reconstructed when read after being stored. ASTModule = 0x04, ///< If set will output AST modules - typically required, but potentially not desired (for example with obsfucation) IRModule = 0x08, ///< If set will output IR modules - typically required }; diff --git a/source/slang/slang-serialize.cpp b/source/slang/slang-serialize.cpp index 34680d860..4f8fdc546 100644 --- a/source/slang/slang-serialize.cpp +++ b/source/slang/slang-serialize.cpp @@ -151,6 +151,58 @@ SerialClass* SerialClasses::_createSerialClass(const SerialClass* cls) return dst; } +bool SerialClasses::isOk() const +{ + StringSlicePool pool(StringSlicePool::Style::Default); + + for (const auto& classes : m_classesByTypeKind) + { + for (const SerialClass* cls : classes) + { + // It is possible potentially to have gaps + if (cls == nullptr) + { + continue; + } + + if (cls->super && cls->super->typeKind != cls->typeKind) + { + // If has a super type, must be the same typeKind + return false; + } + + // Make sure the fields are uniquely named + + pool.clear(); + + { + const SerialClass* curCls = cls; + + do + { + for (Index i = 0; i < curCls->fieldsCount; ++i) + { + const SerialField& field = curCls->fields[i]; + + StringSlicePool::Handle handle; + if (pool.findOrAdd(UnownedStringSlice(field.name), handle)) + { + return false; + } + } + + // Add the fields of the parent + curCls = curCls->super; + } + while (curCls); + } + } + } + + return true; +} + + SerialClasses::SerialClasses(): m_arena(2048) { @@ -375,7 +427,7 @@ SerialIndex SerialWriter::addName(const Name* name) return index; } -SerialIndex SerialWriter::_addArray(size_t elementSize, size_t alignment, const void* elements, Index elementCount) +SerialIndex SerialWriter::addSerialArray(size_t elementSize, size_t alignment, const void* elements, Index elementCount) { typedef SerialInfo::ArrayEntry Entry; @@ -699,12 +751,19 @@ String SerialReader::getString(SerialIndex index) // Okay we need to construct as a string UnownedStringSlice slice = getStringSlice(index); - String string(slice); - StringRepresentation* stringRep = string.getStringRepresentation(); - m_scope.add(stringRep); + StringRepresentation* stringRep = nullptr; + + const Index length = slice.getLength(); + if (length) + { + stringRep = StringRepresentation::createWithCapacityAndLength(length, length); + memcpy(stringRep->getData(), slice.begin(), length * sizeof(char)); + addScope(stringRep); + } + m_objects[Index(index)] = stringRep; - return string; + return String(stringRep); } Name* SerialReader::getName(SerialIndex index) diff --git a/source/slang/slang-serialize.h b/source/slang/slang-serialize.h index a48a7e216..ff402b35c 100644 --- a/source/slang/slang-serialize.h +++ b/source/slang/slang-serialize.h @@ -246,7 +246,9 @@ public: const List& getObjects() const { return m_objects; } /// Add an object to be kept in scope - void addScope(const RefObject* obj) { m_scope.add(obj); } + void addScopeWithoutAddRef(const RefObject* obj) { m_scope.add(obj); } + /// Add obj with a reference + void addScope(const RefObject* obj) { const_cast(obj)->addReference(); m_scope.add(obj); } /// Used for attaching extra objects necessary for serializing SerialExtraObjects& getExtraObjects() { return m_extraObjects; } @@ -324,9 +326,21 @@ public: SerialIndex writeObject(const NodeBase* ptr); SerialIndex writeObject(const RefObject* ptr); + /// Add an array - may need to convert to serialized format template SerialIndex addArray(const T* in, Index count); + template + /// Add an array where all the elements are already in serialized format (ie there is no need to do a conversion) + SerialIndex addSerialArray(const void* elements, Index elementCount) + { + typedef SerialTypeInfo TypeInfo; + return addSerialArray(sizeof(typename TypeInfo::SerialType), SerialTypeInfo::SerialAlignment, elements, elementCount); + } + + /// Add an array where all the elements are already in serialized format (ie there is no need to do a conversion) + SerialIndex addSerialArray(size_t elementSize, size_t alignment, const void* elements, Index elementCount); + /// Add the string SerialIndex addString(const UnownedStringSlice& slice) { return _addStringSlice(SerialTypeKind::String, m_sliceMap, slice); } SerialIndex addString(const String& in); @@ -362,8 +376,6 @@ protected: SerialIndex _addStringSlice(SerialTypeKind typeKind, SliceMap& sliceMap, const UnownedStringSlice& slice); - SerialIndex _addArray(size_t elementSize, size_t alignment, const void* elements, Index elementCount); - SerialIndex _add(const void* nativePtr, SerialInfo::Entry* entry) { m_entries.add(entry); @@ -399,7 +411,7 @@ SerialIndex SerialWriter::addArray(const T* in, Index count) if (std::is_same::value) { // If they are the same we can just write out - return _addArray(sizeof(T), SLANG_ALIGN_OF(ElementSerialType), in, count); + return addSerialArray(sizeof(T), SLANG_ALIGN_OF(ElementSerialType), in, count); } else { @@ -411,7 +423,7 @@ SerialIndex SerialWriter::addArray(const T* in, Index count) { ElementTypeInfo::toSerial(this, &in[i], &work[i]); } - return _addArray(sizeof(ElementSerialType), SLANG_ALIGN_OF(ElementSerialType), work.getBuffer(), count); + return addSerialArray(sizeof(ElementSerialType), SLANG_ALIGN_OF(ElementSerialType), work.getBuffer(), count); } } @@ -478,7 +490,6 @@ struct SerialClass class SerialClasses : public RefObject { public: - /// Will add it's own copy into m_classesByType /// In process will calculate alignment, offset etc for fields /// NOTE! the super set, *must* be an already added to this SerialClasses @@ -492,6 +503,9 @@ public: /// Returns true if this cls is *owned* by this SerialClasses bool isOwned(const SerialClass* cls) const; + /// Returns true if the SerialClasses structure appears ok + bool isOk() const; + /// Get a serial class based on its type/subType const SerialClass* getSerialClass(SerialTypeKind typeKind, SerialSubType subType) const { diff --git a/source/slang/slang-syntax.cpp b/source/slang/slang-syntax.cpp index ea17ad1d1..82e94fb6a 100644 --- a/source/slang/slang-syntax.cpp +++ b/source/slang/slang-syntax.cpp @@ -453,14 +453,14 @@ Index getFilterCountImpl(const ReflectClassInfo& clsInfo, MemberFilterStyle filt } } - if (magicMod->name == "SamplerState") + if (magicMod->magicName == "SamplerState") { auto type = astBuilder->create(); type->declRef = declRef; type->flavor = SamplerStateFlavor(magicMod->tag); return type; } - else if (magicMod->name == "Vector") + else if (magicMod->magicName == "Vector") { SLANG_ASSERT(subst && subst->args.getCount() == 2); auto vecType = astBuilder->create(); @@ -469,14 +469,14 @@ Index getFilterCountImpl(const ReflectClassInfo& clsInfo, MemberFilterStyle filt vecType->elementCount = ExtractGenericArgInteger(subst->args[1]); return vecType; } - else if (magicMod->name == "Matrix") + else if (magicMod->magicName == "Matrix") { SLANG_ASSERT(subst && subst->args.getCount() == 3); auto matType = astBuilder->create(); matType->declRef = declRef; return matType; } - else if (magicMod->name == "Texture") + else if (magicMod->magicName == "Texture") { SLANG_ASSERT(subst && subst->args.getCount() >= 1); auto textureType = astBuilder->create( @@ -485,7 +485,7 @@ Index getFilterCountImpl(const ReflectClassInfo& clsInfo, MemberFilterStyle filt textureType->declRef = declRef; return textureType; } - else if (magicMod->name == "TextureSampler") + else if (magicMod->magicName == "TextureSampler") { SLANG_ASSERT(subst && subst->args.getCount() >= 1); auto textureType = astBuilder->create( @@ -494,7 +494,7 @@ Index getFilterCountImpl(const ReflectClassInfo& clsInfo, MemberFilterStyle filt textureType->declRef = declRef; return textureType; } - else if (magicMod->name == "GLSLImageType") + else if (magicMod->magicName == "GLSLImageType") { SLANG_ASSERT(subst && subst->args.getCount() >= 1); auto textureType = astBuilder->create( @@ -503,7 +503,7 @@ Index getFilterCountImpl(const ReflectClassInfo& clsInfo, MemberFilterStyle filt textureType->declRef = declRef; return textureType; } - else if (magicMod->name == "FeedbackType") + else if (magicMod->magicName == "FeedbackType") { SLANG_ASSERT(subst == nullptr); auto type = astBuilder->create(); @@ -517,7 +517,7 @@ Index getFilterCountImpl(const ReflectClassInfo& clsInfo, MemberFilterStyle filt // of this ridiculously slow `if` cascade. #define CASE(n,T) \ - else if(magicMod->name == #n) { \ + else if(magicMod->magicName == #n) { \ auto type = astBuilder->create(); \ type->declRef = declRef; \ return type; \ @@ -529,7 +529,7 @@ Index getFilterCountImpl(const ReflectClassInfo& clsInfo, MemberFilterStyle filt #undef CASE #define CASE(n,T) \ - else if(magicMod->name == #n) { \ + else if(magicMod->magicName == #n) { \ SLANG_ASSERT(subst && subst->args.getCount() == 1); \ auto type = astBuilder->create(); \ type->elementType = ExtractGenericArgType(subst->args[0]); \ @@ -558,7 +558,7 @@ Index getFilterCountImpl(const ReflectClassInfo& clsInfo, MemberFilterStyle filt // "magic" builtin types which have no generic parameters #define CASE(n,T) \ - else if(magicMod->name == #n) { \ + else if(magicMod->magicName == #n) { \ auto type = astBuilder->create(); \ type->declRef = declRef; \ return type; \ @@ -575,7 +575,7 @@ Index getFilterCountImpl(const ReflectClassInfo& clsInfo, MemberFilterStyle filt else { - auto classInfo = astBuilder->findSyntaxClass(magicMod->name.getUnownedSlice()); + auto classInfo = astBuilder->findSyntaxClass(magicMod->magicName.getUnownedSlice()); if (!classInfo.classInfo) { SLANG_UNEXPECTED("unhandled type"); diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp index 90ddf030c..a66441948 100644 --- a/source/slang/slang.cpp +++ b/source/slang/slang.cpp @@ -136,7 +136,6 @@ void Session::init() // And the global ASTBuilder globalAstBuilder = new ASTBuilder(m_sharedASTBuilder, "globalAstBuilder"); - // Make sure our source manager is initialized builtinSourceManager.initialize(nullptr, nullptr); @@ -175,8 +174,41 @@ void Session::init() slangLanguageScope = new Scope(); slangLanguageScope->nextSibling = hlslLanguageScope; - addBuiltinSource(coreLanguageScope, "core", getCoreLibraryCode()); - addBuiltinSource(hlslLanguageScope, "hlsl", getHLSLLibraryCode()); + if (false) + { + // Let's try loading serialized modules and adding them + _readBuiltinModule(coreLanguageScope, "core"); + _readBuiltinModule(hlslLanguageScope, "hlsl"); + } + else + { + addBuiltinSource(coreLanguageScope, "core", getCoreLibraryCode()); + addBuiltinSource(hlslLanguageScope, "hlsl", getHLSLLibraryCode()); + + // Write out + if (false) + { + for (auto& pair : m_builtinLinkage->mapNameToLoadedModules) + { + const Name* moduleName = pair.Key; + Module* module = pair.Value; + + // Set up options + SerialContainerUtil::WriteOptions options; + //options.optionFlags |= SerialOptionFlag::SourceLocation; + options.sourceManager = m_builtinLinkage->getSourceManager(); + + StringBuilder builder; + builder << moduleName->text << ".slang-module"; + + FileStream stream(builder.ProduceString(), FileMode::Create, FileAccess::Write, FileShare::ReadWrite); + if (SLANG_FAILED(SerialContainerUtil::write(module, options, &stream))) + { + SLANG_UNEXPECTED("Unable to load stdlib"); + } + } + } + } { for (Index i = 0; i < Index(SourceLanguage::CountOf); ++i) @@ -194,6 +226,88 @@ void Session::init() m_languagePreludes[Index(SourceLanguage::HLSL)] = get_slang_hlsl_prelude(); } +SlangResult Session::_readBuiltinModule(Scope* scope, String moduleName) +{ + StringBuilder moduleFilename; + moduleFilename << moduleName << ".slang-module"; + + RiffContainer riffContainer; + try + { + FileStream stream(moduleFilename.ProduceString(), FileMode::Open, FileAccess::Read, FileShare::ReadOnly); + // Load the riff container + SLANG_RETURN_ON_FAIL(RiffUtil::read(&stream, riffContainer)); + } + catch (const IOException&) + { + return SLANG_FAIL; + } + + // Load up the module + + SerialContainerData containerData; + + Linkage* linkage = getBuiltinLinkage(); + + NamePool* sessionNamePool = &namePool; + NamePool* linkageNamePool = linkage->getNamePool(); + + SerialContainerUtil::ReadOptions options; + options.namePool = linkageNamePool; + options.session = this; + options.sharedASTBuilder = linkage->getASTBuilder()->getSharedASTBuilder(); + options.sourceManager = linkage->getSourceManager(); + options.linkage = linkage; + + // Hmm - don't have a suitable sink yet, so attempt to just not have one + options.sink = nullptr; + + SLANG_RETURN_ON_FAIL(SerialContainerUtil::read(&riffContainer, options, containerData)); + + for (auto& srcModule : containerData.modules) + { + RefPtr module(new Module(linkage, srcModule.astBuilder)); + + ModuleDecl* moduleDecl = as(srcModule.astRootNode); + + if (moduleDecl) + { + if (isFromStdLib(moduleDecl)) + { + registerBuiltinDecls(this, moduleDecl); + } + + module->setModuleDecl(moduleDecl); + } + + module->setIRModule(srcModule.irModule); + + // Put in the loaded module map + linkage->mapNameToLoadedModules.Add(sessionNamePool->getName(moduleName), module); + + // Add the resulting code to the appropriate scope + if (!scope->containerDecl) + { + // We are the first chunk of code to be loaded for this scope + scope->containerDecl = moduleDecl; + } + else + { + // We need to create a new scope to link into the whole thing + auto subScope = new Scope(); + subScope->containerDecl = moduleDecl; + subScope->nextSibling = scope->nextSibling; + scope->nextSibling = subScope; + } + + // We need to retain this AST so that we can use it in other code + // (Note that the `Scope` type does not retain the AST it points to) + stdlibModules.add(module); + } + + return SLANG_OK; +} + ISlangUnknown* Session::getInterface(const Guid& guid) { if(guid == IID_ISlangUnknown || guid == IID_IGlobalSession) @@ -1240,7 +1354,7 @@ void FrontEndCompileRequest::generateIR() options.compressionType = SerialCompressionType::None; options.sourceManager = getSourceManager(); - options.optionFlags |= SerialOptionFlag::DebugInfo; + options.optionFlags |= SerialOptionFlag::SourceLocation; // Verify debug information if (SLANG_FAILED(SerialContainerUtil::verifyIRSerialize(irModule, getSession(), options))) @@ -1950,11 +2064,19 @@ void FilePathDependencyList::addDependency(Module* module) // Module // -Module::Module(Linkage* linkage) +Module::Module(Linkage* linkage, ASTBuilder* astBuilder) : ComponentType(linkage) - , m_astBuilder(linkage->getASTBuilder()->getSharedASTBuilder(), "Module") , m_mangledExportPool(StringSlicePool::Style::Empty) { + if (astBuilder) + { + m_astBuilder = astBuilder; + } + else + { + m_astBuilder = new ASTBuilder(linkage->getASTBuilder()->getSharedASTBuilder(), "Module"); + } + addModuleDependency(this); } @@ -2053,7 +2175,7 @@ void Module::_processFindDeclsExportSymbolsRec(Decl* decl) return; } - // process `decl` itself + // If it's a container process it's children if(auto containerDecl = as(decl)) { for (auto child : containerDecl->members) @@ -2061,7 +2183,9 @@ void Module::_processFindDeclsExportSymbolsRec(Decl* decl) _processFindDeclsExportSymbolsRec(child); } } - else if (auto genericDecl = as(decl)) + + // GenericDecl is also a container, so do subsequent test + if (auto genericDecl = as(decl)) { _processFindDeclsExportSymbolsRec(genericDecl->inner); } -- cgit v1.2.3