From d4aeb18cb348104fa6b036393488df780902a353 Mon Sep 17 00:00:00 2001 From: ArielG-NV <159081215+ArielG-NV@users.noreply.github.com> Date: Thu, 5 Sep 2024 10:41:35 -0400 Subject: Document All Capability Atoms and Profiles (#5008) * Document All Capability Atoms and Profiles Fixes: #4125 Unimplemented Considerations: 1. This PR does not add support to query all capability-atom's from a command-line option. It is understood that this might be desired, due to this, the logic to generate `docs\user-guide\a3-02-reference-capability-atoms.md` was made to be "command-line friendly" so minimal changes are needed to pipe our documentation into a command-line option if this change is to be added. Changes: 1. Added a way to document atoms inside `.capdef`. Method to document is described under `source\slang\slang-capabilities.capdef`. The goal is to error if a public atom does not have any form of documentation to ensure we always have up-to-date documentation to guide user on what an atom is/does. * The following `.capdef` file syntax was added * /// [HEADER_GROUP] * /// regular comment 2. When capability generator runs it auto-generates `docs\user-guide\a3-02-reference-capability-atoms.md` 3. Added to the user-guide 3 sections: `Reference`, `Reference -> Capability Profiles`, `Reference -> Capability atoms` section --- .../capability-generator-main.cpp | 330 +++++++++++++++++++-- .../slang-capability-diagnostic-defs.h | 3 + 2 files changed, 309 insertions(+), 24 deletions(-) (limited to 'tools') diff --git a/tools/slang-capability-generator/capability-generator-main.cpp b/tools/slang-capability-generator/capability-generator-main.cpp index 711532273..674aca7fd 100644 --- a/tools/slang-capability-generator/capability-generator-main.cpp +++ b/tools/slang-capability-generator/capability-generator-main.cpp @@ -65,6 +65,95 @@ static void _removeFromOtherAtomsNotInThis(HashSet thisSet otherSet.remove(atomToRemove); } +enum class AutoDocHeaderGroup : UInt +{ + Targets = 0, + Stages, + Versions, + Extensions, + Compound, + Other, + Count, + Invalid, +}; + +UnownedStringSlice getHeaderNameFromAutoDocHeaderGroup(UInt headerGroup) +{ + switch (headerGroup) + { + case (UInt)AutoDocHeaderGroup::Targets: + return UnownedStringSlice("Targets"); + case (UInt)AutoDocHeaderGroup::Stages: + return UnownedStringSlice("Stages"); + case (UInt)AutoDocHeaderGroup::Extensions: + return UnownedStringSlice("Extensions"); + case (UInt)AutoDocHeaderGroup::Versions: + return UnownedStringSlice("Versions"); + case (UInt)AutoDocHeaderGroup::Compound: + return UnownedStringSlice("Compound Capabilities"); + case (UInt)AutoDocHeaderGroup::Other: + return UnownedStringSlice("Other"); + default: + SLANG_ASSERT("Unknown `AutoDocHeaderGroup`"); + return UnownedStringSlice(""); + } +} + +UnownedStringSlice getHeaderDescriptionFromAutoDocHeaderGroup(UInt headerGroup) +{ + switch (headerGroup) + { + case (UInt)AutoDocHeaderGroup::Targets: + return UnownedStringSlice("Capabilities to specify code generation targets (`glsl`, `spirv`...)"); + case (UInt)AutoDocHeaderGroup::Stages: + return UnownedStringSlice("Capabilities to specify code generation stages (`vertex`, `fragment`...)"); + case (UInt)AutoDocHeaderGroup::Extensions: + return UnownedStringSlice("Capabilities to specify extensions (`GL_EXT`, `SPV_EXT`...)"); + case (UInt)AutoDocHeaderGroup::Versions: + return UnownedStringSlice("Capabilities to specify versions of a code generation target (`sm_5_0`, `GLSL_400`...)"); + case (UInt)AutoDocHeaderGroup::Compound: + return UnownedStringSlice("Capabilities to specify capabilities created by other capabilities (`raytracing`, `meshshading`...)"); + case (UInt)AutoDocHeaderGroup::Other: + return UnownedStringSlice("Capabilities which may be deprecated"); + default: + SLANG_ASSERT("Unknown `AutoDocHeaderGroup`"); + return UnownedStringSlice(""); + } +} + +AutoDocHeaderGroup getAutoDocHeaderGroupFromTag(DiagnosticSink* sink, UnownedStringSlice headerGroupName, SourceLoc loc) +{ + if (headerGroupName.caseInsensitiveEquals(UnownedStringSlice("Other"))) + return AutoDocHeaderGroup::Other; + else if (headerGroupName.caseInsensitiveEquals(UnownedStringSlice("Target"))) + return AutoDocHeaderGroup::Targets; + else if (headerGroupName.caseInsensitiveEquals(UnownedStringSlice("Stage"))) + return AutoDocHeaderGroup::Stages; + else if (headerGroupName.caseInsensitiveEquals(UnownedStringSlice("EXT"))) + return AutoDocHeaderGroup::Extensions; + else if (headerGroupName.caseInsensitiveEquals(UnownedStringSlice("Version"))) + return AutoDocHeaderGroup::Versions; + else if (headerGroupName.caseInsensitiveEquals(UnownedStringSlice("Compound"))) + return AutoDocHeaderGroup::Compound; + else + { + sink->diagnose(loc, Diagnostics::invalidDocCommentHeader, headerGroupName); + return AutoDocHeaderGroup::Invalid; + } +} + +struct AutoDocInfo +{ + String comment; + AutoDocHeaderGroup headerGroup; + + AutoDocInfo() + { + comment = {}; + headerGroup = AutoDocHeaderGroup::Other; + } +}; + struct CapabilityDef : public RefObject { public: @@ -80,6 +169,7 @@ public: this->sourceLoc = other.sourceLoc; this->keyAtomsPresent = other.keyAtomsPresent; this->sharedContext = other.sharedContext; + this->docComment = other.docComment; } String name; @@ -91,6 +181,7 @@ public: List> canonicalRepresentation; SerializedArrayView serializedCanonicalRepresentation; SourceLoc sourceLoc; + AutoDocInfo docComment; /// Stores key atoms a CapabilityDef refers to. /// Shared key atoms: key atoms shared between every individual set in a canonicalRepresentation, added together. HashSet keyAtomsPresent; @@ -174,6 +265,24 @@ public: } }; +/// Advances through BlockComment/LineComment, otherwise, "advanceIf 'type' is the next token" +enum class AdvanceOptions : UInt +{ + None = 0 << 0, + SkipComments = 1 << 0, +}; + +template +constexpr bool ContainsOption() +{ + return (UInt)L & (UInt)R; +} + +static bool isInternalDef(RefPtr def) +{ + return def->name.startsWith("_"); +} + struct CapabilityDefParser { CapabilityDefParser( @@ -195,9 +304,19 @@ struct CapabilityDefParser TokenReader m_tokenReader; + template bool advanceIf(TokenType type) { - if (m_tokenReader.peekTokenType() == type) + auto peekToken = m_tokenReader.peekTokenType(); + if constexpr (ContainsOption()) + { + while (peekToken == TokenType::BlockComment || peekToken == TokenType::LineComment) + { + m_tokenReader.advanceToken(); + peekToken = m_tokenReader.peekTokenType(); + } + } + if (peekToken == type) { m_tokenReader.advanceToken(); return true; @@ -205,9 +324,15 @@ struct CapabilityDefParser return false; } + template SlangResult readToken(TokenType type, Token& nextToken) { nextToken = m_tokenReader.advanceToken(); + if constexpr (ContainsOption()) + { + while (nextToken.type == TokenType::BlockComment || nextToken.type == TokenType::LineComment) + nextToken = m_tokenReader.advanceToken(); + } if (nextToken.type != type) { m_sink->diagnose(nextToken.loc, Diagnostics::unexpectedTokenExpectedTokenType, nextToken, type); @@ -216,10 +341,11 @@ struct CapabilityDefParser return SLANG_OK; } + template SlangResult readToken(TokenType type) { Token nextToken; - return readToken(type, nextToken); + return readToken(type, nextToken); } SlangResult parseConjunction(CapabilityConjunctionExpr& expr) @@ -227,7 +353,7 @@ struct CapabilityDefParser for (;;) { Token nameToken; - SLANG_RETURN_ON_FAIL(readToken(TokenType::Identifier, nameToken)); + SLANG_RETURN_ON_FAIL(readToken(TokenType::Identifier, nameToken)); CapabilityDef* def = nullptr; if (m_mapNameToCapability.tryGetValue(nameToken.getContent(), def)) { @@ -238,7 +364,7 @@ struct CapabilityDefParser m_sink->diagnose(nameToken.loc, Diagnostics::undefinedIdentifier, nameToken); return SLANG_FAIL; } - if (!(advanceIf(TokenType::OpAdd))) + if (!(advanceIf(TokenType::OpAdd))) break; } return SLANG_OK; @@ -252,7 +378,7 @@ struct CapabilityDefParser conjunction.sourceLoc = this->m_tokenReader.m_cursor->getLoc(); SLANG_RETURN_ON_FAIL(parseConjunction(conjunction)); expr.conjunctions.add(conjunction); - if (!advanceIf(TokenType::OpBitOr)) + if (!advanceIf(TokenType::OpBitOr)) break; } return SLANG_OK; @@ -277,7 +403,7 @@ struct CapabilityDefParser // Try to pack `_atom` and `atom` into the same per key List String name = i->name; if(i->name.startsWith("_")) - name = name.subString(1, name.getLength()-1); + name = name.subString(1, name.getLength() - 1); nameToInternalAndExternalAtom[name].add(i); } for(auto i : nameToInternalAndExternalAtom) @@ -291,16 +417,29 @@ struct CapabilityDefParser } } } + + bool isLineSuccessive(HumaneSourceLoc above, HumaneSourceLoc below) + { + return above.line + 1 == below.line; + } + SlangResult parseDefs() { - auto tokens = m_lexer->lexAllSemanticTokens(); + auto tokens = m_lexer->lexAllMarkupTokens(); m_tokenReader = TokenReader(tokens); + AutoDocInfo successiveComments = AutoDocInfo(); + HumaneSourceLoc successiveCommentLine = {}; + for (;;) { + auto nextToken = m_tokenReader.advanceToken(); + + if (!isLineSuccessive(successiveCommentLine, m_lexer->m_sourceView->getHumaneLoc(nextToken.getLoc()))) + successiveComments = AutoDocInfo(); + RefPtr def = new CapabilityDef(); def->sharedContext = &m_sharedContext; def->flavor = CapabilityFlavor::Normal; - auto nextToken = m_tokenReader.advanceToken(); if (nextToken.getContent() == "alias") { def->flavor = CapabilityFlavor::Alias; @@ -313,6 +452,51 @@ struct CapabilityDefParser { def->flavor = CapabilityFlavor::Normal; } + else if (nextToken.type == TokenType::BlockComment) + { + // Do not auto-document + continue; + } + else if (nextToken.type == TokenType::LineComment) + { + // Auto-document if the preceeding token to an identifier is '///' + // complete rules described in `source\slang\slang-capabilities.capdef` + auto commentContent = nextToken.getContent(); + + // remove "//" + commentContent = commentContent.subString(2, commentContent.getLength() - 2); + if (commentContent.startsWith("/")) + { + auto commentLine = m_lexer->m_sourceView->getHumaneLoc(nextToken.getLoc()); + + // Reset the `successiveCommentLine` to our newest commentLine + successiveCommentLine = commentLine; + + // remove "/" from "///" + commentContent = commentContent.subString(1, commentContent.getLength() - 1).trim(); + + // Check if we have a `[header]` + if (commentContent.startsWith("[")) + { + // Make a substring of `header]` + auto consumedLeftBracketOfHeader = commentContent.subString(1, commentContent.getLength() - 1); + // Find a `]` of `header]` if it exists + auto indexOfHeaderEnd = consumedLeftBracketOfHeader.indexOf(']'); + if (indexOfHeaderEnd != -1) + { + // We found our `header` + auto headerName = consumedLeftBracketOfHeader.subString(0, indexOfHeaderEnd); + successiveComments.headerGroup = getAutoDocHeaderGroupFromTag(m_sink, headerName, nextToken.getLoc()); + continue; + } + // If we did not find a header this is a regular comment + } + successiveComments.comment.append("> "); + successiveComments.comment.append(commentContent); + successiveComments.comment.append("\n"); + } + continue; + } else if (nextToken.type == TokenType::EndOfFile) { break; @@ -324,35 +508,41 @@ struct CapabilityDefParser } Token nameToken; - SLANG_RETURN_ON_FAIL(readToken(TokenType::Identifier, nameToken)); + SLANG_RETURN_ON_FAIL(readToken(TokenType::Identifier, nameToken)); def->name = nameToken.getContent(); if (def->flavor == CapabilityFlavor::Normal) { - if (advanceIf(TokenType::Colon)) + if (advanceIf(TokenType::Colon)) { SLANG_RETURN_ON_FAIL(parseExpr(def->expr)); } - if (advanceIf(TokenType::OpAssign)) + if (advanceIf(TokenType::OpAssign)) { Token rankToken; - SLANG_RETURN_ON_FAIL(readToken(TokenType::IntegerLiteral, rankToken)); + SLANG_RETURN_ON_FAIL(readToken(TokenType::IntegerLiteral, rankToken)); def->rank = stringToInt(rankToken.getContent()); } + def->docComment = successiveComments; + if(def->docComment.comment.getLength() == 0 && !isInternalDef(def)) + m_sink->diagnose(nextToken.loc, Diagnostics::requiresDocComment, def->name); } else if (def->flavor == CapabilityFlavor::Alias) { - SLANG_RETURN_ON_FAIL(readToken(TokenType::OpAssign)); + SLANG_RETURN_ON_FAIL(readToken(TokenType::OpAssign)); SLANG_RETURN_ON_FAIL(parseExpr(def->expr)); + def->docComment = successiveComments; + if (def->docComment.comment.getLength() == 0 && !isInternalDef(def)) + m_sink->diagnose(nextToken.loc, Diagnostics::requiresDocComment, def->name); } else if (def->flavor == CapabilityFlavor::Abstract) { - if (advanceIf(TokenType::Colon)) + if (advanceIf(TokenType::Colon)) { SLANG_RETURN_ON_FAIL(parseExpr(def->expr)); } } - SLANG_RETURN_ON_FAIL(readToken(TokenType::Semicolon)); + SLANG_RETURN_ON_FAIL(readToken(TokenType::Semicolon)); m_defs.add(def); if (!m_mapNameToCapability.addIfNotExists(def->name, m_defs.getLast())) { @@ -723,6 +913,84 @@ UIntSet atomSetToUIntSet(const List& atomSet) return set; } +void printDocForCapabilityDef(StringBuilder& sbDoc, RefPtr def, List& sbDocSections) +{ + if (isInternalDef(def) + || def->flavor == CapabilityFlavor::Abstract + || def->docComment.headerGroup == AutoDocHeaderGroup::Invalid) + return; + + auto& sbDocSection = sbDocSections[(UInt)def->docComment.headerGroup]; + sbDocSection << "\n" << "`" << def->name << "`\n"; + sbDocSection << def->docComment.comment; +} + +List setupDocCommentHeaderStringBuilders() +{ + List sbDocSections; + sbDocSections.setCount((UInt)AutoDocHeaderGroup::Count); + for (UInt i = 0; i < (UInt)AutoDocHeaderGroup::Count; i++) + { + sbDocSections[i] << "\n" << getHeaderNameFromAutoDocHeaderGroup(i) << "\n----------------------\n"; + sbDocSections[i] << "*" << getHeaderDescriptionFromAutoDocHeaderGroup(i) << "*\n"; + } + return sbDocSections; +} + +/// "[Link Name](fileName#Link-Name)" +void addHyperLink(StringBuilder& sbDoc, UnownedStringSlice suffix) +{ + String suffixReformatted = ""; + + for (auto i : suffix) + { + if (i == ' ') + { + suffixReformatted.appendChar('-'); + continue; + } + suffixReformatted.appendChar(i); + } + sbDoc << "[" << suffix << "](#" << suffixReformatted << ")"; +} + +void setupDocumentationHeader(StringBuilder& sbDoc, const String& outPath) +{ + sbDoc << R"( +--- +layout: user-guide +--- + +Capability Atoms +============================ + +### Sections: + +)"; + + // Hyper-Links + for (UInt i = 0; i < (UInt)AutoDocHeaderGroup::Count; i++) + { + auto headerName = getHeaderNameFromAutoDocHeaderGroup(i); + sbDoc << i + 1 << ". "; // "i. " + addHyperLink(sbDoc, headerName); + sbDoc << "\n"; + } +} + +SlangResult generateDocumentation(DiagnosticSink* sink, List>& defs, StringBuilder& sbDoc, const String& outPath) +{ + setupDocumentationHeader(sbDoc, outPath); + + List sbDocSections = setupDocCommentHeaderStringBuilders(); + for (auto def : defs) + { + printDocForCapabilityDef(sbDoc, def, sbDocSections); + } + for (auto stringBuilder : sbDocSections) + sbDoc << stringBuilder.toString(); + return 1; +} SlangResult generateDefinitions(DiagnosticSink* sink, List>& defs, StringBuilder& sbHeader, StringBuilder& sbCpp) { @@ -934,14 +1202,14 @@ SlangResult generateDefinitions(DiagnosticSink* sink, List sbCpp << "};\n"; - sbCpp - << "void freeCapabilityDefs()\n" - << "{\n" - << " for (auto& cap : kCapabilityArray) { cap = CapabilityAtomSet(); }\n" - << " kAnyTargetUIntSetBuffer = CapabilityAtomSet();\n" - << " kAnyStageUIntSetBuffer = CapabilityAtomSet();\n" - << "}\n"; - return SLANG_OK; +sbCpp +<< "void freeCapabilityDefs()\n" +<< "{\n" +<< " for (auto& cap : kCapabilityArray) { cap = CapabilityAtomSet(); }\n" +<< " kAnyTargetUIntSetBuffer = CapabilityAtomSet();\n" +<< " kAnyStageUIntSetBuffer = CapabilityAtomSet();\n" +<< "}\n"; +return SLANG_OK; } @@ -959,7 +1227,7 @@ SlangResult parseDefFile(DiagnosticSink* sink, String inputPath, ListgetMemoryArena()); - + CapabilityDefParser parser(&lexer, sink, capabilitySharedContext); SLANG_RETURN_ON_FAIL(parser.parseDefs()); @@ -1031,8 +1299,22 @@ int main(int argc, const char* const* argv) return 1; } + auto outDocPath = Path::combine(targetDir, "../../docs/user-guide/a3-02-reference-capability-atoms.md"); + if (!File::exists(outDocPath)) + { + sink.diagnose(SourceLoc(), Diagnostics::couldNotFindValidDocumentationOutputPath, outDocPath); + } + + StringBuilder sbDoc; + if (SLANG_FAILED(generateDocumentation(&sink, defs, sbDoc, outDocPath))) + { + printDiagnostics(&sink); + return 1; + } + writeIfChanged(outHeaderPath, sbHeader.produceString()); writeIfChanged(outCppPath, sbCpp.produceString()); + writeIfChanged(outDocPath, sbDoc.produceString()); List opnames; for (auto def : defs) diff --git a/tools/slang-capability-generator/slang-capability-diagnostic-defs.h b/tools/slang-capability-generator/slang-capability-diagnostic-defs.h index b6c4425a2..7f80bda6a 100644 --- a/tools/slang-capability-generator/slang-capability-diagnostic-defs.h +++ b/tools/slang-capability-generator/slang-capability-diagnostic-defs.h @@ -33,6 +33,7 @@ DIAGNOSTIC( 2, Error, cannotFindFile, "cannot find file '$0'.") DIAGNOSTIC( 4, Error, cannotWriteOutputFile, "cannot write output file '$0'.") DIAGNOSTIC( 5, Error, failedToLoadDynamicLibrary, "failed to load dynamic library '$0'") DIAGNOSTIC( 6, Error, tooManyOutputPathsSpecified, "$0 output paths specified, but only $1 entry points given") +DIAGNOSTIC( 7, Warning, couldNotFindValidDocumentationOutputPath, "could not find valid documentation output path at $0") // // 2xxxx - Parsing @@ -41,6 +42,8 @@ DIAGNOSTIC( 6, Error, tooManyOutputPathsSpecified, "$0 output paths specified DIAGNOSTIC(20003, Error, unexpectedToken, "unexpected $0") DIAGNOSTIC(20001, Error, unexpectedTokenExpectedTokenType, "unexpected $0, expected $1") DIAGNOSTIC(20001, Error, unexpectedTokenExpectedTokenName, "unexpected $0, expected '$1'") +DIAGNOSTIC(20004, Warning, requiresDocComment, "'$0' requires a documentation comment \"///\"") +DIAGNOSTIC(20004, Warning, invalidDocCommentHeader, "got documentation comment '[$0]', expected one of: [Target] [Stage] [EXT] [Version] [Compound] [Other]") DIAGNOSTIC(0, Error, tokenNameExpectedButEOF, "\"$0\" expected but end of file encountered.") DIAGNOSTIC(0, Error, tokenTypeExpectedButEOF, "$0 expected but end of file encountered.") -- cgit v1.2.3