From 860d17b6876822ef7023fdce70c725d3f8be37b1 Mon Sep 17 00:00:00 2001 From: jsmall-nvidia Date: Fri, 5 Mar 2021 14:34:46 -0500 Subject: Doc tooling improvements (#1734) * #include an absolute path didn't work - because paths were taken to always be relative. * Split out AST 'printing'. * Replace listener with List
* Section -> Part. * Kind -> Type Flags -> Kind for ASTPrinter::Part * Improve comments around ASTPrinter. * toString -> toText on Val derived types. toText appends to a StringBuilder. * Added toSlice free function. Added operator<< for Val derived types. Use << where appropriate in doing toText. * More work at mark down output. * Fill in sourceloc for enum case. Add more sophisticated location determination for EnumCase. Refactored documentation output into DocMarkdownWriter. * Improvements for sig output. * Split up slang-doc into extractor and writer. * WIP generic support for doc support. * Some refactoring to make DocExtractor have potential to be used without Decls. * Made doc extraction work without Decls. * Output generic parameters. * Add generic parameter extraction. * Added writing variables. * Add an interface test. * Fix toArray. * Support for extensions, and inheritance. * Disable the doc test. Co-authored-by: Tim Foley --- source/slang/slang-ast-support-types.h | 4 +- source/slang/slang-doc-extractor.cpp | 911 +++++++++++++++++++++++ source/slang/slang-doc-extractor.h | 210 ++++++ source/slang/slang-doc-mark-down.cpp | 477 ++++++++++++ source/slang/slang-doc-mark-down.h | 70 ++ source/slang/slang-doc.cpp | 1249 -------------------------------- source/slang/slang-doc.h | 73 -- source/slang/slang.cpp | 12 +- 8 files changed, 1676 insertions(+), 1330 deletions(-) create mode 100644 source/slang/slang-doc-extractor.cpp create mode 100644 source/slang/slang-doc-extractor.h create mode 100644 source/slang/slang-doc-mark-down.cpp create mode 100644 source/slang/slang-doc-mark-down.h delete mode 100644 source/slang/slang-doc.cpp delete mode 100644 source/slang/slang-doc.h (limited to 'source/slang') diff --git a/source/slang/slang-ast-support-types.h b/source/slang/slang-ast-support-types.h index 6e4c90cc4..6697e878a 100644 --- a/source/slang/slang-ast-support-types.h +++ b/source/slang/slang-ast-support-types.h @@ -840,9 +840,9 @@ namespace Slang /// Returns true if non empty (equivalent to getCount() != 0 but faster) bool isNonEmpty() const { return !isEmpty(); } - List> toArray() + List toList() { - List> result; + List result; for (auto element : (*this)) { result.add(element); diff --git a/source/slang/slang-doc-extractor.cpp b/source/slang/slang-doc-extractor.cpp new file mode 100644 index 000000000..f44257877 --- /dev/null +++ b/source/slang/slang-doc-extractor.cpp @@ -0,0 +1,911 @@ +// slang-doc.cpp +#include "slang-doc-extractor.h" + +#include "../core/slang-string-util.h" + +#include "slang-ast-builder.h" +#include "slang-ast-print.h" + +namespace Slang { + +/* TODO(JS): + +* If Decls hand SourceRange, then we could use the range to simplify getting the Post markup, as will be trivial to get to the 'end' +* Need to handle preceeding * in some markup styles +* If we want to be able to disable markup we need a mechanism to do this. Probably define source ranges. + +* Need a way to take the extracted markup and produce suitable markdown +** This will need to display the decoration appropriately +*/ + +/* static */UnownedStringSlice DocMarkupExtractor::removeStart(MarkupType type, const UnownedStringSlice& comment) +{ + switch (type) + { + case MarkupType::BlockBefore: + { + if (comment.startsWith(UnownedStringSlice::fromLiteral("/**")) || + comment.startsWith(UnownedStringSlice::fromLiteral("/*!"))) + { + /// /** */ or /*! */. + return comment.tail(3); + } + return comment; + } + case MarkupType::BlockAfter: + { + + if (comment.startsWith(UnownedStringSlice::fromLiteral("/**<")) || + comment.startsWith(UnownedStringSlice::fromLiteral("/*!<"))) + { + /// /*!< */ or /**< */ + return comment.tail(4); + } + return comment; + } + + case MarkupType::LineBangBefore: + { + return comment.startsWith(UnownedStringSlice::fromLiteral("//!")) ? comment.tail(3) : comment; + } + case MarkupType::LineSlashBefore: + { + return comment.startsWith(UnownedStringSlice::fromLiteral("///")) ? comment.tail(3) : comment; + } + + case MarkupType::LineBangAfter: + { + /// //!< Can be multiple lines + return comment.startsWith(UnownedStringSlice::fromLiteral("//!<")) ? comment.tail(4) : comment; + } + case MarkupType::LineSlashAfter: + { + return comment.startsWith(UnownedStringSlice::fromLiteral("///<")) ? comment.tail(4) : comment; + } + default: break; + } + return comment; +} + +static Index _findTokenIndex(SourceLoc loc, const Token* toks, Index numToks) +{ + // Use a binary search to find the token + Index lo = 0; + Index hi = numToks; + + while (lo + 1 < hi) + { + const Index mid = (hi + lo) >> 1; + const Token& midToken = toks[mid]; + + if (midToken.loc == loc) + { + return mid; + } + + if (midToken.loc.getRaw() <= loc.getRaw()) + { + lo = mid; + } + else + { + hi = mid; + } + } + + // Not found + return -1; +} + +/* static */DocMarkupExtractor::MarkupFlags DocMarkupExtractor::getFlags(MarkupType type) +{ + switch (type) + { + default: + case MarkupType::None: return 0; + case MarkupType::BlockBefore: return MarkupFlag::Before | MarkupFlag::IsBlock; + case MarkupType::BlockAfter: return MarkupFlag::After | MarkupFlag::IsBlock; + + case MarkupType::LineBangBefore: return MarkupFlag::Before | MarkupFlag::IsMultiToken; + case MarkupType::LineSlashBefore: return MarkupFlag::Before | MarkupFlag::IsMultiToken; + + case MarkupType::LineBangAfter: return MarkupFlag::After | MarkupFlag::IsMultiToken; + case MarkupType::LineSlashAfter: return MarkupFlag::After | MarkupFlag::IsMultiToken; + } +} + +/* static */DocMarkupExtractor::MarkupType DocMarkupExtractor::findMarkupType(const Token& tok) +{ + switch (tok.type) + { + case TokenType::BlockComment: + { + UnownedStringSlice slice = tok.getContent(); + if (slice.getLength() >= 3 && (slice[2] == '!' || slice[2] == '*')) + { + return (slice.getLength() >= 4 && slice[3] == '<') ? MarkupType::BlockAfter : MarkupType::BlockBefore; + } + break; + } + case TokenType::LineComment: + { + UnownedStringSlice slice = tok.getContent(); + if (slice.getLength() >= 3) + { + if (slice[2] == '!') + { + return (slice.getLength() >= 4 && slice[3] == '<') ? MarkupType::LineBangAfter : MarkupType::LineBangBefore; + } + else if (slice[2] == '/') + { + return (slice.getLength() >= 4 && slice[3] == '<') ? MarkupType::LineSlashAfter : MarkupType::LineSlashBefore; + } + } + break; + } + default: break; + } + return MarkupType::None; +} + +static Index _calcWhitespaceIndent(const UnownedStringSlice& line) +{ + // TODO(JS): For now we ignore tabs and just work out indentation based on spaces/assume ASCII + Index indent = 0; + const Index count = line.getLength(); + for (; indent < count && line[indent] == ' '; indent++); + return indent; +} + +static Index _calcIndent(const UnownedStringSlice& line) +{ + // TODO(JS): For now we just assume no tabs, and that every char is ASCII + return line.getLength(); +} + +static void _appendUnindenttedLine(const UnownedStringSlice& line, Index maxIndent, StringBuilder& out) +{ + Index indent = _calcWhitespaceIndent(line); + + // We want to remove indenting remove no more than maxIndent + if (maxIndent >= 0) + { + indent = (indent > maxIndent) ? maxIndent : indent; + } + + // Remove the indenting, and append to out + out.append(line.tail(indent)); +} + +SlangResult DocMarkupExtractor::_extractMarkup(const FindInfo& info, const FoundMarkup& foundMarkup, StringBuilder& out) +{ + SourceView* sourceView = info.sourceView; + SourceFile* sourceFile = sourceView->getSourceFile(); + + // Here we want to produce the text that is implied by the markup tokens. + // We want to removing surrounding markup, and to also keep appropriate indentation + + switch (foundMarkup.type) + { + case MarkupType::BlockBefore: + case MarkupType::BlockAfter: + { + // We should only have a single line + SLANG_ASSERT(foundMarkup.range.getCount() == 1); + + const auto& tok = info.tokenList->m_tokens[foundMarkup.range.start]; + uint32_t offset = sourceView->getRange().getOffset(tok.loc); + + const UnownedStringSlice startLine = sourceFile->getLineContainingOffset(offset); + + UnownedStringSlice content = tok.getContent(); + + // Split into lines + List lines; + + StringUtil::calcLines(content, lines); + + Index maxIndent = -1; + + StringBuilder unindentedLine; + + const Index linesCount = lines.getCount(); + for (Index i = 0; i < linesCount; ++i) + { + UnownedStringSlice line = lines[i]; + unindentedLine.Clear(); + + if (i == 0) + { + if (startLine.isMemoryContained(line.begin())) + { + // For now we'll ignore tabs, and that the indent amount is, the amount of *byte* + // NOTE! This is only appropriate for ASCII without tabs. + maxIndent = _calcIndent(UnownedStringSlice(startLine.begin(), line.begin())); + + // Let's strip the start stuff + line = removeStart(foundMarkup.type, line); + } + } + + if (i == linesCount - 1) + { + SLANG_ASSERT(line.tail(line.getLength() - 2) == UnownedStringSlice::fromLiteral("*/")); + // Remove the */ at the end of the line + line = line.head(line.getLength() - 2); + } + + if (i > 0) + { + _appendUnindenttedLine(line, maxIndent, unindentedLine); + } + else + { + unindentedLine.append(line); + } + + // If the first or last line are all white space, just ignore them + if ((i == linesCount - 1 || i == 0) && unindentedLine.getUnownedSlice().trim().getLength() == 0) + { + continue; + } + + out.append(unindentedLine); + out.appendChar('\n'); + } + + break; + } + case MarkupType::LineBangBefore: + case MarkupType::LineSlashBefore: + case MarkupType::LineBangAfter: + case MarkupType::LineSlashAfter: + { + // Holds the lines extracted, they may have some white space indenting (like the space at the start of //) + List lines; + + const auto& range = foundMarkup.range; + for (Index i = range.start; i < range.end; ++ i) + { + const auto& tok = info.tokenList->m_tokens[i]; + UnownedStringSlice line = tok.getContent(); + line = removeStart(foundMarkup.type, line); + + // If the first or last line are all white space, just ignore them + if ((i == range.start || i == range.end - 1) && line.trim().getLength() == 0) + { + continue; + } + lines.add(line); + } + + if (lines.getCount() == 0) + { + // If there are no lines, theres no content + return SLANG_OK; + } + + Index minIndent = 0x7fffffff; + for (const auto& line : lines) + { + const Index indent = _calcWhitespaceIndent(line); + minIndent = (indent < minIndent) ? indent : minIndent; + } + + for (const auto& line : lines) + { + _appendUnindenttedLine(line, minIndent, out); + out.appendChar('\n'); + } + + break; + } + default: return SLANG_FAIL; + } + + return SLANG_OK; +} + +Index DocMarkupExtractor::_findStartIndex(const FindInfo& info, Location location) +{ + Index openCount = 0; + + const TokenList& toks = *info.tokenList; + const Index tokIndex = info.tokenIndex; + + Index direction = isBefore(location) ? -1 : 1; + + const Index count = toks.m_tokens.getCount(); + for (Index i = tokIndex; i >= 0 && i < count; i += direction) + { + const Token& tok = toks.m_tokens[i]; + + switch (tok.type) + { + case TokenType::LBracket: + case TokenType::LParent: + case TokenType::OpLess: + { + openCount += direction; + if (openCount < 0) return -1; + break; + } + case TokenType::RBracket: + { + openCount -= direction; + if (openCount < 0) return -1; + break; + } + case TokenType::OpGreater: + { + if (location == Location::AfterGenericParam && openCount == 0) + { + return i + 1; + } + + openCount -= direction; + if (openCount < 0) return -1; + + break; + } + case TokenType::RParent: + { + if (openCount == 0 && location == Location::AfterParam) + { + return i + 1; + } + + openCount -= direction; + if (openCount < 0) return -1; + break; + } + case TokenType::RBrace: + { + // If we haven't hit a candidate yet before hitting } it's not going to work + if (location == Location::Before || location == Location::AfterEnumCase) + { + return -1; + } + break; + } + case TokenType::BlockComment: + case TokenType::LineComment: + { + if (openCount == 0) + { + // Determine the markup type + const MarkupType markupType = findMarkupType(tok); + // If the location wanted is before and the markup is, we'll assume this is it + if (isBefore(location) && isBefore(markupType)) + { + return i; + } + // If we are looking for enum cases, and the markup is after, we'll assume this is it + if (isAfter(location) && isAfter(markupType)) + { + return i; + } + } + break; + } + case TokenType::Comma: + { + if (openCount == 0) + { + if (location == Location::AfterParam || location == Location::AfterEnumCase || location == Location::AfterGenericParam) + { + return i + 1; + } + } + + if (location == Location::Before) + { + return -1; + } + + break; + } + case TokenType::Semicolon: + { + // If we haven't hit a candidate yet it's not going to work + if (location == Location::Before) + { + return -1; + } + if (openCount == 0 && location == Location::AfterSemicolon) + { + return i + 1; + } + break; + } + default: break; + } + } + + return -1; +} + +/* static */bool DocMarkupExtractor::_isTokenOnLineIndex(SourceView* sourceView, MarkupType type, const Token& tok, Index lineIndex) +{ + SourceFile* sourceFile = sourceView->getSourceFile(); + const int offset = sourceView->getRange().getOffset(tok.loc); + + auto const flags = getFlags(type); + + if (flags & MarkupFlag::IsBlock) + { + // Either the start or the end of the block have to be on the specified line + return sourceFile->isOffsetOnLine(offset, lineIndex) || sourceFile->isOffsetOnLine(offset + tok.charsCount, lineIndex); + } + else + { + // Has to be exactly on the specified line + return sourceFile->isOffsetOnLine(offset, lineIndex); + } +} + +SlangResult DocMarkupExtractor::_findMarkup(const FindInfo& info, Location location, FoundMarkup& out) +{ + out.reset(); + + const auto& toks = info.tokenList->m_tokens; + const Index tokIndex = info.tokenIndex; + + // The starting token index + Index startIndex = _findStartIndex(info, location); + if (startIndex <= 0) + { + return SLANG_E_NOT_FOUND; + } + + SourceView* sourceView = info.sourceView; + SourceFile* sourceFile = sourceView->getSourceFile(); + + // Let's lookup the line index where this occurred + const int startOffset = sourceView->getRange().getOffset(toks[startIndex - 1].loc); + + // The line index that the markoff starts from + Index lineIndex = sourceFile->calcLineIndexFromOffset(startOffset); + if (lineIndex < 0) + { + return SLANG_E_NOT_FOUND; + } + + const Index searchDirection = isBefore(location) ? -1 : 1; + + // Get the type and flags + const MarkupType type = findMarkupType(toks[startIndex]); + const MarkupFlags flags = getFlags(type); + + const MarkupFlag::Enum requiredFlag = isBefore(location) ? MarkupFlag::Before : MarkupFlag::After; + if ((flags & requiredFlag) == 0) + { + return SLANG_E_NOT_FOUND; + } + +#if 0 + // The token still isn't accepted, unless it's on the expected line + if (_isTokenOnLineIndex(info.sourceView, type, toks[startIndex], expectedLineIndex)) + { + return SLANG_E_NOT_FOUND; + } +#endif + + Index endIndex = startIndex; + + // If it's multiline, so look for the end index + if (flags & MarkupFlag::IsMultiToken) + { + Index expectedLineIndex = lineIndex; + + // TODO(JS): + // We should probably do the work here to confirm indentation - but that + // requires knowing something about tabs, so for now we leave. + + while (true) + { + endIndex += searchDirection; + expectedLineIndex += searchDirection; + + if (endIndex < 0 || endIndex >= toks.getCount()) + { + break; + } + + // Do we find a token of the right type? + if (findMarkupType(toks[endIndex]) != type) + { + break; + } + + // Is it on the right line? + if (_isTokenOnLineIndex(info.sourceView, type, toks[startIndex], expectedLineIndex)) + { + break; + } + } + + // Fix the end index (it's the last one that worked) + endIndex -= searchDirection; + } + + // Put start < end order + if (endIndex < startIndex) + { + Swap(endIndex, startIndex); + } + // The range excludes end so increase + endIndex++; + + // Okay we've found the markup + out.type = type; + out.location = location; + out.range = IndexRange{ startIndex, endIndex }; + + SLANG_ASSERT(out.range.getCount() > 0); + + return SLANG_OK; +} + +SlangResult DocMarkupExtractor::_findFirstMarkup(const FindInfo& info, const Location* locs, Index locCount, FoundMarkup& out, Index& outIndex) +{ + Index i = 0; + for (; i < locCount; ++i) + { + SlangResult res = _findMarkup(info, locs[i], out); + if (SLANG_SUCCEEDED(res) || (SLANG_FAILED(res) && res != SLANG_E_NOT_FOUND)) + { + outIndex = i; + return res; + } + } + return SLANG_E_NOT_FOUND; +} + +SlangResult DocMarkupExtractor::_findMarkup(const FindInfo& info, const Location* locs, Index locCount, FoundMarkup& out) +{ + Index foundIndex; + SLANG_RETURN_ON_FAIL(_findFirstMarkup(info, locs, locCount, out, foundIndex)); + + // Lets see if the remaining ones match + { + FoundMarkup otherMarkup; + for (Index i = foundIndex + 1; i < locCount; ++i) + { + SlangResult res = _findMarkup(info, locs[i], otherMarkup); + if (SLANG_SUCCEEDED(res)) + { + // TODO(JS): Warning found markup in another location + } + } + } + + return SLANG_OK; +} + +/* static */DocMarkupExtractor::SearchStyle DocMarkupExtractor::getSearchStyle(Decl* decl) +{ + if (auto enumCaseDecl = as(decl)) + { + return SearchStyle::EnumCase; + } + if (auto paramDecl = as(decl)) + { + return SearchStyle::Param; + } + else if (auto callableDecl = as(decl)) + { + return SearchStyle::Function; + } + else if (as(decl) || as(decl) || as(decl)) + { + return SearchStyle::Variable; + } + else if (auto genericDecl = as(decl)) + { + return getSearchStyle(genericDecl->inner); + } + else if (as(decl) || as(decl)) + { + return SearchStyle::GenericParam; + } + else + { + // If can't determine just allow before + return SearchStyle::Before; + } +} + +SlangResult DocMarkupExtractor::_findMarkup(const FindInfo& info, SearchStyle searchStyle, FoundMarkup& out) +{ + switch (searchStyle) + { + default: + case SearchStyle::None: + { + return SLANG_E_NOT_FOUND; + } + case SearchStyle::EnumCase: + { + Location locs[] = { Location::Before, Location::AfterEnumCase }; + return _findMarkup(info, locs, SLANG_COUNT_OF(locs), out); + } + case SearchStyle::Param: + { + Location locs[] = { Location::Before, Location::AfterParam }; + return _findMarkup(info, locs, SLANG_COUNT_OF(locs), out); + } + case SearchStyle::Before: + case SearchStyle::Function: + { + return _findMarkup(info, Location::Before, out); + } + case SearchStyle::Variable: + { + Location locs[] = { Location::Before, Location::AfterSemicolon }; + return _findMarkup(info, locs, SLANG_COUNT_OF(locs), out); + } + case SearchStyle::GenericParam: + { + Location locs[] = { Location::Before, Location::AfterGenericParam }; + return _findMarkup(info, locs, SLANG_COUNT_OF(locs), out); + } + } +} + +SlangResult DocMarkupExtractor::extract(const SearchItemInput* inputs, Index inputCount, SourceManager* sourceManager, DiagnosticSink* sink, List& outViews, List& out) +{ + struct Entry + { + typedef Entry ThisType; + + Index viewIndex; ///< The view/file index this loc is found in + SourceLoc::RawValue locOrOffset; ///< Can be a loc or an offset into the file + + SearchStyle searchStyle; ///< The search style when looking for an item + Index inputIndex; ///< The index to this item in the input + }; + + List entries; + + { + entries.setCount(inputCount); + for (Index i = 0; i < inputCount; ++i) + { + const auto& input = inputs[i]; + Entry& entry = entries[i]; + entry.inputIndex = i; + entry.viewIndex = -1; //< We don't know what file/view it's in + entry.locOrOffset = input.sourceLoc.getRaw(); + entry.searchStyle = input.searchStyle; + } + } + + // Sort them into loc order + entries.sort([](Entry& a, Entry& b) { return a.locOrOffset < b.locOrOffset; }); + + { + SourceView* sourceView = nullptr; + Index viewIndex = -1; + + for (auto& entry : entries) + { + const SourceLoc loc = SourceLoc::fromRaw(entry.locOrOffset); + + if (sourceView == nullptr || !sourceView->getRange().contains(loc)) + { + // Find the new view + sourceView = sourceManager->findSourceView(loc); + SLANG_ASSERT(sourceView); + + // We want only one view per SourceFile + SourceFile* sourceFile = sourceView->getSourceFile(); + + // NOTE! The view found might be different than sourceView. + viewIndex = outViews.findFirstIndex([&](SourceView* currentView) -> bool { return currentView->getSourceFile() == sourceFile; }); + + if (viewIndex < 0) + { + viewIndex = outViews.getCount(); + outViews.add(sourceView); + } + } + + SLANG_ASSERT(viewIndex >= 0); + SLANG_ASSERT(sourceView && sourceView->getRange().contains(loc)); + + // Set the file index + entry.viewIndex = viewIndex; + // Set as the offset within the file + entry.locOrOffset = sourceView->getRange().getOffset(loc); + } + + // Sort into view/file and then offset order + entries.sort([](Entry& a, Entry& b) { return (a.viewIndex < b.viewIndex) || ((a.viewIndex == b.viewIndex) && a.locOrOffset < b.locOrOffset); }); + } + + { + TokenList tokens; + + MemoryArena memoryArena; + RootNamePool rootNamePool; + NamePool namePool; + namePool.setRootNamePool(&rootNamePool); + + Index viewIndex = -1; + SourceView* sourceView = nullptr; + + const Int entryCount = entries.getCount(); + + out.setCount(entryCount); + + for (Index i = 0; i < entryCount; ++i) + { + const auto& entry = entries[i]; + auto& dst = out[i]; + + dst.viewIndex = -1; + dst.inputIndex = entry.inputIndex; + + // If there isn't a mechanism to search with, just move on + if (entry.searchStyle == SearchStyle::None) + { + continue; + } + + if (viewIndex != entry.viewIndex) + { + viewIndex = entry.viewIndex; + sourceView = outViews[viewIndex]; + + // Make all memory free again + memoryArena.reset(); + + // Run the lexer + Lexer lexer; + lexer.initialize(sourceView, sink, &namePool, &memoryArena, Lexer::OptionFlag::TokenizeComments); + + // Lex everything + tokens = lexer.lexAllTokens(); + } + + dst.viewIndex = viewIndex; + + // Get the offset within the source file + const uint32_t offset = entry.locOrOffset; + + // We need to get the loc in the source views space, so we look up appropriately in the list of tokens (which uses the views loc range) + const SourceLoc loc = sourceView->getRange().getSourceLocFromOffset(offset); + + // Work out the line number + SourceFile* sourceFile = sourceView->getSourceFile(); + const Index lineIndex = sourceFile->calcLineIndexFromOffset(int(offset)); + + // Okay, lets find the token index with a binary chop + Index tokenIndex = _findTokenIndex(loc, tokens.m_tokens.getBuffer(), tokens.m_tokens.getCount()); + if (tokenIndex >= 0 && lineIndex >= 0) + { + FindInfo findInfo; + findInfo.tokenIndex = tokenIndex; + findInfo.lineIndex = lineIndex; + findInfo.tokenList = &tokens; + findInfo.sourceView = sourceView; + + // Okay let's see if we extract some documentation then for this. + FoundMarkup foundMarkup; + SlangResult res = _findMarkup(findInfo, entry.searchStyle, foundMarkup); + + if (SLANG_SUCCEEDED(res)) + { + // We need to extract + StringBuilder buf; + SLANG_RETURN_ON_FAIL(_extractMarkup(findInfo, foundMarkup, buf)); + + // Save the extracted text in the output + dst.text = buf; + + } + else if (res != SLANG_E_NOT_FOUND) + { + return res; + } + } + } + } + + return SLANG_OK; +} + +static void _addDeclRec(Decl* decl, List& outDecls) +{ + if (decl == nullptr) + { + return; + } + + // If we don't have a loc, we have no way of locating documentation. + if (decl->loc.isValid() || decl->nameAndLoc.loc.isValid()) + { + outDecls.add(decl); + } + else + { + SLANG_ASSERT(!"Decl without a location!"); + } + + if (GenericDecl* genericDecl = as(decl)) + { + _addDeclRec(genericDecl->inner, outDecls); + } + + if (ContainerDecl* containerDecl = as(decl)) + { + // Add the container - which could be a class, struct, enum, namespace, extension, generic etc. + // Now add what the container contains + for (Decl* childDecl : containerDecl->members) + { + _addDeclRec(childDecl, outDecls); + } + } +} + +/* static */void DocMarkupExtractor::findDecls(ModuleDecl* moduleDecl, List& outDecls) +{ + for (Decl* decl : moduleDecl->members) + { + _addDeclRec(decl, outDecls); + } +} + +SlangResult DocMarkupExtractor::extract(ModuleDecl* moduleDecl, SourceManager* sourceManager, DiagnosticSink* sink, DocMarkup* outDoc) +{ + List decls; + findDecls(moduleDecl, decls); + + const Index declsCount = decls.getCount(); + + List inputItems; + List outItems; + + { + inputItems.setCount(declsCount); + + for (Index i = 0; i < declsCount; ++i) + { + Decl* decl = decls[i]; + auto& item = inputItems[i]; + + item.sourceLoc = decl->loc.isValid() ? decl->loc : decl->nameAndLoc.loc; + // Has to be valid to be lookupable + SLANG_ASSERT(item.sourceLoc.isValid()); + + item.searchStyle = getSearchStyle(decl); + } + + DocMarkupExtractor extractor; + + List views; + SLANG_RETURN_ON_FAIL(extractor.extract(inputItems.getBuffer(), declsCount, sourceManager, sink, views, outItems)); + } + + // Set back + for (Index i = 0; i < declsCount; ++i) + { + const auto& outputItem = outItems[i]; + const auto& inputItem = inputItems[outputItem.inputIndex]; + + // If we don't know how to search add to the output + if (inputItem.searchStyle != SearchStyle::None) + { + Decl* decl = decls[outputItem.inputIndex]; + + // Add to the documentation + DocMarkup::Entry& docEntry = outDoc->addEntry(decl); + docEntry.m_markup = outputItem.text; + } + } + + return SLANG_OK; +} + +} // namespace Slang diff --git a/source/slang/slang-doc-extractor.h b/source/slang/slang-doc-extractor.h new file mode 100644 index 000000000..ca03fc60b --- /dev/null +++ b/source/slang/slang-doc-extractor.h @@ -0,0 +1,210 @@ +// slang-doc.h +#ifndef SLANG_DOC_EXTRACTOR_H +#define SLANG_DOC_EXTRACTOR_H + +#include "../core/slang-basic.h" +#include "slang-ast-all.h" + +namespace Slang { + +/* Holds the documentation markup that is associated with each node (typically a decl) from a module */ +class DocMarkup : public RefObject +{ +public: + struct Entry + { + NodeBase* m_node; ///< The node this documentation is associated with + String m_markup; ///< The raw contents of of markup associated with the decoration + }; + + /// Adds an entry, returns the reference to pre-existing node if there is one + Entry& addEntry(NodeBase* base); + /// Gets an entry for a node. Returns nullptr if there is no markup. + Entry* getEntry(NodeBase* base); + + /// Get list of all of the entries in source order + const List& getEntries() const { return m_entries; } + +protected: + + /// Map from AST nodes to documentation entries + Dictionary m_entryMap; + /// All of the documentation entries in source order + List m_entries; +}; + +// --------------------------------------------------------------------------- +SLANG_INLINE DocMarkup::Entry& DocMarkup::addEntry(NodeBase* base) +{ + const Index count = m_entries.getCount(); + const Index index = m_entryMap.GetOrAddValue(base, count); + + if (index == count) + { + Entry entry; + entry.m_node = base; + m_entries.add(entry); + } + return m_entries[index]; +} + +// --------------------------------------------------------------------------- +SLANG_INLINE DocMarkup::Entry* DocMarkup::getEntry(NodeBase* base) +{ + Index* indexPtr = m_entryMap.TryGetValue(base); + return (indexPtr) ? &m_entries[*indexPtr] : nullptr; +} + +/* Extracts 'markup' from comments in Slang source core. The comments are extracted and associated in declarations. The association +is held in DocMarkup type. The comment style follows the doxygen style */ +class DocMarkupExtractor +{ +public: + + typedef uint32_t MarkupFlags; + struct MarkupFlag + { + enum Enum : MarkupFlags + { + Before = 0x1, + After = 0x2, + IsMultiToken = 0x4, ///< Can use more than one token + IsBlock = 0x8, ///< + }; + }; + + // NOTE! Don't change order without fixing isBefore and isAfter + enum class MarkupType + { + None, + + BlockBefore, /// /** */ or /*! */. + LineBangBefore, /// //! Can be multiple lines + LineSlashBefore, /// /// Can be multiple lines + + BlockAfter, /// /*!< */ or /**< */ + LineBangAfter, /// //!< Can be multiple lines + LineSlashAfter, /// ///< Can be multiple lines + }; + + static bool isBefore(MarkupType type) { return Index(type) >= Index(MarkupType::BlockBefore) && Index(type) <= Index(MarkupType::LineSlashBefore); } + static bool isAfter(MarkupType type) { return Index(type) >= Index(MarkupType::BlockAfter); } + + struct IndexRange + { + SLANG_FORCE_INLINE Index getCount() const { return end - start; } + + Index start; + Index end; + }; + + enum class Location + { + None, ///< No defined location + Before, + AfterParam, ///< Can have trailing , or ) + AfterSemicolon, ///< Can have a trailing ; + AfterEnumCase, ///< Can have a , or before } + AfterGenericParam, ///< Can have trailing , or > + }; + + static bool isAfter(Location location) { return Index(location) >= Index(Location::AfterParam); } + static bool isBefore(Location location) { return location == Location::Before; } + + struct FoundMarkup + { + void reset() + { + location = Location::None; + type = MarkupType::None; + range = IndexRange{ 0, 0 }; + } + + Location location = Location::None; + MarkupType type = MarkupType::None; + IndexRange range; + }; + + enum SearchStyle + { + None, ///< Cannot be searched for + EnumCase, ///< An enum case + Param, ///< A parameter in a function/method + Variable, ///< A variable-like declaration + Before, ///< Only allows before + Function, ///< Function/method + GenericParam, ///< Generic parameter + }; + + /// An input search item + struct SearchItemInput + { + SourceLoc sourceLoc; + SearchStyle searchStyle; ///< The search style when looking for an item + }; + + /// The items will be in source order + struct SearchItemOutput + { + Index viewIndex; ///< Index into the array of views on the output + Index inputIndex; ///< The index to this item in the input + String text; ///< The found text + }; + + struct FindInfo + { + SourceView* sourceView; ///< The source view the tokens were generated from + TokenList* tokenList; ///< The token list + Index tokenIndex; ///< The token index location (where searches start from) + Index lineIndex; ///< The line number for the decl + }; + + /// Extracts documentation from the nodes held in the module using the source manager. Found documentation is placed + /// in outMarkup + static SlangResult extract(ModuleDecl* moduleDecl, SourceManager* sourceManager, DiagnosticSink* sink, DocMarkup* outMarkup); + + /// Extracts 'markup' doc information for the specified input items + /// The output is placed in out - with the items now in the source order *not* the order of the input items + /// The inputIndex on the output holds the input item index + /// The outViews holds the views specified in viewIndex in the output, which may be useful for determining where the documentation was placed in source + SlangResult extract(const SearchItemInput* inputItems, Index inputCount, SourceManager* sourceManager, DiagnosticSink* sink, List& outViews, List& out); + + /// Given a module finds all the decls, and places in outDecls + static void findDecls(ModuleDecl* moduleDecl, List& outDecls); + + /// Given a decl determines the search style that is appropriate. Returns None if can't determine a suitable style + static SearchStyle getSearchStyle(Decl* decl); + + static MarkupFlags getFlags(MarkupType type); + static MarkupType findMarkupType(const Token& tok); + static UnownedStringSlice removeStart(MarkupType type, const UnownedStringSlice& comment); + +protected: + /// returns SLANG_E_NOT_FOUND if not found, SLANG_OK on success else an error + SlangResult _findMarkup(const FindInfo& info, Location location, FoundMarkup& out); + + /// Locations are processed in order, and the first successful used. If found in another location will issue a warning. + /// returns SLANG_E_NOT_FOUND if not found, SLANG_OK on success else an error + SlangResult _findFirstMarkup(const FindInfo& info, const Location* locs, Index locCount, FoundMarkup& out, Index& outIndex); + + SlangResult _findMarkup(const FindInfo& info, const Location* locs, Index locCount, FoundMarkup& out); + + /// Given the decl, the token stream, and the decls tokenIndex, try to find some associated markup + SlangResult _findMarkup(const FindInfo& info, SearchStyle searchStyle, FoundMarkup& out); + + /// Given a found markup location extracts the contents of the tokens into out + SlangResult _extractMarkup(const FindInfo& info, const FoundMarkup& foundMarkup, StringBuilder& out); + + /// Given a location, try to find the first token index that could potentially be markup + /// Will return -1 if not found + Index _findStartIndex(const FindInfo& info, Location location); + + /// True if the tok is 'on' lineIndex. Interpretation of 'on' depends on the markup type. + static bool _isTokenOnLineIndex(SourceView* sourceView, MarkupType type, const Token& tok, Index lineIndex); + + DiagnosticSink* m_sink; +}; + +} // namespace Slang + +#endif diff --git a/source/slang/slang-doc-mark-down.cpp b/source/slang/slang-doc-mark-down.cpp new file mode 100644 index 000000000..82de7052e --- /dev/null +++ b/source/slang/slang-doc-mark-down.cpp @@ -0,0 +1,477 @@ +// slang-doc-mark-down.cpp +#include "slang-doc-mark-down.h" + +#include "../core/slang-string-util.h" + +#include "slang-ast-builder.h" + +namespace Slang { + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DocMarkDownWriter !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +template +static void _getDecls(ContainerDecl* containerDecl, List& out) +{ + for (Decl* decl : containerDecl->members) + { + if (T* declAsType = as(decl)) + { + out.add(declAsType); + } + } +} + +template +static void _getDeclsOfType(ContainerDecl* containerDecl, List& out) +{ + for (Decl* decl : containerDecl->members) + { + if (as(decl)) + { + out.add(decl); + } + } +} + +template +static void _toList(FilteredMemberList& list, List& out) +{ + for (Decl* decl : list) + { + out.add(decl); + } +} + +static void _appendAsSingleLine(const UnownedStringSlice& in, StringBuilder& out) +{ + List lines; + StringUtil::calcLines(in, lines); + + // Ideally we'd remove any extraneous whitespace, but for now just join + StringUtil::join(lines.getBuffer(), lines.getCount(), ' ', out); +} + +void DocMarkDownWriter::_appendAsBullets(const List& in) +{ + auto& out = m_builder; + for (auto decl : in) + { + DocMarkup::Entry* paramEntry = m_markup->getEntry(decl); + + out << "* "; + + Name* name = decl->getName(); + if (name) + { + out << toSlice("_") << name->text << toSlice("_ "); + } + + if (paramEntry) + { + // Hmm, we'll want to make something multiline into a single line + _appendAsSingleLine(paramEntry->m_markup.getUnownedSlice(), out); + } + + out << "\n"; + } + + out << toSlice("\n"); +} + +template +void DocMarkDownWriter::_appendAsBullets(FilteredMemberList& list) +{ + List decls; + _toList(list, decls); + _appendAsBullets(decls); +} + +/* static */void DocMarkDownWriter::getSignature(const List& parts, Signature& outSig) +{ + const Index count = parts.getCount(); + for (Index i = 0; i < count; ++i) + { + const auto& part = parts[i]; + switch (part.type) + { + case Part::Type::ParamType: + { + PartPair pair; + pair.first = part; + if ((i + 1) < count && parts[i + 1].type == Part::Type::ParamName) + { + pair.second = parts[i + 1]; + i++; + } + outSig.params.add(pair); + break; + } + case Part::Type::ReturnType: + { + outSig.returnType = part; + break; + } + case Part::Type::DeclPath: + { + outSig.name = part; + break; + } + case Part::Type::GenericParamValue: + case Part::Type::GenericParamType: + { + Signature::GenericParam genericParam; + genericParam.name = part; + + if ((i + 1) < count && parts[i + 1].type == Part::Type::GenericParamValueType) + { + genericParam.type = parts[i + 1]; + i++; + } + + outSig.genericParams.add(genericParam); + break; + } + + default: break; + } + } +} + +void DocMarkDownWriter::writeVar(const DocMarkup::Entry& entry, VarDecl* varDecl) +{ + writePreamble(entry); + auto& out = m_builder; + + out << toSlice("# ") << varDecl->getName()->text << toSlice("\n\n"); + + // TODO(JS): The outputting of types this way isn't right - it doesn't handle int a[10] for example. + //ASTPrinter printer(m_astBuilder, ASTPrinter::OptionFlag::ParamNames); + + out << toSlice("```\n"); + out << varDecl->type << toSlice(" ") << varDecl << toSlice("\n"); + out << toSlice("```\n"); + + writeDescription(entry); +} + +void DocMarkDownWriter::writeCallable(const DocMarkup::Entry& entry, CallableDecl* callableDecl) +{ + writePreamble(entry); + + auto& out = m_builder; + + List parts; + ASTPrinter printer(m_astBuilder, ASTPrinter::OptionFlag::ParamNames, &parts); + + GenericDecl* genericDecl = as(callableDecl->parentDecl); + + if (genericDecl) + { + printer.addDeclSignature(DeclRef(genericDecl, nullptr)); + } + else + { + printer.addDeclSignature(DeclRef(callableDecl, nullptr)); + } + + Signature signature; + getSignature(parts, signature); + + const Index paramCount = signature.params.getCount(); + + // Output the signature + { + // Extract the name + out << toSlice("# ") << printer.getPartSlice(signature.name) << toSlice("\n\n"); + + out << toSlice("## Signature \n"); + out << toSlice("```\n"); + out << printer.getPartSlice(signature.returnType) << toSlice(" "); + + out << printer.getPartSlice(signature.name); + + if (signature.genericParams.getCount()) + { + out << toSlice("<"); + const Index count = signature.genericParams.getCount(); + for (Index i = 0; i < count; ++i) + { + const auto& genericParam = signature.genericParams[i]; + if (i > 0) + { + out << toSlice(", "); + } + out << printer.getPartSlice(genericParam.name); + + if (genericParam.type.type != Part::Type::None) + { + out << toSlice(" : "); + out << printer.getPartSlice(genericParam.type); + } + } + out << toSlice(">"); + } + + if (paramCount > 0) + { + out << toSlice("(\n"); + + StringBuilder line; + for (Index i = 0; i < paramCount; ++i) + { + const auto& param = signature.params[i]; + line.Clear(); + // If we want to tab these over... we'll need to know how must space I have + line << " " << printer.getPartSlice(param.first); + + Index indent = 25; + if (line.getLength() < indent) + { + line.appendRepeatedChar(' ', indent - line.getLength()); + } + else + { + line.appendChar(' '); + } + + line << printer.getPartSlice(param.second); + if (i < paramCount - 1) + { + line << ",\n"; + } + + out << line; + } + + out << ");\n"; + } + else + { + out << toSlice("();\n"); + } + + out << "```\n\n"; + } + + { + // The parameters, in order + List params; + + if (genericDecl) + { + for (Decl* decl : genericDecl->members) + { + if (as(decl) || + as(decl)) + { + params.add(decl); + } + } + } + + for (ParamDecl* paramDecl : callableDecl->getParameters()) + { + params.add(paramDecl); + } + + if (params.getCount()) + { + out << "## Parameters\n\n"; + // We have generic params and regular parameters, in this list + _appendAsBullets(params); + } + } + + writeDescription(entry); +} + +void DocMarkDownWriter::writeEnum(const DocMarkup::Entry& entry, EnumDecl* enumDecl) +{ + writePreamble(entry); + + auto& out = m_builder; + + out << toSlice("# enum "); + Name* name = enumDecl->getName(); + if (name) + { + out << name->text; + } + out << toSlice("\n\n"); + + out << toSlice("## Values \n\n"); + + auto cases = enumDecl->getMembersOfType(); + _appendAsBullets(cases); + + writeDescription(entry); +} + +void DocMarkDownWriter::_appendDerivedFrom(const UnownedStringSlice& prefix, AggTypeDeclBase* aggTypeDecl) +{ + auto& out = m_builder; + + List inheritanceDecls; + _getDecls(aggTypeDecl, inheritanceDecls); + + const Index count = inheritanceDecls.getCount(); + if (count) + { + out << prefix; + for (Index i = 0; i < count; ++i) + { + InheritanceDecl* inheritanceDecl = inheritanceDecls[i]; + if (i > 0) + { + out << toSlice(", "); + } + out << inheritanceDecl->base; + } + } +} + +void DocMarkDownWriter::writeAggType(const DocMarkup::Entry& entry, AggTypeDeclBase* aggTypeDecl) +{ + writePreamble(entry); + + auto& out = m_builder; + + // This could be lots of different things - struct/class/extension/interface/.. + + out << toSlice("# "); + if (as(aggTypeDecl)) + { + out << toSlice("struct "); + } + else if (as(aggTypeDecl)) + { + out << toSlice("class "); + } + else if (as(aggTypeDecl)) + { + out << toSlice("interface "); + } + else if (ExtensionDecl* extensionDecl = as(aggTypeDecl)) + { + out << toSlice("extension ") << extensionDecl->targetType; + _appendDerivedFrom(toSlice(" : "), extensionDecl); + } + else + { + out << toSlice("?"); + } + + Name* name = aggTypeDecl->getName(); + if (name) + { + out << name->text; + } + out << toSlice("\n\n"); + + { + List inheritanceDecls; + _getDecls(aggTypeDecl, inheritanceDecls); + + if (inheritanceDecls.getCount()) + { + out << "*Derives from:* "; + + for (Index i = 0; i < inheritanceDecls.getCount(); ++i) + { + if (i > 0) + { + out << toSlice(", "); + } + out << inheritanceDecls[i]->base; + } + out << toSlice("\n\n"); + } + } + + { + List fields; + _getDeclsOfType(aggTypeDecl, fields); + if (fields.getCount()) + { + out << "## Fields\n\n"; + _appendAsBullets(fields); + } + } + + { + List methods; + _getDeclsOfType(aggTypeDecl, methods); + if (methods.getCount()) + { + out << "## Methods\n\n"; + _appendAsBullets(methods); + } + } + + writeDescription(entry); +} + +void DocMarkDownWriter::writePreamble(const DocMarkup::Entry& entry) +{ + SLANG_UNUSED(entry); + auto& out = m_builder; + + out << toSlice("\n"); + out.appendRepeatedChar('-', 80); + out << toSlice("\n"); +} + + +void DocMarkDownWriter::writeDescription(const DocMarkup::Entry& entry) +{ + auto& out = m_builder; + + out << toSlice("\n## Description\n\n"); + out << entry.m_markup; +} + +void DocMarkDownWriter::writeDecl(const DocMarkup::Entry& entry, Decl* decl) +{ + // Skip these they will be output as part of their respective 'containers' + if (as(decl) || as(decl)) + { + return; + } + + if (CallableDecl* callableDecl = as(decl)) + { + writeCallable(entry, callableDecl); + } + else if (EnumDecl* enumDecl = as(decl)) + { + writeEnum(entry, enumDecl); + } + else if (AggTypeDeclBase* aggType = as(decl)) + { + writeAggType(entry, aggType); + } + else if (VarDecl* varDecl = as(decl)) + { + writeVar(entry, varDecl); + } + else if (as(decl)) + { + // We can ignore as inner decls will be picked up, and written + } +} + + +void DocMarkDownWriter::writeAll() +{ + for (const auto& entry : m_markup->getEntries()) + { + NodeBase* node = entry.m_node; + Decl* decl = as(node); + if (decl) + { + writeDecl(entry, decl); + } + } +} + +} // namespace Slang diff --git a/source/slang/slang-doc-mark-down.h b/source/slang/slang-doc-mark-down.h new file mode 100644 index 000000000..e1728d18a --- /dev/null +++ b/source/slang/slang-doc-mark-down.h @@ -0,0 +1,70 @@ +// slang-doc-markdown.h +#ifndef SLANG_DOC_MARK_DOWN_H +#define SLANG_DOC_MARK_DOWN_H + +#include "slang-doc-extractor.h" +#include "slang-ast-print.h" + +namespace Slang { + +class ASTBuilder; + +struct DocMarkDownWriter +{ + typedef ASTPrinter::Part Part; + typedef ASTPrinter::PartPair PartPair; + + struct Signature + { + struct GenericParam + { + Part name; + Part type; + }; + + Part returnType; + List params; + List genericParams; + Part name; + }; + + /// Write out all documentation to the output buffer + void writeAll(); + + void writeCallable(const DocMarkup::Entry& entry, CallableDecl* callable); + void writeEnum(const DocMarkup::Entry& entry, EnumDecl* enumDecl); + void writeAggType(const DocMarkup::Entry& entry, AggTypeDeclBase* aggTypeDecl); + void writeDecl(const DocMarkup::Entry& entry, Decl* decl); + void writeVar(const DocMarkup::Entry& entry, VarDecl* varDecl); + + void writePreamble(const DocMarkup::Entry& entry); + void writeDescription(const DocMarkup::Entry& entry); + + /// Get the output string + const StringBuilder& getOutput() const { return m_builder; } + + /// Ctor. + DocMarkDownWriter(DocMarkup* markup, ASTBuilder* astBuilder) : + m_markup(markup), + m_astBuilder(astBuilder) + { + } + + /// Given a list of ASTPrinter::Parts, works out the different parts of the sig + static void getSignature(const List& parts, Signature& outSig); + + template + void _appendAsBullets(FilteredMemberList& in); + void _appendAsBullets(const List& in); + + /// Appends prefix and the list of types derived from + void _appendDerivedFrom(const UnownedStringSlice& prefix, AggTypeDeclBase* aggTypeDecl); + + DocMarkup* m_markup; + ASTBuilder* m_astBuilder; + StringBuilder m_builder; +}; + +} // namespace Slang + +#endif diff --git a/source/slang/slang-doc.cpp b/source/slang/slang-doc.cpp deleted file mode 100644 index c72250122..000000000 --- a/source/slang/slang-doc.cpp +++ /dev/null @@ -1,1249 +0,0 @@ -// slang-doc.cpp -#include "slang-doc.h" - -#include "../core/slang-string-util.h" - -#include "slang-ast-builder.h" -#include "slang-ast-print.h" - -namespace Slang { - -/* TODO(JS): - -* If Decls hand SourceRange, then we could use the range to simplify getting the Post markup, as will be trivial to get to the 'end' -* Need to handle preceeding * in some markup styles -* If we want to be able to disable markup we need a mechanism to do this. Probably define source ranges. - -* Need a way to take the extracted markup and produce suitable markdown -** This will need to display the decoration appropriately -*/ - -/* Extracts 'markup' from comments in Slang source core. The comments are extracted and associated in declarations. The association -is held in DocMarkup type. The comment style follows the doxygen style */ -class DocMarkupExtractor -{ -public: - - typedef uint32_t MarkupFlags; - struct MarkupFlag - { - enum Enum : MarkupFlags - { - Before = 0x1, - After = 0x2, - IsMultiToken = 0x4, ///< Can use more than one token - IsBlock = 0x8, ///< - }; - }; - - // NOTE! Don't change order without fixing isBefore and isAfter - enum class MarkupType - { - None, - - BlockBefore, /// /** */ or /*! */. - LineBangBefore, /// //! Can be multiple lines - LineSlashBefore, /// /// Can be multiple lines - - BlockAfter, /// /*!< */ or /**< */ - LineBangAfter, /// //!< Can be multiple lines - LineSlashAfter, /// ///< Can be multiple lines - }; - - static bool isBefore(MarkupType type) { return Index(type) >= Index(MarkupType::BlockBefore) && Index(type) <= Index(MarkupType::LineSlashBefore); } - static bool isAfter(MarkupType type) { return Index(type) >= Index(MarkupType::BlockAfter); } - - struct IndexRange - { - SLANG_FORCE_INLINE Index getCount() const { return end - start; } - - Index start; - Index end; - }; - - enum class Location - { - None, ///< No defined location - Before, - AfterParam, ///< Can have trailing , or ) - AfterSemicolon, ///< Can have a trailing ; - AfterEnumCase, ///< Can have a , or before } - }; - - static bool isAfter(Location location) { return Index(location) >= Index(Location::AfterParam); } - static bool isBefore(Location location) { return location == Location::Before; } - - struct FoundMarkup - { - void reset() - { - location = Location::None; - type = MarkupType::None; - range = IndexRange { 0, 0 }; - } - - Location location = Location::None; - MarkupType type = MarkupType::None; - IndexRange range; - }; - - struct FindInfo - { - - SourceView* sourceView; ///< The source view the tokens were generated from - TokenList* tokenList; ///< The token list - Index declTokenIndex; ///< The token index location (where searches start from) - Index declLineIndex; ///< The line number for the decl - }; - - SlangResult extract(DocMarkup* doc, ModuleDecl* moduleDecl, SourceManager* sourceManager, DiagnosticSink* sink); - - static MarkupFlags getFlags(MarkupType type); - static MarkupType findMarkupType(const Token& tok); - static UnownedStringSlice removeStart(MarkupType type, const UnownedStringSlice& comment); - -protected: - /// returns SLANG_E_NOT_FOUND if not found, SLANG_OK on success else an error - SlangResult _findMarkup(const FindInfo& info, Location location, FoundMarkup& out); - - /// Locations are processed in order, and the first successful used. If found in another location will issue a warning. - /// returns SLANG_E_NOT_FOUND if not found, SLANG_OK on success else an error - SlangResult _findFirstMarkup(const FindInfo& info, const Location* locs, Index locCount, FoundMarkup& out, Index& outIndex); - - SlangResult _findMarkup(const FindInfo& info, const Location* locs, Index locCount, FoundMarkup& out); - - /// Given the decl, the token stream, and the decls tokenIndex, try to find some associated markup - SlangResult _findMarkup(const FindInfo& info, Decl* decl, FoundMarkup& out); - - /// Given a found markup location extracts the contents of the tokens into out - SlangResult _extractMarkup(const FindInfo& info, const FoundMarkup& foundMarkup, StringBuilder& out); - - /// Given a location, try to find the first token index that could potentially be markup - /// Will return -1 if not found - Index _findStartIndex(const FindInfo& info, Location location); - - /// True if the tok is 'on' lineIndex. Interpretation of 'on' depends on the markup type. - static bool _isTokenOnLineIndex(SourceView* sourceView, MarkupType type, const Token& tok, Index lineIndex); - - void _addDecl(Decl* decl); - void _addDeclRec(Decl* decl); - void _findDecls(ModuleDecl* moduleDecl); - - List m_decls; - - DocMarkup* m_doc; - ModuleDecl* m_moduleDecl; - SourceManager* m_sourceManager; - DiagnosticSink* m_sink; -}; - -/* static */UnownedStringSlice DocMarkupExtractor::removeStart(MarkupType type, const UnownedStringSlice& comment) -{ - switch (type) - { - case MarkupType::BlockBefore: - { - if (comment.startsWith(UnownedStringSlice::fromLiteral("/**")) || - comment.startsWith(UnownedStringSlice::fromLiteral("/*!"))) - { - /// /** */ or /*! */. - return comment.tail(3); - } - return comment; - } - case MarkupType::BlockAfter: - { - - if (comment.startsWith(UnownedStringSlice::fromLiteral("/**<")) || - comment.startsWith(UnownedStringSlice::fromLiteral("/*!<"))) - { - /// /*!< */ or /**< */ - return comment.tail(4); - } - return comment; - } - - case MarkupType::LineBangBefore: - { - return comment.startsWith(UnownedStringSlice::fromLiteral("//!")) ? comment.tail(3) : comment; - } - case MarkupType::LineSlashBefore: - { - return comment.startsWith(UnownedStringSlice::fromLiteral("///")) ? comment.tail(3) : comment; - } - - case MarkupType::LineBangAfter: - { - /// //!< Can be multiple lines - return comment.startsWith(UnownedStringSlice::fromLiteral("//!<")) ? comment.tail(4) : comment; - } - case MarkupType::LineSlashAfter: - { - return comment.startsWith(UnownedStringSlice::fromLiteral("///<")) ? comment.tail(4) : comment; - } - default: break; - } - return comment; -} - -void DocMarkupExtractor::_addDecl(Decl* decl) -{ - if (!decl->loc.isValid()) - { - return; - } - m_decls.add(decl); -} - -void DocMarkupExtractor::_addDeclRec(Decl* decl) -{ - // Just add. - // There may be things we don't want to add, but just add them all of now - _addDecl(decl); - -#if 0 - if (CallableDecl* callableDecl = as(decl)) - { - // For callables (like functions), - - m_decls.add(callableDecl); - } - else -#endif - - if (ContainerDecl* containerDecl = as(decl)) - { - // Add the container - which could be a class, struct, enum, namespace, extension, generic etc. - // Now add what the container contains - for (Decl* childDecl : containerDecl->members) - { - _addDeclRec(childDecl); - } - } -} - -void DocMarkupExtractor::_findDecls(ModuleDecl* moduleDecl) -{ - for (Decl* decl : moduleDecl->members) - { - _addDeclRec(decl); - } -} - -static Index _findTokenIndex(SourceLoc loc, const Token* toks, Index numToks) -{ - // Use a binary search to find the token - Index lo = 0; - Index hi = numToks; - - while (lo + 1 < hi) - { - const Index mid = (hi + lo) >> 1; - const Token& midToken = toks[mid]; - - if (midToken.loc == loc) - { - return mid; - } - - if (midToken.loc.getRaw() <= loc.getRaw()) - { - lo = mid; - } - else - { - hi = mid; - } - } - - // Not found - return -1; -} - -/* static */DocMarkupExtractor::MarkupFlags DocMarkupExtractor::getFlags(MarkupType type) -{ - switch (type) - { - default: - case MarkupType::None: return 0; - case MarkupType::BlockBefore: return MarkupFlag::Before | MarkupFlag::IsBlock; - case MarkupType::BlockAfter: return MarkupFlag::After | MarkupFlag::IsBlock; - - case MarkupType::LineBangBefore: return MarkupFlag::Before | MarkupFlag::IsMultiToken; - case MarkupType::LineSlashBefore: return MarkupFlag::Before | MarkupFlag::IsMultiToken; - - case MarkupType::LineBangAfter: return MarkupFlag::After | MarkupFlag::IsMultiToken; - case MarkupType::LineSlashAfter: return MarkupFlag::After | MarkupFlag::IsMultiToken; - } -} - -/* static */DocMarkupExtractor::MarkupType DocMarkupExtractor::findMarkupType(const Token& tok) -{ - switch (tok.type) - { - case TokenType::BlockComment: - { - UnownedStringSlice slice = tok.getContent(); - if (slice.getLength() >= 3 && (slice[2] == '!' || slice[2] == '*')) - { - return (slice.getLength() >= 4 && slice[3] == '<') ? MarkupType::BlockAfter : MarkupType::BlockBefore; - } - break; - } - case TokenType::LineComment: - { - UnownedStringSlice slice = tok.getContent(); - if (slice.getLength() >= 3) - { - if (slice[2] == '!') - { - return (slice.getLength() >= 4 && slice[3] == '<') ? MarkupType::LineBangAfter : MarkupType::LineBangBefore; - } - else if (slice[2] == '/') - { - return (slice.getLength() >= 4 && slice[3] == '<') ? MarkupType::LineSlashAfter : MarkupType::LineSlashBefore; - } - } - break; - } - default: break; - } - return MarkupType::None; -} - -static Index _calcWhitespaceIndent(const UnownedStringSlice& line) -{ - // TODO(JS): For now we ignore tabs and just work out indentation based on spaces/assume ASCII - Index indent = 0; - const Index count = line.getLength(); - for (; indent < count && line[indent] == ' '; indent++); - return indent; -} - -static Index _calcIndent(const UnownedStringSlice& line) -{ - // TODO(JS): For now we just assume no tabs, and that every char is ASCII - return line.getLength(); -} - -static void _appendUnindenttedLine(const UnownedStringSlice& line, Index maxIndent, StringBuilder& out) -{ - Index indent = _calcWhitespaceIndent(line); - - // We want to remove indenting remove no more than maxIndent - if (maxIndent >= 0) - { - indent = (indent > maxIndent) ? maxIndent : indent; - } - - // Remove the indenting, and append to out - out.append(line.tail(indent)); -} - -SlangResult DocMarkupExtractor::_extractMarkup(const FindInfo& info, const FoundMarkup& foundMarkup, StringBuilder& out) -{ - SourceView* sourceView = info.sourceView; - SourceFile* sourceFile = sourceView->getSourceFile(); - - // Here we want to produce the text that is implied by the markup tokens. - // We want to removing surrounding markup, and to also keep appropriate indentation - - switch (foundMarkup.type) - { - case MarkupType::BlockBefore: - case MarkupType::BlockAfter: - { - // We should only have a single line - SLANG_ASSERT(foundMarkup.range.getCount() == 1); - - const auto& tok = info.tokenList->m_tokens[foundMarkup.range.start]; - uint32_t offset = sourceView->getRange().getOffset(tok.loc); - - const UnownedStringSlice startLine = sourceFile->getLineContainingOffset(offset); - - UnownedStringSlice content = tok.getContent(); - - // Split into lines - List lines; - - StringUtil::calcLines(content, lines); - - Index maxIndent = -1; - - StringBuilder unindentedLine; - - const Index linesCount = lines.getCount(); - for (Index i = 0; i < linesCount; ++i) - { - UnownedStringSlice line = lines[i]; - unindentedLine.Clear(); - - if (i == 0) - { - if (startLine.isMemoryContained(line.begin())) - { - // For now we'll ignore tabs, and that the indent amount is, the amount of *byte* - // NOTE! This is only appropriate for ASCII without tabs. - maxIndent = _calcIndent(UnownedStringSlice(startLine.begin(), line.begin())); - - // Let's strip the start stuff - line = removeStart(foundMarkup.type, line); - } - } - - if (i == linesCount - 1) - { - SLANG_ASSERT(line.tail(line.getLength() - 2) == UnownedStringSlice::fromLiteral("*/")); - // Remove the */ at the end of the line - line = line.head(line.getLength() - 2); - } - - if (i > 0) - { - _appendUnindenttedLine(line, maxIndent, unindentedLine); - } - else - { - unindentedLine.append(line); - } - - // If the first or last line are all white space, just ignore them - if ((i == linesCount - 1 || i == 0) && unindentedLine.getUnownedSlice().trim().getLength() == 0) - { - continue; - } - - out.append(unindentedLine); - out.appendChar('\n'); - } - - break; - } - case MarkupType::LineBangBefore: - case MarkupType::LineSlashBefore: - case MarkupType::LineBangAfter: - case MarkupType::LineSlashAfter: - { - // Holds the lines extracted, they may have some white space indenting (like the space at the start of //) - List lines; - - const auto& range = foundMarkup.range; - for (Index i = range.start; i < range.end; ++ i) - { - const auto& tok = info.tokenList->m_tokens[i]; - UnownedStringSlice line = tok.getContent(); - line = removeStart(foundMarkup.type, line); - - // If the first or last line are all white space, just ignore them - if ((i == range.start || i == range.end - 1) && line.trim().getLength() == 0) - { - continue; - } - lines.add(line); - } - - if (lines.getCount() == 0) - { - // If there are no lines, theres no content - return SLANG_OK; - } - - Index minIndent = 0x7fffffff; - for (const auto& line : lines) - { - const Index indent = _calcWhitespaceIndent(line); - minIndent = (indent < minIndent) ? indent : minIndent; - } - - for (const auto& line : lines) - { - _appendUnindenttedLine(line, minIndent, out); - out.appendChar('\n'); - } - - break; - } - default: return SLANG_FAIL; - } - - return SLANG_OK; -} - -Index DocMarkupExtractor::_findStartIndex(const FindInfo& info, Location location) -{ - Index openParensCount = 0; - Index openBracketCount = 0; - - const TokenList& toks = *info.tokenList; - const Index tokIndex = info.declTokenIndex; - - Index direction = isBefore(location) ? -1 : 1; - - const Index count = toks.m_tokens.getCount(); - for (Index i = tokIndex; i >= 0 && i < count; i += direction) - { - const Token& tok = toks.m_tokens[i]; - - switch (tok.type) - { - case TokenType::BlockComment: - case TokenType::LineComment: - { - if (openParensCount == 0 && openBracketCount == 0) - { - // Determine the markup type - const MarkupType markupType = findMarkupType(tok); - // If the location wanted is before and the markup is, we'll assume this is it - if (isBefore(location) && isBefore(markupType)) - { - return i; - } - // If we are looking for enum cases, and the markup is after, we'll assume this is it - if (location == Location::AfterEnumCase && isAfter(markupType)) - { - return i; - } - } - break; - } - case TokenType::LParent: - { - ++openParensCount; - break; - } - case TokenType::RBracket: - { - openBracketCount += Index(isBefore(location)); - break; - } - case TokenType::LBracket: - { - openBracketCount -= Index(isBefore(location)); - break; - } - case TokenType::RParent: - { - if (openParensCount == 0 && - location == Location::AfterParam) - { - return i + 1; - } - - --openParensCount; - if (openParensCount < 0) - { - // Not found - or weird parens at least - return -1; - } - break; - } - case TokenType::Comma: - { - if (location == Location::AfterParam || location == Location::AfterEnumCase) - { - return i + 1; - } - break; - } - case TokenType::RBrace: - { - // If we haven't hit a candidate yet before hitting } it's not going to work - if (location == Location::Before || location == Location::AfterEnumCase) - { - return -1; - } - break; - } - case TokenType::Semicolon: - { - // If we haven't hit a candidate yet it's not going to work - if (location == Location::Before) - { - return -1; - } - - if (openParensCount == 0 && location == Location::AfterSemicolon) - { - return i + 1; - } - break; - } - default: break; - } - } - - return -1; -} - -/* static */bool DocMarkupExtractor::_isTokenOnLineIndex(SourceView* sourceView, MarkupType type, const Token& tok, Index lineIndex) -{ - SourceFile* sourceFile = sourceView->getSourceFile(); - const int offset = sourceView->getRange().getOffset(tok.loc); - - auto const flags = getFlags(type); - - if (flags & MarkupFlag::IsBlock) - { - // Either the start or the end of the block have to be on the specified line - return sourceFile->isOffsetOnLine(offset, lineIndex) || sourceFile->isOffsetOnLine(offset + tok.charsCount, lineIndex); - } - else - { - // Has to be exactly on the specified line - return sourceFile->isOffsetOnLine(offset, lineIndex); - } -} - - -SlangResult DocMarkupExtractor::_findMarkup(const FindInfo& info, Location location, FoundMarkup& out) -{ - out.reset(); - - const auto& toks = info.tokenList->m_tokens; - const Index tokIndex = info.declTokenIndex; - - // The starting token index - Index startIndex = _findStartIndex(info, location); - if (startIndex <= 0) - { - return SLANG_E_NOT_FOUND; - } - - SourceView* sourceView = info.sourceView; - SourceFile* sourceFile = sourceView->getSourceFile(); - - // Let's lookup the line index where this occurred - const int startOffset = sourceView->getRange().getOffset(toks[startIndex - 1].loc); - - // The line index that the markoff starts from - Index lineIndex = sourceFile->calcLineIndexFromOffset(startOffset); - if (lineIndex < 0) - { - return SLANG_E_NOT_FOUND; - } - - const Index searchDirection = isBefore(location) ? -1 : 1; - - // Get the type and flags - const MarkupType type = findMarkupType(toks[startIndex]); - const MarkupFlags flags = getFlags(type); - - const MarkupFlag::Enum requiredFlag = isBefore(location) ? MarkupFlag::Before : MarkupFlag::After; - if ((flags & requiredFlag) == 0) - { - return SLANG_E_NOT_FOUND; - } - -#if 0 - // The token still isn't accepted, unless it's on the expected line - if (_isTokenOnLineIndex(info.sourceView, type, toks[startIndex], expectedLineIndex)) - { - return SLANG_E_NOT_FOUND; - } -#endif - - Index endIndex = startIndex; - - // If it's multiline, so look for the end index - if (flags & MarkupFlag::IsMultiToken) - { - Index expectedLineIndex = lineIndex; - - // TODO(JS): - // We should probably do the work here to confirm indentation - but that - // requires knowing something about tabs, so for now we leave. - - while (true) - { - endIndex += searchDirection; - expectedLineIndex += searchDirection; - - if (endIndex < 0 || endIndex >= toks.getCount()) - { - break; - } - - // Do we find a token of the right type? - if (findMarkupType(toks[endIndex]) != type) - { - break; - } - - // Is it on the right line? - if (_isTokenOnLineIndex(info.sourceView, type, toks[startIndex], expectedLineIndex)) - { - break; - } - } - - // Fix the end index (it's the last one that worked) - endIndex -= searchDirection; - } - - // Put start < end order - if (endIndex < startIndex) - { - Swap(endIndex, startIndex); - } - // The range excludes end so increase - endIndex++; - - // Okay we've found the markup - out.type = type; - out.location = location; - out.range = IndexRange{ startIndex, endIndex }; - - SLANG_ASSERT(out.range.getCount() > 0); - - return SLANG_OK; -} - -SlangResult DocMarkupExtractor::_findFirstMarkup(const FindInfo& info, const Location* locs, Index locCount, FoundMarkup& out, Index& outIndex) -{ - Index i = 0; - for (; i < locCount; ++i) - { - SlangResult res = _findMarkup(info, locs[i], out); - if (SLANG_SUCCEEDED(res) || (SLANG_FAILED(res) && res != SLANG_E_NOT_FOUND)) - { - outIndex = i; - return res; - } - } - return SLANG_E_NOT_FOUND; -} - -SlangResult DocMarkupExtractor::_findMarkup(const FindInfo& info, const Location* locs, Index locCount, FoundMarkup& out) -{ - Index foundIndex; - SLANG_RETURN_ON_FAIL(_findFirstMarkup(info, locs, locCount, out, foundIndex)); - - // Lets see if the remaining ones match - { - FoundMarkup otherMarkup; - for (Index i = foundIndex + 1; i < locCount; ++i) - { - SlangResult res = _findMarkup(info, locs[i], otherMarkup); - if (SLANG_SUCCEEDED(res)) - { - // TODO(JS): Warning found markup in another location - } - } - } - - return SLANG_OK; -} - -SlangResult DocMarkupExtractor::_findMarkup(const FindInfo& info, Decl* decl, FoundMarkup& out) -{ - if (auto enumCaseDecl = as(decl)) - { - Location locs[] = { Location::Before, Location::AfterEnumCase }; - return _findMarkup(info, locs, SLANG_COUNT_OF(locs), out); - } - if (auto paramDecl = as(decl)) - { - Location locs[] = { Location::Before, Location::AfterParam }; - return _findMarkup(info, locs, SLANG_COUNT_OF(locs), out); - } - else if (auto callableDecl = as(decl)) - { - // We allow it defined before - return _findMarkup(info, Location::Before, out); - } - else if (as(decl) || as(decl) || as(decl)) - { - Location locs[] = { Location::Before, Location::AfterSemicolon }; - return _findMarkup(info, locs, SLANG_COUNT_OF(locs), out); - } - else - { - // We'll only allow before - return _findMarkup(info, Location::Before, out); - } -} - -SlangResult DocMarkupExtractor::extract(DocMarkup* doc, ModuleDecl* moduleDecl, SourceManager* sourceManager, DiagnosticSink* sink) -{ - m_doc = doc; - m_moduleDecl = moduleDecl; - m_sourceManager = sourceManager; - m_sink = sink; - - _findDecls(moduleDecl); - - struct Entry - { - typedef Entry ThisType; - - bool operator<(const ThisType& rhs) const { return locOrOffset < rhs.locOrOffset; } - - Index viewIndex; ///< The view/file index this loc is found in - SourceLoc::RawValue locOrOffset; ///< Can be a loc or an offset into the file - - Decl* decl; ///< The decl - }; - - List entries; - - { - const Index count = m_decls.getCount(); - entries.setCount(count); - - for (Index i = 0; i < count; ++i) - { - Entry& entry = entries[i]; - auto decl = m_decls[i]; - entry.decl = decl; - entry.viewIndex = -1; //< We don't know what file/view it's in - entry.locOrOffset = decl->loc.getRaw(); - } - } - - // We hold one view per *SourceFile* - List views; - - // Sort them into loc order - entries.sort([](Entry& a, Entry& b) { return a.locOrOffset < b.locOrOffset; }); - - { - SourceView* sourceView = nullptr; - Index viewIndex = -1; - - for (auto& entry : entries) - { - const SourceLoc loc = SourceLoc::fromRaw(entry.locOrOffset); - - if (sourceView == nullptr || !sourceView->getRange().contains(loc)) - { - // Find the new view - sourceView = m_sourceManager->findSourceView(loc); - SLANG_ASSERT(sourceView); - - // We want only one view per SourceFile - SourceFile* sourceFile = sourceView->getSourceFile(); - - // NOTE! The view found might be different than sourceView. - viewIndex = views.findFirstIndex([&](SourceView* currentView) -> bool { return currentView->getSourceFile() == sourceFile; }); - - if (viewIndex < 0) - { - viewIndex = views.getCount(); - views.add(sourceView); - } - } - - SLANG_ASSERT(viewIndex >= 0); - SLANG_ASSERT(sourceView && sourceView->getRange().contains(loc)); - - // Set the file index - entry.viewIndex = viewIndex; - // Set as the offset within the file - entry.locOrOffset = sourceView->getRange().getOffset(loc); - } - - // Sort into view/file and then offset order - entries.sort([](Entry& a, Entry& b) { return (a.viewIndex < b.viewIndex) || ((a.viewIndex == b.viewIndex) && a.locOrOffset < b.locOrOffset); }); - } - - { - TokenList tokens; - - MemoryArena memoryArena; - RootNamePool rootNamePool; - NamePool namePool; - namePool.setRootNamePool(&rootNamePool); - - Index viewIndex = -1; - SourceView* sourceView = nullptr; - - for (auto& entry : entries) - { - if (viewIndex != entry.viewIndex) - { - viewIndex = entry.viewIndex; - sourceView = views[viewIndex]; - - // Make all memory free again - memoryArena.reset(); - - // Run the lexer - Lexer lexer; - lexer.initialize(sourceView, sink, &namePool, &memoryArena, Lexer::OptionFlag::TokenizeComments); - - // Lex everything - tokens = lexer.lexAllTokens(); - } - - // Get the offset within the source file - const uint32_t offset = entry.locOrOffset; - - // We need to get the loc in the source views space, so we look up appropriately in the list of tokens (which uses the views loc range) - const SourceLoc loc = sourceView->getRange().getSourceLocFromOffset(offset); - - // Work out the line number - SourceFile* sourceFile = sourceView->getSourceFile(); - const Index lineIndex = sourceFile->calcLineIndexFromOffset(int(offset)); - - // Okay, lets find the token index with a binary chop - Index tokenIndex = _findTokenIndex(loc, tokens.m_tokens.getBuffer(), tokens.m_tokens.getCount()); - if (tokenIndex >= 0 && lineIndex >= 0) - { - FindInfo findInfo; - findInfo.declTokenIndex = tokenIndex; - findInfo.declLineIndex = lineIndex; - findInfo.tokenList = &tokens; - findInfo.sourceView = sourceView; - - // Okay let's see if we extract some documentation then for this. - FoundMarkup foundMarkup; - SlangResult res = _findMarkup(findInfo, entry.decl, foundMarkup); - - if (SLANG_SUCCEEDED(res)) - { - // We need to extract - StringBuilder buf; - SLANG_RETURN_ON_FAIL(_extractMarkup(findInfo, foundMarkup, buf)); - - // Add to the documentation - DocMarkup::Entry& docEntry = m_doc->addEntry(entry.decl); - docEntry.m_markup = buf; - } - else if (res != SLANG_E_NOT_FOUND) - { - return res; - } - } - } - } - - return SLANG_OK; -} - -SlangResult DocMarkup::extract(ModuleDecl* moduleDecl, SourceManager* sourceManager, DiagnosticSink* sink) -{ - m_moduleDecl = moduleDecl; - - DocMarkupExtractor context; - return context.extract(this, moduleDecl, sourceManager, sink); -} - -/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DocMarkDownWriter !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ - - -struct DocMarkDownWriter -{ - typedef ASTPrinter::Part Part; - typedef ASTPrinter::PartPair PartPair; - - struct Signature - { - Part returnType; - List params; - Part name; - }; - - void write(); - - void writeCallable(const DocMarkup::Entry& entry, CallableDecl* callable); - void writeEnum(const DocMarkup::Entry& entry, EnumDecl* enumDecl); - void writeAggType(const DocMarkup::Entry& entry, AggTypeDecl* aggTypeDecl); - - void writePreamble(const DocMarkup::Entry& entry); - void writeDescription(const DocMarkup::Entry& entry); - - DocMarkDownWriter(DocMarkup* markup, ASTBuilder* astBuilder): - m_markup(markup), - m_astBuilder(astBuilder) - { - } - - static void getSignature(const List& parts, Signature& outSig); - - template - void _appendAsBullets(FilteredMemberList& in); - - DocMarkup* m_markup; - ASTBuilder* m_astBuilder; - StringBuilder m_builder; -}; - -static void _appendAsSingleLine(const UnownedStringSlice& in, StringBuilder& out) -{ - List lines; - StringUtil::calcLines(in, lines); - - // Ideally we'd remove any extraneous whitespace, but for now just join - StringUtil::join(lines.getBuffer(), lines.getCount(), ' ', out); -} - -template -void DocMarkDownWriter::_appendAsBullets(FilteredMemberList& list) -{ - auto& out = m_builder; - for (auto element : list) - { - DocMarkup::Entry* paramEntry = m_markup->getEntry(element); - - out << "* "; - - Name* name = element->getName(); - if (name) - { - out << toSlice("_") << name->text << toSlice("_ "); - } - - if (paramEntry) - { - // Hmm, we'll want to make something multiline into a single line - _appendAsSingleLine(paramEntry->m_markup.getUnownedSlice(), out); - } - - out << "\n"; - } - - out << toSlice("\n"); -} - -/* static */void DocMarkDownWriter::getSignature(const List& parts, Signature& outSig) -{ - const Index count = parts.getCount(); - for (Index i = 0; i < count; ++i) - { - const auto& part = parts[i]; - switch (part.type) - { - case Part::Type::ParamType: - { - PartPair pair; - pair.first = part; - if (parts[i + 1].type == Part::Type::ParamName) - { - pair.second = parts[i + 1]; - i++; - } - outSig.params.add(pair); - break; - } - case Part::Type::ReturnType: - { - outSig.returnType = part; - break; - } - case Part::Type::DeclPath: - { - outSig.name = part; - break; - } - default: break; - } - } -} - -void DocMarkDownWriter::writeCallable(const DocMarkup::Entry& entry, CallableDecl* callableDecl) -{ - writePreamble(entry); - - auto& out = m_builder; - - StringBuilder sigBuffer; - List parts; - ASTPrinter printer(m_astBuilder, ASTPrinter::OptionFlag::ParamNames, &parts); - - printer.addDeclSignature(DeclRef(callableDecl, nullptr)); - - Signature signature; - getSignature(parts, signature); - - const Index paramCount = signature.params.getCount(); - - // Output the signature - { - // Extract the name - out << toSlice("# ") << printer.getPartSlice(signature.name) << toSlice("\n\n"); - - out << toSlice("## Signature \n"); - out << toSlice("```\n"); - out << printer.getPartSlice(signature.returnType) << toSlice(" "); - - out << printer.getPartSlice(signature.name); - - - if (paramCount > 0) - { - out << toSlice("(\n"); - - StringBuilder line; - for (Index i = 0; i < paramCount; ++i) - { - const auto& param = signature.params[i]; - line.Clear(); - // If we want to tab these over... we'll need to know how must space I have - line << " " << printer.getPartSlice(param.first); - - Index indent = 25; - if (line.getLength() < indent) - { - line.appendRepeatedChar(' ', indent - line.getLength()); - } - else - { - line.appendChar(' '); - } - - line << printer.getPartSlice(param.second); - if (i < paramCount - 1) - { - line << ",\n"; - } - - out << line; - } - - out << ");\n"; - } - else - { - out << toSlice("();\n"); - } - - out << "```\n\n"; - } - - // Only output params if there are any - if (paramCount) - { - out << "## Parameters\n\n"; - - auto params = callableDecl->getParameters(); - _appendAsBullets(params); - } - - writeDescription(entry); -} - -void DocMarkDownWriter::writeEnum(const DocMarkup::Entry& entry, EnumDecl* enumDecl) -{ - writePreamble(entry); - - auto& out = m_builder; - - out << toSlice("# enum "); - Name* name = enumDecl->getName(); - if (name) - { - out << name->text; - } - out << toSlice("\n\n"); - - out << toSlice("## Values \n\n"); - - auto cases = enumDecl->getMembersOfType(); - _appendAsBullets(cases); - - writeDescription(entry); -} - -void DocMarkDownWriter::writeAggType(const DocMarkup::Entry& entry, AggTypeDecl* aggTypeDecl) -{ - writePreamble(entry); - - auto& out = m_builder; - - // This could be lots of different things - struct/class/extension/interface/.. - - out << toSlice("# "); - if (as(aggTypeDecl)) - { - out << toSlice("struct "); - } - else if (as(aggTypeDecl)) - { - out << toSlice("class "); - } - else - { - out << toSlice("?"); - } - - Name* name = aggTypeDecl->getName(); - if (name) - { - out << name->text; - } - out << toSlice("\n\n"); - - out << "## Fields\n\n"; - - auto fields = aggTypeDecl->getMembersOfType(); - _appendAsBullets(fields); - - writeDescription(entry); -} - -void DocMarkDownWriter::writePreamble(const DocMarkup::Entry& entry) -{ - SLANG_UNUSED(entry); - auto& out = m_builder; - - out << toSlice("\n"); - out.appendRepeatedChar('-', 80); - out << toSlice("\n"); -} - - -void DocMarkDownWriter::writeDescription(const DocMarkup::Entry& entry) -{ - auto& out = m_builder; - - out << toSlice("\n## Description\n\n"); - out << entry.m_markup; -} - -void DocMarkDownWriter::write() -{ - for (const auto& entry : m_markup->getEntries()) - { - NodeBase* node = entry.m_node; - Decl* decl = as(node); - if (!decl) - { - continue; - } - - // Skip these they will be output as part of their respective 'containers' - if (as(decl) || as(decl)) - { - continue; - } - - if (CallableDecl* callableDecl = as(decl)) - { - writeCallable(entry, callableDecl); - } - else if (EnumDecl* enumDecl = as(decl)) - { - writeEnum(entry, enumDecl); - } - else if (AggTypeDecl* aggType = as(decl)) - { - writeAggType(entry, aggType); - } - } -} - -/* static */SlangResult DocumentationUtil::writeMarkdown(DocMarkup* markup, ASTBuilder* astBuilder, StringBuilder& out) -{ - // The ASTBuilder is needed in order to be able to create ast types that can then be printed. - // It is *assumed* here, that them being transient on this temporary ASTBuilder, doesn't mutate - // any of the nodes from the ASTBuilder/s for the things being documented - - DocMarkDownWriter writer(markup, astBuilder); - writer.write(); - - Swap(out, writer.m_builder); - - return SLANG_OK; -} - -} // namespace Slang diff --git a/source/slang/slang-doc.h b/source/slang/slang-doc.h deleted file mode 100644 index b2fd8c664..000000000 --- a/source/slang/slang-doc.h +++ /dev/null @@ -1,73 +0,0 @@ -// slang-doc.h -#ifndef SLANG_DOC_H -#define SLANG_DOC_H - -#include "../core/slang-basic.h" -#include "slang-ast-all.h" -#include "slang-ast-print.h" - -namespace Slang { - -/* Holds the documentation markup that is associated with each node (typically a decl) from a module */ -class DocMarkup : public RefObject -{ -public: - struct Entry - { - NodeBase* m_node; ///< The node this documentation is associated with - String m_markup; ///< The raw contents of of markup associated with the decoration - }; - - /// Adds an entry, returns the reference to pre-existing node if there is one - Entry& addEntry(NodeBase* base); - /// Gets an entry for a node. Returns nullptr if there is no markup. - Entry* getEntry(NodeBase* base); - - /// Get list of all of the entries in source order - const List& getEntries() const { return m_entries; } - - /// Given a module extracts all the associated markup. - SlangResult extract(ModuleDecl* moduleDecl, SourceManager* sourceManager, DiagnosticSink* sink); - -protected: - - /// The module this information was extracted from - ModuleDecl* m_moduleDecl; - /// Map from AST nodes to documentation entries - Dictionary m_entryMap; - /// All of the documentation entries in source order - List m_entries; -}; - -// --------------------------------------------------------------------------- -SLANG_INLINE DocMarkup::Entry& DocMarkup::addEntry(NodeBase* base) -{ - const Index count = m_entries.getCount(); - const Index index = m_entryMap.GetOrAddValue(base, count); - - if (index == count) - { - Entry entry; - entry.m_node = base; - m_entries.add(entry); - } - return m_entries[index]; -} - -// --------------------------------------------------------------------------- -SLANG_INLINE DocMarkup::Entry* DocMarkup::getEntry(NodeBase* base) -{ - Index* indexPtr = m_entryMap.TryGetValue(base); - return (indexPtr) ? &m_entries[*indexPtr] : nullptr; -} - -class SharedASTBuilder; - -struct DocumentationUtil -{ - static SlangResult writeMarkdown(DocMarkup* markup, ASTBuilder* astBuilder, StringBuilder& out); -}; - -} // namespace Slang - -#endif diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp index 21f60c090..1820b24e5 100644 --- a/source/slang/slang.cpp +++ b/source/slang/slang.cpp @@ -30,7 +30,8 @@ #include "slang-serialize-ir.h" #include "slang-serialize-container.h" -#include "slang-doc.h" +#include "slang-doc-extractor.h" +#include "slang-doc-mark-down.h" #include "slang-check-impl.h" @@ -1808,8 +1809,7 @@ SlangResult FrontEndCompileRequest::executeActionsInner() for (TranslationUnitRequest* translationUnit : translationUnits) { RefPtr markup(new DocMarkup); - - markup->extract(translationUnit->getModuleDecl(), getSourceManager(), getSink()); + DocMarkupExtractor::extract(translationUnit->getModuleDecl(), getSourceManager(), getSink(), markup); // Hmm.. we can have multiple sourcefiles. So fir now we just pick the first, so as to come up with // a reasonable name @@ -1822,10 +1822,10 @@ SlangResult FrontEndCompileRequest::executeActionsInner() String fileName = Path::getFileNameWithoutExt(path); fileName.append(".md"); - StringBuilder buf; - DocumentationUtil::writeMarkdown(markup, astBuilder, buf); + DocMarkDownWriter writer(markup, astBuilder); + writer.writeAll(); - File::writeAllText(fileName, buf); + File::writeAllText(fileName, writer.getOutput()); } } } -- cgit v1.2.3