From f67d929c24babc302eb2807251fc09b084abac2e Mon Sep 17 00:00:00 2001 From: jsmall-nvidia Date: Wed, 9 Mar 2022 18:38:00 -0500 Subject: Initial support for documentation extraction in C++ (#2156) * #include an absolute path didn't work - because paths were taken to always be relative. * Split doc extractor such that can be used in C++ extractor. * Compiles. Update the stdlib docs. * Fix issue on release builds. * Add support for extracting documentation to C++ extractor. * Dump out markup. Make enum value backing type take tokens. * Node::Type -> Node::Kind * More improvements around Node::Type -> Node::Kind --- .../compiler-core/compiler-core.vcxproj | 2 + .../compiler-core/compiler-core.vcxproj.filters | 6 + build/visual-studio/slang/slang.vcxproj | 4 +- build/visual-studio/slang/slang.vcxproj.filters | 4 +- docs/stdlib-doc.md | 49 + source/compiler-core/slang-doc-extractor.cpp | 864 ++++++++++++++++++ source/compiler-core/slang-doc-extractor.h | 163 ++++ source/slang/slang-ast-print.h | 2 +- source/slang/slang-doc-ast.cpp | 139 +++ source/slang/slang-doc-ast.h | 82 ++ source/slang/slang-doc-extractor.cpp | 989 --------------------- source/slang/slang-doc-extractor.h | 219 ----- source/slang/slang-doc-markdown-writer.cpp | 22 +- source/slang/slang-doc-markdown-writer.h | 25 +- source/slang/slang.cpp | 10 +- tools/slang-cpp-extractor/cpp-extractor-main.cpp | 118 +++ tools/slang-cpp-extractor/macro-writer.cpp | 4 +- tools/slang-cpp-extractor/node-tree.cpp | 2 +- tools/slang-cpp-extractor/node.cpp | 132 ++- tools/slang-cpp-extractor/node.h | 126 ++- tools/slang-cpp-extractor/options.h | 1 + tools/slang-cpp-extractor/parser.cpp | 109 +-- tools/slang-cpp-extractor/parser.h | 10 +- tools/slang-cpp-extractor/unit-test.cpp | 4 +- 24 files changed, 1714 insertions(+), 1372 deletions(-) create mode 100644 source/compiler-core/slang-doc-extractor.cpp create mode 100644 source/compiler-core/slang-doc-extractor.h create mode 100644 source/slang/slang-doc-ast.cpp create mode 100644 source/slang/slang-doc-ast.h delete mode 100644 source/slang/slang-doc-extractor.cpp delete mode 100644 source/slang/slang-doc-extractor.h diff --git a/build/visual-studio/compiler-core/compiler-core.vcxproj b/build/visual-studio/compiler-core/compiler-core.vcxproj index e4149f12b..028b85bce 100644 --- a/build/visual-studio/compiler-core/compiler-core.vcxproj +++ b/build/visual-studio/compiler-core/compiler-core.vcxproj @@ -277,6 +277,7 @@ + @@ -309,6 +310,7 @@ + diff --git a/build/visual-studio/compiler-core/compiler-core.vcxproj.filters b/build/visual-studio/compiler-core/compiler-core.vcxproj.filters index 0bb53f1b8..76255bedf 100644 --- a/build/visual-studio/compiler-core/compiler-core.vcxproj.filters +++ b/build/visual-studio/compiler-core/compiler-core.vcxproj.filters @@ -18,6 +18,9 @@ Header Files + + Header Files + Header Files @@ -110,6 +113,9 @@ Source Files + + Source Files + Source Files diff --git a/build/visual-studio/slang/slang.vcxproj b/build/visual-studio/slang/slang.vcxproj index c2387346d..61ddc40f4 100644 --- a/build/visual-studio/slang/slang.vcxproj +++ b/build/visual-studio/slang/slang.vcxproj @@ -325,7 +325,7 @@ IF EXIST ..\..\..\external\slang-binaries\bin\windows-aarch64\slang-glslang.dll\ - + @@ -458,7 +458,7 @@ IF EXIST ..\..\..\external\slang-binaries\bin\windows-aarch64\slang-glslang.dll\ - + diff --git a/build/visual-studio/slang/slang.vcxproj.filters b/build/visual-studio/slang/slang.vcxproj.filters index d86cdcbea..ba0e4fe52 100644 --- a/build/visual-studio/slang/slang.vcxproj.filters +++ b/build/visual-studio/slang/slang.vcxproj.filters @@ -72,7 +72,7 @@ Header Files - + Header Files @@ -467,7 +467,7 @@ Source Files - + Source Files diff --git a/docs/stdlib-doc.md b/docs/stdlib-doc.md index 126a4555e..f2489bb97 100644 --- a/docs/stdlib-doc.md +++ b/docs/stdlib-doc.md @@ -57578,6 +57578,8 @@ T OutputPatch.subscript(uint index); * `InterlockedOrU64` * `InterlockedXorU64` * `InterlockedExchangeU64` +* `InterlockedAdd64` +* `InterlockedCompareExchange64` * `InterlockedAdd` * `InterlockedAnd` * `InterlockedCompareExchange` @@ -57900,6 +57902,53 @@ uint64_t RWByteAddressBuffer.InterlockedExchangeU64( * `byteAddress` * `value` +-------------------------------------------------------------------------------- +# `RWByteAddressBuffer.InterlockedAdd64` + +## Signature + +``` +void RWByteAddressBuffer.InterlockedAdd64( + uint byteAddress, + int64_t valueToAdd, + out int64_t outOriginalValue); +void RWByteAddressBuffer.InterlockedAdd64( + uint byteAddress, + uint64_t valueToAdd, + out uint64_t outOriginalValue); +``` + +## Parameters + +* `byteAddress` +* `valueToAdd` +* `outOriginalValue` + +-------------------------------------------------------------------------------- +# `RWByteAddressBuffer.InterlockedCompareExchange64` + +## Signature + +``` +void RWByteAddressBuffer.InterlockedCompareExchange64( + uint byteAddress, + int64_t compareValue, + int64_t value, + out int64_t outOriginalValue); +void RWByteAddressBuffer.InterlockedCompareExchange64( + uint byteAddress, + uint64_t compareValue, + uint64_t value, + out uint64_t outOriginalValue); +``` + +## Parameters + +* `byteAddress` +* `compareValue` +* `value` +* `outOriginalValue` + -------------------------------------------------------------------------------- # `RWByteAddressBuffer.InterlockedAdd` diff --git a/source/compiler-core/slang-doc-extractor.cpp b/source/compiler-core/slang-doc-extractor.cpp new file mode 100644 index 000000000..d26ed01d1 --- /dev/null +++ b/source/compiler-core/slang-doc-extractor.cpp @@ -0,0 +1,864 @@ +// slang-doc-extractor.cpp +#include "slang-doc-extractor.h" + +#include "../core/slang-string-util.h" + +namespace Slang { + +/* TODO(JS): + +* If Decls hand SourceRange, then we could use the range to simplify getting the Post markup, as will be trivial to get to the 'end' +* Need to handle preceeding * in some markup styles +* If we want to be able to disable markup we need a mechanism to do this. Probably define source ranges. + +* Need a way to take the extracted markup and produce suitable markdown +** This will need to display the decoration appropriately +*/ + +/* static */UnownedStringSlice DocMarkupExtractor::removeStart(MarkupType type, const UnownedStringSlice& comment) +{ + switch (type) + { + case MarkupType::BlockBefore: + { + if (comment.startsWith(UnownedStringSlice::fromLiteral("/**")) || + comment.startsWith(UnownedStringSlice::fromLiteral("/*!"))) + { + /// /** */ or /*! */. + return comment.tail(3); + } + return comment; + } + case MarkupType::BlockAfter: + { + + if (comment.startsWith(UnownedStringSlice::fromLiteral("/**<")) || + comment.startsWith(UnownedStringSlice::fromLiteral("/*!<"))) + { + /// /*!< */ or /**< */ + return comment.tail(4); + } + return comment; + } + + case MarkupType::LineBangBefore: + { + return comment.startsWith(UnownedStringSlice::fromLiteral("//!")) ? comment.tail(3) : comment; + } + case MarkupType::LineSlashBefore: + { + return comment.startsWith(UnownedStringSlice::fromLiteral("///")) ? comment.tail(3) : comment; + } + + case MarkupType::LineBangAfter: + { + /// //!< Can be multiple lines + return comment.startsWith(UnownedStringSlice::fromLiteral("//!<")) ? comment.tail(4) : comment; + } + case MarkupType::LineSlashAfter: + { + return comment.startsWith(UnownedStringSlice::fromLiteral("///<")) ? comment.tail(4) : comment; + } + default: break; + } + return comment; +} + +static Index _findTokenIndex(SourceLoc loc, const Token* toks, Index numToks) +{ + // Use a binary search to find the token + Index lo = 0; + Index hi = numToks; + + while (lo + 1 < hi) + { + const Index mid = (hi + lo) >> 1; + const Token& midToken = toks[mid]; + + if (midToken.loc == loc) + { + return mid; + } + + if (midToken.loc.getRaw() <= loc.getRaw()) + { + lo = mid; + } + else + { + hi = mid; + } + } + + // Not found + return -1; +} + +/* static */DocMarkupExtractor::MarkupFlags DocMarkupExtractor::getFlags(MarkupType type) +{ + switch (type) + { + default: + case MarkupType::None: return 0; + case MarkupType::BlockBefore: return MarkupFlag::Before | MarkupFlag::IsBlock; + case MarkupType::BlockAfter: return MarkupFlag::After | MarkupFlag::IsBlock; + + case MarkupType::LineBangBefore: return MarkupFlag::Before | MarkupFlag::IsMultiToken; + case MarkupType::LineSlashBefore: return MarkupFlag::Before | MarkupFlag::IsMultiToken; + + case MarkupType::LineBangAfter: return MarkupFlag::After | MarkupFlag::IsMultiToken; + case MarkupType::LineSlashAfter: return MarkupFlag::After | MarkupFlag::IsMultiToken; + } +} + +/* static */DocMarkupExtractor::MarkupType DocMarkupExtractor::findMarkupType(const Token& tok) +{ + switch (tok.type) + { + case TokenType::BlockComment: + { + UnownedStringSlice slice = tok.getContent(); + if (slice.getLength() >= 3 && (slice[2] == '!' || slice[2] == '*')) + { + return (slice.getLength() >= 4 && slice[3] == '<') ? MarkupType::BlockAfter : MarkupType::BlockBefore; + } + break; + } + case TokenType::LineComment: + { + UnownedStringSlice slice = tok.getContent(); + if (slice.getLength() >= 3) + { + if (slice[2] == '!') + { + return (slice.getLength() >= 4 && slice[3] == '<') ? MarkupType::LineBangAfter : MarkupType::LineBangBefore; + } + else if (slice[2] == '/') + { + return (slice.getLength() >= 4 && slice[3] == '<') ? MarkupType::LineSlashAfter : MarkupType::LineSlashBefore; + } + } + break; + } + default: break; + } + return MarkupType::None; +} + +static Index _calcWhitespaceIndent(const UnownedStringSlice& line) +{ + // TODO(JS): For now we ignore tabs and just work out indentation based on spaces/assume ASCII + Index indent = 0; + const Index count = line.getLength(); + for (; indent < count && line[indent] == ' '; indent++); + return indent; +} + +static Index _calcIndent(const UnownedStringSlice& line) +{ + // TODO(JS): For now we just assume no tabs, and that every char is ASCII + return line.getLength(); +} + +static void _appendUnindenttedLine(const UnownedStringSlice& line, Index maxIndent, StringBuilder& out) +{ + Index indent = _calcWhitespaceIndent(line); + + // We want to remove indenting remove no more than maxIndent + if (maxIndent >= 0) + { + indent = (indent > maxIndent) ? maxIndent : indent; + } + + // Remove the indenting, and append to out + out.append(line.tail(indent)); +} + +SlangResult DocMarkupExtractor::_extractMarkup(const FindInfo& info, const FoundMarkup& foundMarkup, StringBuilder& out) +{ + SourceView* sourceView = info.sourceView; + SourceFile* sourceFile = sourceView->getSourceFile(); + + // Here we want to produce the text that is implied by the markup tokens. + // We want to removing surrounding markup, and to also keep appropriate indentation + + switch (foundMarkup.type) + { + case MarkupType::BlockBefore: + case MarkupType::BlockAfter: + { + // We should only have a single line + SLANG_ASSERT(foundMarkup.range.getCount() == 1); + + const auto& tok = info.tokenList->m_tokens[foundMarkup.range.start]; + uint32_t offset = sourceView->getRange().getOffset(tok.loc); + + const UnownedStringSlice startLine = sourceFile->getLineContainingOffset(offset); + + UnownedStringSlice content = tok.getContent(); + + // Split into lines + List lines; + + StringUtil::calcLines(content, lines); + + Index maxIndent = -1; + + StringBuilder unindentedLine; + + const Index linesCount = lines.getCount(); + for (Index i = 0; i < linesCount; ++i) + { + UnownedStringSlice line = lines[i]; + unindentedLine.Clear(); + + if (i == 0) + { + if (startLine.isMemoryContained(line.begin())) + { + // For now we'll ignore tabs, and that the indent amount is, the amount of *byte* + // NOTE! This is only appropriate for ASCII without tabs. + maxIndent = _calcIndent(UnownedStringSlice(startLine.begin(), line.begin())); + + // Let's strip the start stuff + line = removeStart(foundMarkup.type, line); + } + } + + if (i == linesCount - 1) + { + SLANG_ASSERT(line.tail(line.getLength() - 2) == UnownedStringSlice::fromLiteral("*/")); + // Remove the */ at the end of the line + line = line.head(line.getLength() - 2); + } + + if (i > 0) + { + _appendUnindenttedLine(line, maxIndent, unindentedLine); + } + else + { + unindentedLine.append(line); + } + + // If the first or last line are all white space, just ignore them + if ((i == linesCount - 1 || i == 0) && unindentedLine.getUnownedSlice().trim().getLength() == 0) + { + continue; + } + + out.append(unindentedLine); + out.appendChar('\n'); + } + + break; + } + case MarkupType::LineBangBefore: + case MarkupType::LineSlashBefore: + case MarkupType::LineBangAfter: + case MarkupType::LineSlashAfter: + { + // Holds the lines extracted, they may have some white space indenting (like the space at the start of //) + List lines; + + const auto& range = foundMarkup.range; + for (Index i = range.start; i < range.end; ++ i) + { + const auto& tok = info.tokenList->m_tokens[i]; + UnownedStringSlice line = tok.getContent(); + line = removeStart(foundMarkup.type, line); + + // If the first or last line are all white space, just ignore them + if ((i == range.start || i == range.end - 1) && line.trim().getLength() == 0) + { + continue; + } + lines.add(line); + } + + if (lines.getCount() == 0) + { + // If there are no lines, theres no content + return SLANG_OK; + } + + Index minIndent = 0x7fffffff; + for (const auto& line : lines) + { + const Index indent = _calcWhitespaceIndent(line); + minIndent = (indent < minIndent) ? indent : minIndent; + } + + for (const auto& line : lines) + { + _appendUnindenttedLine(line, minIndent, out); + out.appendChar('\n'); + } + + break; + } + default: return SLANG_FAIL; + } + + return SLANG_OK; +} + +Index DocMarkupExtractor::_findStartIndex(const FindInfo& info, Location location) +{ + Index openCount = 0; + + const TokenList& toks = *info.tokenList; + const Index tokIndex = info.tokenIndex; + + Index direction = isBefore(location) ? -1 : 1; + + const Index count = toks.m_tokens.getCount(); + for (Index i = tokIndex; i >= 0 && i < count; i += direction) + { + const Token& tok = toks.m_tokens[i]; + + switch (tok.type) + { + case TokenType::LBrace: + case TokenType::LBracket: + case TokenType::LParent: + case TokenType::OpLess: + { + openCount += direction; + if (openCount < 0) return -1; + break; + } + case TokenType::RBracket: + { + openCount -= direction; + if (openCount < 0) return -1; + break; + } + case TokenType::OpGreater: + { + if (location == Location::AfterGenericParam && openCount == 0) + { + return i + 1; + } + + openCount -= direction; + if (openCount < 0) return -1; + + break; + } + case TokenType::RParent: + { + if (openCount == 0 && location == Location::AfterParam) + { + return i + 1; + } + + openCount -= direction; + if (openCount < 0) return -1; + break; + } + case TokenType::RBrace: + { + // If we haven't hit a candidate yet before hitting } it's not going to work + if (location == Location::Before || location == Location::AfterEnumCase) + { + return -1; + } + break; + } + case TokenType::BlockComment: + case TokenType::LineComment: + { + if (openCount == 0) + { + // Determine the markup type + const MarkupType markupType = findMarkupType(tok); + // If the location wanted is before and the markup is, we'll assume this is it + if (isBefore(location) && isBefore(markupType)) + { + return i; + } + // If we are looking for enum cases, and the markup is after, we'll assume this is it + if (isAfter(location) && isAfter(markupType)) + { + return i; + } + } + break; + } + case TokenType::Comma: + { + if (openCount == 0) + { + if (location == Location::AfterParam || location == Location::AfterEnumCase || location == Location::AfterGenericParam) + { + return i + 1; + } + + if (location == Location::Before) + { + return -1; + } + } + + break; + } + case TokenType::Semicolon: + { + // If we haven't hit a candidate yet it's not going to work + if (location == Location::Before) + { + return -1; + } + if (openCount == 0 && location == Location::AfterSemicolon) + { + return i + 1; + } + break; + } + default: break; + } + } + + return -1; +} + +/* static */bool DocMarkupExtractor::_isTokenOnLineIndex(SourceView* sourceView, MarkupType type, const Token& tok, Index lineIndex) +{ + SourceFile* sourceFile = sourceView->getSourceFile(); + const int offset = sourceView->getRange().getOffset(tok.loc); + + auto const flags = getFlags(type); + + if (flags & MarkupFlag::IsBlock) + { + // Either the start or the end of the block have to be on the specified line + return sourceFile->isOffsetOnLine(offset, lineIndex) || sourceFile->isOffsetOnLine(offset + tok.charsCount, lineIndex); + } + else + { + // Has to be exactly on the specified line + return sourceFile->isOffsetOnLine(offset, lineIndex); + } +} + +SlangResult DocMarkupExtractor::_findMarkup(const FindInfo& info, Location location, FoundMarkup& out) +{ + out.reset(); + + const auto& toks = info.tokenList->m_tokens; + const Index tokIndex = info.tokenIndex; + + // The starting token index + Index startIndex = _findStartIndex(info, location); + if (startIndex <= 0) + { + return SLANG_E_NOT_FOUND; + } + + SourceView* sourceView = info.sourceView; + SourceFile* sourceFile = sourceView->getSourceFile(); + + // Let's lookup the line index where this occurred + const int startOffset = sourceView->getRange().getOffset(toks[startIndex - 1].loc); + + // The line index that the markoff starts from + Index lineIndex = sourceFile->calcLineIndexFromOffset(startOffset); + if (lineIndex < 0) + { + return SLANG_E_NOT_FOUND; + } + + const Index searchDirection = isBefore(location) ? -1 : 1; + + // Get the type and flags + const MarkupType type = findMarkupType(toks[startIndex]); + const MarkupFlags flags = getFlags(type); + + const MarkupFlag::Enum requiredFlag = isBefore(location) ? MarkupFlag::Before : MarkupFlag::After; + if ((flags & requiredFlag) == 0) + { + return SLANG_E_NOT_FOUND; + } + +#if 0 + // The token still isn't accepted, unless it's on the expected line + if (_isTokenOnLineIndex(info.sourceView, type, toks[startIndex], expectedLineIndex)) + { + return SLANG_E_NOT_FOUND; + } +#endif + + Index endIndex = startIndex; + + // If it's multiline, so look for the end index + if (flags & MarkupFlag::IsMultiToken) + { + Index expectedLineIndex = lineIndex; + + // TODO(JS): + // We should probably do the work here to confirm indentation - but that + // requires knowing something about tabs, so for now we leave. + + while (true) + { + endIndex += searchDirection; + expectedLineIndex += searchDirection; + + if (endIndex < 0 || endIndex >= toks.getCount()) + { + break; + } + + // Do we find a token of the right type? + if (findMarkupType(toks[endIndex]) != type) + { + break; + } + + // Is it on the right line? + if (_isTokenOnLineIndex(info.sourceView, type, toks[startIndex], expectedLineIndex)) + { + break; + } + } + + // Fix the end index (it's the last one that worked) + endIndex -= searchDirection; + } + + // Put start < end order + if (endIndex < startIndex) + { + Swap(endIndex, startIndex); + } + // The range excludes end so increase + endIndex++; + + // Okay we've found the markup + out.type = type; + out.location = location; + out.range = IndexRange{ startIndex, endIndex }; + + SLANG_ASSERT(out.range.getCount() > 0); + + return SLANG_OK; +} + +SlangResult DocMarkupExtractor::_findFirstMarkup(const FindInfo& info, const Location* locs, Index locCount, FoundMarkup& out, Index& outIndex) +{ + Index i = 0; + for (; i < locCount; ++i) + { + SlangResult res = _findMarkup(info, locs[i], out); + if (SLANG_SUCCEEDED(res) || (SLANG_FAILED(res) && res != SLANG_E_NOT_FOUND)) + { + outIndex = i; + return res; + } + } + return SLANG_E_NOT_FOUND; +} + +SlangResult DocMarkupExtractor::_findMarkup(const FindInfo& info, const Location* locs, Index locCount, FoundMarkup& out) +{ + Index foundIndex; + SLANG_RETURN_ON_FAIL(_findFirstMarkup(info, locs, locCount, out, foundIndex)); + + // Lets see if the remaining ones match + { + FoundMarkup otherMarkup; + for (Index i = foundIndex + 1; i < locCount; ++i) + { + SlangResult res = _findMarkup(info, locs[i], otherMarkup); + if (SLANG_SUCCEEDED(res)) + { + // TODO(JS): Warning found markup in another location + } + } + } + + return SLANG_OK; +} + + +SlangResult DocMarkupExtractor::_findMarkup(const FindInfo& info, SearchStyle searchStyle, FoundMarkup& out) +{ + switch (searchStyle) + { + default: + case SearchStyle::None: + { + return SLANG_E_NOT_FOUND; + } + case SearchStyle::EnumCase: + { + Location locs[] = { Location::Before, Location::AfterEnumCase }; + return _findMarkup(info, locs, SLANG_COUNT_OF(locs), out); + } + case SearchStyle::Param: + { + Location locs[] = { Location::Before, Location::AfterParam }; + return _findMarkup(info, locs, SLANG_COUNT_OF(locs), out); + } + case SearchStyle::Before: + { + return _findMarkup(info, Location::Before, out); + } + case SearchStyle::Function: + { + return _findMarkup(info, Location::Before, out); + } + case SearchStyle::Variable: + { + Location locs[] = { Location::Before, Location::AfterSemicolon }; + return _findMarkup(info, locs, SLANG_COUNT_OF(locs), out); + } + case SearchStyle::GenericParam: + { + Location locs[] = { Location::Before, Location::AfterGenericParam }; + return _findMarkup(info, locs, SLANG_COUNT_OF(locs), out); + } + } +} + +static void _calcLineVisibility(SourceView* sourceView, const TokenList& toks, List& outLineVisibility) +{ + SourceFile* sourceFile = sourceView->getSourceFile(); + const auto& lineOffsets = sourceFile->getLineBreakOffsets(); + + outLineVisibility.setCount(lineOffsets.getCount() + 1); + + MarkupVisibility lastVisibility = MarkupVisibility::Public; + Index lastLine = 0; + + for (const auto& tok : toks) + { + if (tok.type == TokenType::LineComment) + { + UnownedStringSlice contents = tok.getContent(); + + MarkupVisibility newVisibility = lastVisibility; + + // Distinct from other markup + if (contents.startsWith(toSlice("//@"))) + { + UnownedStringSlice access = contents.tail(3).trim(); + if (access == "hidden:" || access == "private:") + { + newVisibility = MarkupVisibility::Hidden; + } + else if (access == "internal:") + { + newVisibility = MarkupVisibility::Internal; + } + else if (access == "public:") + { + newVisibility = MarkupVisibility::Public; + } + } + + if (newVisibility != lastVisibility) + { + // Work up the line it's on + const int offset = sourceView->getRange().getOffset(tok.loc); + Index line = sourceFile->calcLineIndexFromOffset(offset); + + // Fill in the span + for (Index i = lastLine; i < line; ++i) + { + outLineVisibility[i] = lastVisibility; + } + + // Record the new access and where we are up to + lastLine = line; + lastVisibility = newVisibility; + } + } + } + + // Fill in the remaining + for (Index i = lastLine; i < outLineVisibility.getCount(); ++ i) + { + outLineVisibility[i] = lastVisibility; + } +} + +SlangResult DocMarkupExtractor::extract(const SearchItemInput* inputs, Index inputCount, SourceManager* sourceManager, DiagnosticSink* sink, List& outViews, List& out) +{ + struct Entry + { + typedef Entry ThisType; + + Index viewIndex; ///< The view/file index this loc is found in + SourceLoc::RawValue locOrOffset; ///< Can be a loc or an offset into the file + + SearchStyle searchStyle; ///< The search style when looking for an item + Index inputIndex; ///< The index to this item in the input + }; + + List entries; + + { + entries.setCount(inputCount); + for (Index i = 0; i < inputCount; ++i) + { + const auto& input = inputs[i]; + Entry& entry = entries[i]; + entry.inputIndex = i; + entry.viewIndex = -1; //< We don't know what file/view it's in + entry.locOrOffset = input.sourceLoc.getRaw(); + entry.searchStyle = input.searchStyle; + } + } + + // Sort them into loc order + entries.sort([](const Entry& a, const Entry& b) -> bool { return a.locOrOffset < b.locOrOffset; }); + + { + SourceView* sourceView = nullptr; + Index viewIndex = -1; + + for (auto& entry : entries) + { + if (entry.searchStyle == SearchStyle::None) + { + continue; + } + + const SourceLoc loc = SourceLoc::fromRaw(entry.locOrOffset); + + if (sourceView == nullptr || !sourceView->getRange().contains(loc)) + { + // Find the new view + sourceView = sourceManager->findSourceView(loc); + SLANG_ASSERT(sourceView); + + // We want only one view per SourceFile + SourceFile* sourceFile = sourceView->getSourceFile(); + + // NOTE! The view found might be different than sourceView. + viewIndex = outViews.findFirstIndex([&](SourceView* currentView) -> bool { return currentView->getSourceFile() == sourceFile; }); + + if (viewIndex < 0) + { + viewIndex = outViews.getCount(); + outViews.add(sourceView); + } + } + + SLANG_ASSERT(viewIndex >= 0); + SLANG_ASSERT(sourceView && sourceView->getRange().contains(loc)); + + // Set the file index + entry.viewIndex = viewIndex; + // Set as the offset within the file + entry.locOrOffset = sourceView->getRange().getOffset(loc); + } + + // Sort into view/file and then offset order + entries.sort([](const Entry& a, const Entry& b) -> bool { return (a.viewIndex < b.viewIndex) || ((a.viewIndex == b.viewIndex) && a.locOrOffset < b.locOrOffset); }); + } + + { + TokenList tokens; + List lineVisibility; + + MemoryArena memoryArena(4096); + + RootNamePool rootNamePool; + NamePool namePool; + namePool.setRootNamePool(&rootNamePool); + + Index viewIndex = -1; + SourceView* sourceView = nullptr; + + const Int entryCount = entries.getCount(); + + out.setCount(entryCount); + + for (Index i = 0; i < entryCount; ++i) + { + const auto& entry = entries[i]; + auto& dst = out[i]; + + dst.viewIndex = -1; + dst.inputIndex = entry.inputIndex; + dst.visibilty = MarkupVisibility::Public; + + // If there isn't a mechanism to search with, just move on + if (entry.searchStyle == SearchStyle::None) + { + continue; + } + + if (viewIndex != entry.viewIndex) + { + viewIndex = entry.viewIndex; + sourceView = outViews[viewIndex]; + + // Make all memory free again + memoryArena.reset(); + + // Run the lexer + Lexer lexer; + lexer.initialize(sourceView, sink, &namePool, &memoryArena); + + // Lex everything + tokens = lexer.lexAllMarkupTokens(); + + // Let's work out the access + + _calcLineVisibility(sourceView, tokens, lineVisibility); + } + + dst.viewIndex = viewIndex; + + // Get the offset within the source file + const uint32_t offset = entry.locOrOffset; + + // We need to get the loc in the source views space, so we look up appropriately in the list of tokens (which uses the views loc range) + const SourceLoc loc = sourceView->getRange().getSourceLocFromOffset(offset); + + // Work out the line number + SourceFile* sourceFile = sourceView->getSourceFile(); + const Index lineIndex = sourceFile->calcLineIndexFromOffset(int(offset)); + + dst.visibilty = lineVisibility[lineIndex]; + + // Okay, lets find the token index with a binary chop + Index tokenIndex = _findTokenIndex(loc, tokens.m_tokens.getBuffer(), tokens.m_tokens.getCount()); + if (tokenIndex >= 0 && lineIndex >= 0) + { + FindInfo findInfo; + findInfo.tokenIndex = tokenIndex; + findInfo.lineIndex = lineIndex; + findInfo.tokenList = &tokens; + findInfo.sourceView = sourceView; + + // Okay let's see if we extract some documentation then for this. + FoundMarkup foundMarkup; + SlangResult res = _findMarkup(findInfo, entry.searchStyle, foundMarkup); + + if (SLANG_SUCCEEDED(res)) + { + // We need to extract + StringBuilder buf; + SLANG_RETURN_ON_FAIL(_extractMarkup(findInfo, foundMarkup, buf)); + + // Save the extracted text in the output + dst.text = buf; + + } + else if (res != SLANG_E_NOT_FOUND) + { + return res; + } + } + } + } + + return SLANG_OK; +} + +} // namespace Slang diff --git a/source/compiler-core/slang-doc-extractor.h b/source/compiler-core/slang-doc-extractor.h new file mode 100644 index 000000000..5b4c67782 --- /dev/null +++ b/source/compiler-core/slang-doc-extractor.h @@ -0,0 +1,163 @@ +// slang-doc.h +#ifndef SLANG_DOC_EXTRACTOR_H +#define SLANG_DOC_EXTRACTOR_H + +#include "../core/slang-basic.h" + +#include "slang-source-loc.h" +#include "slang-lexer.h" + +namespace Slang { + +enum class MarkupVisibility : uint8_t +{ + Public, ///< Always available + Internal, ///< Can be available in more verbose 'internal' documentation + Hidden, ///< Not generally available +}; + +/* Extracts 'markup' from comments in Slang source core. The comments are extracted and associated in declarations. The association +is held in DocMarkup type. The comment style follows the doxygen style */ +class DocMarkupExtractor +{ +public: + + typedef uint32_t MarkupFlags; + struct MarkupFlag + { + enum Enum : MarkupFlags + { + Before = 0x1, + After = 0x2, + IsMultiToken = 0x4, ///< Can use more than one token + IsBlock = 0x8, ///< + }; + }; + + // NOTE! Don't change order without fixing isBefore and isAfter + enum class MarkupType + { + None, + + BlockBefore, /// /** */ or /*! */. + LineBangBefore, /// //! Can be multiple lines + LineSlashBefore, /// /// Can be multiple lines + + BlockAfter, /// /*!< */ or /**< */ + LineBangAfter, /// //!< Can be multiple lines + LineSlashAfter, /// ///< Can be multiple lines + }; + + static bool isBefore(MarkupType type) { return Index(type) >= Index(MarkupType::BlockBefore) && Index(type) <= Index(MarkupType::LineSlashBefore); } + static bool isAfter(MarkupType type) { return Index(type) >= Index(MarkupType::BlockAfter); } + + struct IndexRange + { + SLANG_FORCE_INLINE Index getCount() const { return end - start; } + + Index start; + Index end; + }; + + enum class Location + { + None, ///< No defined location + Before, + AfterParam, ///< Can have trailing , or ) + AfterSemicolon, ///< Can have a trailing ; + AfterEnumCase, ///< Can have a , or before } + AfterGenericParam, ///< Can have trailing , or > + }; + + static bool isAfter(Location location) { return Index(location) >= Index(Location::AfterParam); } + static bool isBefore(Location location) { return location == Location::Before; } + + struct FoundMarkup + { + void reset() + { + location = Location::None; + type = MarkupType::None; + range = IndexRange{ 0, 0 }; + } + + Location location = Location::None; + MarkupType type = MarkupType::None; + IndexRange range; + }; + + enum SearchStyle + { + None, ///< Cannot be searched for + EnumCase, ///< An enum case + Param, ///< A parameter in a function/method + Variable, ///< A variable-like declaration + Before, ///< Only allows before + Function, ///< Function/method + GenericParam, ///< Generic parameter + }; + + /// An input search item + struct SearchItemInput + { + SourceLoc sourceLoc; + SearchStyle searchStyle; ///< The search style when looking for an item + }; + + /// The items will be in source order + struct SearchItemOutput + { + Index viewIndex; ///< Index into the array of views on the output + Index inputIndex; ///< The index to this item in the input + String text; ///< The found text + MarkupVisibility visibilty; ///< Visibility of the item + }; + + struct FindInfo + { + SourceView* sourceView; ///< The source view the tokens were generated from + TokenList* tokenList; ///< The token list + Index tokenIndex; ///< The token index location (where searches start from) + Index lineIndex; ///< The line number for the decl + }; + + + /// Extracts 'markup' doc information for the specified input items + /// The output is placed in out - with the items now in the source order *not* the order of the input items + /// The inputIndex on the output holds the input item index + /// The outViews holds the views specified in viewIndex in the output, which may be useful for determining where the documentation was placed in source + SlangResult extract(const SearchItemInput* inputItems, Index inputCount, SourceManager* sourceManager, DiagnosticSink* sink, List& outViews, List& out); + + static MarkupFlags getFlags(MarkupType type); + static MarkupType findMarkupType(const Token& tok); + static UnownedStringSlice removeStart(MarkupType type, const UnownedStringSlice& comment); + +protected: + /// returns SLANG_E_NOT_FOUND if not found, SLANG_OK on success else an error + SlangResult _findMarkup(const FindInfo& info, Location location, FoundMarkup& out); + + /// Locations are processed in order, and the first successful used. If found in another location will issue a warning. + /// returns SLANG_E_NOT_FOUND if not found, SLANG_OK on success else an error + SlangResult _findFirstMarkup(const FindInfo& info, const Location* locs, Index locCount, FoundMarkup& out, Index& outIndex); + + SlangResult _findMarkup(const FindInfo& info, const Location* locs, Index locCount, FoundMarkup& out); + + /// Given the decl, the token stream, and the decls tokenIndex, try to find some associated markup + SlangResult _findMarkup(const FindInfo& info, SearchStyle searchStyle, FoundMarkup& out); + + /// Given a found markup location extracts the contents of the tokens into out + SlangResult _extractMarkup(const FindInfo& info, const FoundMarkup& foundMarkup, StringBuilder& out); + + /// Given a location, try to find the first token index that could potentially be markup + /// Will return -1 if not found + Index _findStartIndex(const FindInfo& info, Location location); + + /// True if the tok is 'on' lineIndex. Interpretation of 'on' depends on the markup type. + static bool _isTokenOnLineIndex(SourceView* sourceView, MarkupType type, const Token& tok, Index lineIndex); + + DiagnosticSink* m_sink; +}; + +} // namespace Slang + +#endif diff --git a/source/slang/slang-ast-print.h b/source/slang/slang-ast-print.h index 1c4c20e28..15ca58acf 100644 --- a/source/slang/slang-ast-print.h +++ b/source/slang/slang-ast-print.h @@ -139,7 +139,7 @@ public: UnownedStringSlice getPartSlice(const Part& part) const { return getPart(getSlice(), part); } /// Gets the specified part type - static UnownedStringSlice getPart(const UnownedStringSlice& slice, const Part& part) { return UnownedStringSlice(slice.begin() + part.start, slice.begin() + part.end); } + static UnownedStringSlice getPart(const UnownedStringSlice& slice, const Part& part) { return (part.type != Part::Type::None) ? UnownedStringSlice(slice.begin() + part.start, slice.begin() + part.end) : UnownedStringSlice(); } static UnownedStringSlice getPart(Part::Type partType, const UnownedStringSlice& slice, const List& parts); static void appendDeclName(Decl* decl, StringBuilder& out); diff --git a/source/slang/slang-doc-ast.cpp b/source/slang/slang-doc-ast.cpp new file mode 100644 index 000000000..8301b1a63 --- /dev/null +++ b/source/slang/slang-doc-ast.cpp @@ -0,0 +1,139 @@ +// slang-doc-ast.cpp +#include "slang-doc-ast.h" + +#include "../core/slang-string-util.h" + +//#include "slang-ast-builder.h" +//#include "slang-ast-print.h" + +namespace Slang { + +/* static */DocMarkupExtractor::SearchStyle ASTMarkupUtil::getSearchStyle(Decl* decl) +{ + typedef Extractor::SearchStyle SearchStyle; + + if (auto enumCaseDecl = as(decl)) + { + return SearchStyle::EnumCase; + } + if (auto paramDecl = as(decl)) + { + return SearchStyle::Param; + } + else if (auto callableDecl = as(decl)) + { + return SearchStyle::Function; + } + else if (as(decl) || as(decl) || as(decl)) + { + return SearchStyle::Variable; + } + else if (auto genericDecl = as(decl)) + { + return getSearchStyle(genericDecl->inner); + } + else if (as(decl) || as(decl)) + { + return SearchStyle::GenericParam; + } + else + { + // If can't determine just allow before + return SearchStyle::Before; + } +} + +static void _addDeclRec(Decl* decl, List& outDecls) +{ + if (decl == nullptr) + { + return; + } + + // If we don't have a loc, we have no way of locating documentation. + if (decl->loc.isValid() || decl->nameAndLoc.loc.isValid()) + { + outDecls.add(decl); + } + else + { + SLANG_ASSERT(!"Decl without a location!"); + } + + if (GenericDecl* genericDecl = as(decl)) + { + _addDeclRec(genericDecl->inner, outDecls); + } + + if (ContainerDecl* containerDecl = as(decl)) + { + // Add the container - which could be a class, struct, enum, namespace, extension, generic etc. + // Now add what the container contains + for (Decl* childDecl : containerDecl->members) + { + _addDeclRec(childDecl, outDecls); + } + } +} + +/* static */void ASTMarkupUtil::findDecls(ModuleDecl* moduleDecl, List& outDecls) +{ + for (Decl* decl : moduleDecl->members) + { + _addDeclRec(decl, outDecls); + } +} + +SlangResult ASTMarkupUtil::extract(ModuleDecl* moduleDecl, SourceManager* sourceManager, DiagnosticSink* sink, ASTMarkup* outDoc) +{ + List decls; + findDecls(moduleDecl, decls); + + const Index declsCount = decls.getCount(); + + List inputItems; + List outItems; + + { + inputItems.setCount(declsCount); + + for (Index i = 0; i < declsCount; ++i) + { + Decl* decl = decls[i]; + auto& item = inputItems[i]; + + item.sourceLoc = decl->loc.isValid() ? decl->loc : decl->nameAndLoc.loc; + // Has to be valid to be lookupable + SLANG_ASSERT(item.sourceLoc.isValid()); + + item.searchStyle = getSearchStyle(decl); + } + + DocMarkupExtractor extractor; + + List views; + SLANG_RETURN_ON_FAIL(extractor.extract(inputItems.getBuffer(), declsCount, sourceManager, sink, views, outItems)); + } + + // Set back + for (Index i = 0; i < declsCount; ++i) + { + const auto& outputItem = outItems[i]; + const auto& inputItem = inputItems[outputItem.inputIndex]; + + // If we don't know how to search add to the output + if (inputItem.searchStyle != Extractor::SearchStyle::None) + { + Decl* decl = decls[outputItem.inputIndex]; + + // Add to the documentation + ASTMarkup::Entry& docEntry = outDoc->addEntry(decl); + docEntry.m_markup = outputItem.text; + docEntry.m_visibility = outputItem.visibilty; + } + } + + return SLANG_OK; +} + +} // namespace Slang diff --git a/source/slang/slang-doc-ast.h b/source/slang/slang-doc-ast.h new file mode 100644 index 000000000..c5291a31a --- /dev/null +++ b/source/slang/slang-doc-ast.h @@ -0,0 +1,82 @@ +// slang-doc-ast.h +#ifndef SLANG_DOC_AST_H +#define SLANG_DOC_AST_H + +#include "../core/slang-basic.h" + +#include "../compiler-core/slang-doc-extractor.h" + +#include "slang-ast-all.h" + +namespace Slang { + +/* Holds the documentation markup that is associated with each node (typically a decl) from a module */ +class ASTMarkup : public RefObject +{ +public: + struct Entry + { + NodeBase* m_node; ///< The node this documentation is associated with + String m_markup; ///< The raw contents of of markup associated with the decoration + MarkupVisibility m_visibility = MarkupVisibility::Public; ///< How visible this decl is + }; + + /// Adds an entry, returns the reference to pre-existing node if there is one + Entry& addEntry(NodeBase* base); + /// Gets an entry for a node. Returns nullptr if there is no markup. + Entry* getEntry(NodeBase* base); + + /// Get list of all of the entries in source order + const List& getEntries() const { return m_entries; } + +protected: + + /// Map from AST nodes to documentation entries + Dictionary m_entryMap; + /// All of the documentation entries in source order + List m_entries; +}; + +// --------------------------------------------------------------------------- +SLANG_INLINE ASTMarkup::Entry& ASTMarkup::addEntry(NodeBase* base) +{ + const Index count = m_entries.getCount(); + const Index index = m_entryMap.GetOrAddValue(base, count); + + if (index == count) + { + Entry entry; + entry.m_node = base; + m_entries.add(entry); + } + return m_entries[index]; +} + +// --------------------------------------------------------------------------- +SLANG_INLINE ASTMarkup::Entry* ASTMarkup::getEntry(NodeBase* base) +{ + Index* indexPtr = m_entryMap.TryGetValue(base); + return (indexPtr) ? &m_entries[*indexPtr] : nullptr; +} + +/* Extracts documentation markup from source. +The comments are extracted and associated in declarations. The association +is held in DocMarkup type. The comment style follows the doxygen style */ +struct ASTMarkupUtil +{ + typedef DocMarkupExtractor Extractor; + + /// Given a module finds all the decls, and places in outDecls + static void findDecls(ModuleDecl* moduleDecl, List& outDecls); + + /// Given a decl determines the search style that is appropriate. Returns None if can't determine a suitable style + static Extractor::SearchStyle getSearchStyle(Decl* decl); + + /// Extracts documentation from the nodes held in the module using the source manager. Found documentation is placed + /// in outMarkup + static SlangResult extract(ModuleDecl* moduleDecl, SourceManager* sourceManager, DiagnosticSink* sink, ASTMarkup* outMarkup); +}; + +} // namespace Slang + +#endif diff --git a/source/slang/slang-doc-extractor.cpp b/source/slang/slang-doc-extractor.cpp deleted file mode 100644 index 0677e2af4..000000000 --- a/source/slang/slang-doc-extractor.cpp +++ /dev/null @@ -1,989 +0,0 @@ -// slang-doc.cpp -#include "slang-doc-extractor.h" - -#include "../core/slang-string-util.h" - -#include "slang-ast-builder.h" -#include "slang-ast-print.h" - -namespace Slang { - -/* TODO(JS): - -* If Decls hand SourceRange, then we could use the range to simplify getting the Post markup, as will be trivial to get to the 'end' -* Need to handle preceeding * in some markup styles -* If we want to be able to disable markup we need a mechanism to do this. Probably define source ranges. - -* Need a way to take the extracted markup and produce suitable markdown -** This will need to display the decoration appropriately -*/ - -/* static */UnownedStringSlice DocMarkupExtractor::removeStart(MarkupType type, const UnownedStringSlice& comment) -{ - switch (type) - { - case MarkupType::BlockBefore: - { - if (comment.startsWith(UnownedStringSlice::fromLiteral("/**")) || - comment.startsWith(UnownedStringSlice::fromLiteral("/*!"))) - { - /// /** */ or /*! */. - return comment.tail(3); - } - return comment; - } - case MarkupType::BlockAfter: - { - - if (comment.startsWith(UnownedStringSlice::fromLiteral("/**<")) || - comment.startsWith(UnownedStringSlice::fromLiteral("/*!<"))) - { - /// /*!< */ or /**< */ - return comment.tail(4); - } - return comment; - } - - case MarkupType::LineBangBefore: - { - return comment.startsWith(UnownedStringSlice::fromLiteral("//!")) ? comment.tail(3) : comment; - } - case MarkupType::LineSlashBefore: - { - return comment.startsWith(UnownedStringSlice::fromLiteral("///")) ? comment.tail(3) : comment; - } - - case MarkupType::LineBangAfter: - { - /// //!< Can be multiple lines - return comment.startsWith(UnownedStringSlice::fromLiteral("//!<")) ? comment.tail(4) : comment; - } - case MarkupType::LineSlashAfter: - { - return comment.startsWith(UnownedStringSlice::fromLiteral("///<")) ? comment.tail(4) : comment; - } - default: break; - } - return comment; -} - -static Index _findTokenIndex(SourceLoc loc, const Token* toks, Index numToks) -{ - // Use a binary search to find the token - Index lo = 0; - Index hi = numToks; - - while (lo + 1 < hi) - { - const Index mid = (hi + lo) >> 1; - const Token& midToken = toks[mid]; - - if (midToken.loc == loc) - { - return mid; - } - - if (midToken.loc.getRaw() <= loc.getRaw()) - { - lo = mid; - } - else - { - hi = mid; - } - } - - // Not found - return -1; -} - -/* static */DocMarkupExtractor::MarkupFlags DocMarkupExtractor::getFlags(MarkupType type) -{ - switch (type) - { - default: - case MarkupType::None: return 0; - case MarkupType::BlockBefore: return MarkupFlag::Before | MarkupFlag::IsBlock; - case MarkupType::BlockAfter: return MarkupFlag::After | MarkupFlag::IsBlock; - - case MarkupType::LineBangBefore: return MarkupFlag::Before | MarkupFlag::IsMultiToken; - case MarkupType::LineSlashBefore: return MarkupFlag::Before | MarkupFlag::IsMultiToken; - - case MarkupType::LineBangAfter: return MarkupFlag::After | MarkupFlag::IsMultiToken; - case MarkupType::LineSlashAfter: return MarkupFlag::After | MarkupFlag::IsMultiToken; - } -} - -/* static */DocMarkupExtractor::MarkupType DocMarkupExtractor::findMarkupType(const Token& tok) -{ - switch (tok.type) - { - case TokenType::BlockComment: - { - UnownedStringSlice slice = tok.getContent(); - if (slice.getLength() >= 3 && (slice[2] == '!' || slice[2] == '*')) - { - return (slice.getLength() >= 4 && slice[3] == '<') ? MarkupType::BlockAfter : MarkupType::BlockBefore; - } - break; - } - case TokenType::LineComment: - { - UnownedStringSlice slice = tok.getContent(); - if (slice.getLength() >= 3) - { - if (slice[2] == '!') - { - return (slice.getLength() >= 4 && slice[3] == '<') ? MarkupType::LineBangAfter : MarkupType::LineBangBefore; - } - else if (slice[2] == '/') - { - return (slice.getLength() >= 4 && slice[3] == '<') ? MarkupType::LineSlashAfter : MarkupType::LineSlashBefore; - } - } - break; - } - default: break; - } - return MarkupType::None; -} - -static Index _calcWhitespaceIndent(const UnownedStringSlice& line) -{ - // TODO(JS): For now we ignore tabs and just work out indentation based on spaces/assume ASCII - Index indent = 0; - const Index count = line.getLength(); - for (; indent < count && line[indent] == ' '; indent++); - return indent; -} - -static Index _calcIndent(const UnownedStringSlice& line) -{ - // TODO(JS): For now we just assume no tabs, and that every char is ASCII - return line.getLength(); -} - -static void _appendUnindenttedLine(const UnownedStringSlice& line, Index maxIndent, StringBuilder& out) -{ - Index indent = _calcWhitespaceIndent(line); - - // We want to remove indenting remove no more than maxIndent - if (maxIndent >= 0) - { - indent = (indent > maxIndent) ? maxIndent : indent; - } - - // Remove the indenting, and append to out - out.append(line.tail(indent)); -} - -SlangResult DocMarkupExtractor::_extractMarkup(const FindInfo& info, const FoundMarkup& foundMarkup, StringBuilder& out) -{ - SourceView* sourceView = info.sourceView; - SourceFile* sourceFile = sourceView->getSourceFile(); - - // Here we want to produce the text that is implied by the markup tokens. - // We want to removing surrounding markup, and to also keep appropriate indentation - - switch (foundMarkup.type) - { - case MarkupType::BlockBefore: - case MarkupType::BlockAfter: - { - // We should only have a single line - SLANG_ASSERT(foundMarkup.range.getCount() == 1); - - const auto& tok = info.tokenList->m_tokens[foundMarkup.range.start]; - uint32_t offset = sourceView->getRange().getOffset(tok.loc); - - const UnownedStringSlice startLine = sourceFile->getLineContainingOffset(offset); - - UnownedStringSlice content = tok.getContent(); - - // Split into lines - List lines; - - StringUtil::calcLines(content, lines); - - Index maxIndent = -1; - - StringBuilder unindentedLine; - - const Index linesCount = lines.getCount(); - for (Index i = 0; i < linesCount; ++i) - { - UnownedStringSlice line = lines[i]; - unindentedLine.Clear(); - - if (i == 0) - { - if (startLine.isMemoryContained(line.begin())) - { - // For now we'll ignore tabs, and that the indent amount is, the amount of *byte* - // NOTE! This is only appropriate for ASCII without tabs. - maxIndent = _calcIndent(UnownedStringSlice(startLine.begin(), line.begin())); - - // Let's strip the start stuff - line = removeStart(foundMarkup.type, line); - } - } - - if (i == linesCount - 1) - { - SLANG_ASSERT(line.tail(line.getLength() - 2) == UnownedStringSlice::fromLiteral("*/")); - // Remove the */ at the end of the line - line = line.head(line.getLength() - 2); - } - - if (i > 0) - { - _appendUnindenttedLine(line, maxIndent, unindentedLine); - } - else - { - unindentedLine.append(line); - } - - // If the first or last line are all white space, just ignore them - if ((i == linesCount - 1 || i == 0) && unindentedLine.getUnownedSlice().trim().getLength() == 0) - { - continue; - } - - out.append(unindentedLine); - out.appendChar('\n'); - } - - break; - } - case MarkupType::LineBangBefore: - case MarkupType::LineSlashBefore: - case MarkupType::LineBangAfter: - case MarkupType::LineSlashAfter: - { - // Holds the lines extracted, they may have some white space indenting (like the space at the start of //) - List lines; - - const auto& range = foundMarkup.range; - for (Index i = range.start; i < range.end; ++ i) - { - const auto& tok = info.tokenList->m_tokens[i]; - UnownedStringSlice line = tok.getContent(); - line = removeStart(foundMarkup.type, line); - - // If the first or last line are all white space, just ignore them - if ((i == range.start || i == range.end - 1) && line.trim().getLength() == 0) - { - continue; - } - lines.add(line); - } - - if (lines.getCount() == 0) - { - // If there are no lines, theres no content - return SLANG_OK; - } - - Index minIndent = 0x7fffffff; - for (const auto& line : lines) - { - const Index indent = _calcWhitespaceIndent(line); - minIndent = (indent < minIndent) ? indent : minIndent; - } - - for (const auto& line : lines) - { - _appendUnindenttedLine(line, minIndent, out); - out.appendChar('\n'); - } - - break; - } - default: return SLANG_FAIL; - } - - return SLANG_OK; -} - -Index DocMarkupExtractor::_findStartIndex(const FindInfo& info, Location location) -{ - Index openCount = 0; - - const TokenList& toks = *info.tokenList; - const Index tokIndex = info.tokenIndex; - - Index direction = isBefore(location) ? -1 : 1; - - const Index count = toks.m_tokens.getCount(); - for (Index i = tokIndex; i >= 0 && i < count; i += direction) - { - const Token& tok = toks.m_tokens[i]; - - switch (tok.type) - { - case TokenType::LBrace: - case TokenType::LBracket: - case TokenType::LParent: - case TokenType::OpLess: - { - openCount += direction; - if (openCount < 0) return -1; - break; - } - case TokenType::RBracket: - { - openCount -= direction; - if (openCount < 0) return -1; - break; - } - case TokenType::OpGreater: - { - if (location == Location::AfterGenericParam && openCount == 0) - { - return i + 1; - } - - openCount -= direction; - if (openCount < 0) return -1; - - break; - } - case TokenType::RParent: - { - if (openCount == 0 && location == Location::AfterParam) - { - return i + 1; - } - - openCount -= direction; - if (openCount < 0) return -1; - break; - } - case TokenType::RBrace: - { - // If we haven't hit a candidate yet before hitting } it's not going to work - if (location == Location::Before || location == Location::AfterEnumCase) - { - return -1; - } - break; - } - case TokenType::BlockComment: - case TokenType::LineComment: - { - if (openCount == 0) - { - // Determine the markup type - const MarkupType markupType = findMarkupType(tok); - // If the location wanted is before and the markup is, we'll assume this is it - if (isBefore(location) && isBefore(markupType)) - { - return i; - } - // If we are looking for enum cases, and the markup is after, we'll assume this is it - if (isAfter(location) && isAfter(markupType)) - { - return i; - } - } - break; - } - case TokenType::Comma: - { - if (openCount == 0) - { - if (location == Location::AfterParam || location == Location::AfterEnumCase || location == Location::AfterGenericParam) - { - return i + 1; - } - - if (location == Location::Before) - { - return -1; - } - } - - break; - } - case TokenType::Semicolon: - { - // If we haven't hit a candidate yet it's not going to work - if (location == Location::Before) - { - return -1; - } - if (openCount == 0 && location == Location::AfterSemicolon) - { - return i + 1; - } - break; - } - default: break; - } - } - - return -1; -} - -/* static */bool DocMarkupExtractor::_isTokenOnLineIndex(SourceView* sourceView, MarkupType type, const Token& tok, Index lineIndex) -{ - SourceFile* sourceFile = sourceView->getSourceFile(); - const int offset = sourceView->getRange().getOffset(tok.loc); - - auto const flags = getFlags(type); - - if (flags & MarkupFlag::IsBlock) - { - // Either the start or the end of the block have to be on the specified line - return sourceFile->isOffsetOnLine(offset, lineIndex) || sourceFile->isOffsetOnLine(offset + tok.charsCount, lineIndex); - } - else - { - // Has to be exactly on the specified line - return sourceFile->isOffsetOnLine(offset, lineIndex); - } -} - -SlangResult DocMarkupExtractor::_findMarkup(const FindInfo& info, Location location, FoundMarkup& out) -{ - out.reset(); - - const auto& toks = info.tokenList->m_tokens; - const Index tokIndex = info.tokenIndex; - - // The starting token index - Index startIndex = _findStartIndex(info, location); - if (startIndex <= 0) - { - return SLANG_E_NOT_FOUND; - } - - SourceView* sourceView = info.sourceView; - SourceFile* sourceFile = sourceView->getSourceFile(); - - // Let's lookup the line index where this occurred - const int startOffset = sourceView->getRange().getOffset(toks[startIndex - 1].loc); - - // The line index that the markoff starts from - Index lineIndex = sourceFile->calcLineIndexFromOffset(startOffset); - if (lineIndex < 0) - { - return SLANG_E_NOT_FOUND; - } - - const Index searchDirection = isBefore(location) ? -1 : 1; - - // Get the type and flags - const MarkupType type = findMarkupType(toks[startIndex]); - const MarkupFlags flags = getFlags(type); - - const MarkupFlag::Enum requiredFlag = isBefore(location) ? MarkupFlag::Before : MarkupFlag::After; - if ((flags & requiredFlag) == 0) - { - return SLANG_E_NOT_FOUND; - } - -#if 0 - // The token still isn't accepted, unless it's on the expected line - if (_isTokenOnLineIndex(info.sourceView, type, toks[startIndex], expectedLineIndex)) - { - return SLANG_E_NOT_FOUND; - } -#endif - - Index endIndex = startIndex; - - // If it's multiline, so look for the end index - if (flags & MarkupFlag::IsMultiToken) - { - Index expectedLineIndex = lineIndex; - - // TODO(JS): - // We should probably do the work here to confirm indentation - but that - // requires knowing something about tabs, so for now we leave. - - while (true) - { - endIndex += searchDirection; - expectedLineIndex += searchDirection; - - if (endIndex < 0 || endIndex >= toks.getCount()) - { - break; - } - - // Do we find a token of the right type? - if (findMarkupType(toks[endIndex]) != type) - { - break; - } - - // Is it on the right line? - if (_isTokenOnLineIndex(info.sourceView, type, toks[startIndex], expectedLineIndex)) - { - break; - } - } - - // Fix the end index (it's the last one that worked) - endIndex -= searchDirection; - } - - // Put start < end order - if (endIndex < startIndex) - { - Swap(endIndex, startIndex); - } - // The range excludes end so increase - endIndex++; - - // Okay we've found the markup - out.type = type; - out.location = location; - out.range = IndexRange{ startIndex, endIndex }; - - SLANG_ASSERT(out.range.getCount() > 0); - - return SLANG_OK; -} - -SlangResult DocMarkupExtractor::_findFirstMarkup(const FindInfo& info, const Location* locs, Index locCount, FoundMarkup& out, Index& outIndex) -{ - Index i = 0; - for (; i < locCount; ++i) - { - SlangResult res = _findMarkup(info, locs[i], out); - if (SLANG_SUCCEEDED(res) || (SLANG_FAILED(res) && res != SLANG_E_NOT_FOUND)) - { - outIndex = i; - return res; - } - } - return SLANG_E_NOT_FOUND; -} - -SlangResult DocMarkupExtractor::_findMarkup(const FindInfo& info, const Location* locs, Index locCount, FoundMarkup& out) -{ - Index foundIndex; - SLANG_RETURN_ON_FAIL(_findFirstMarkup(info, locs, locCount, out, foundIndex)); - - // Lets see if the remaining ones match - { - FoundMarkup otherMarkup; - for (Index i = foundIndex + 1; i < locCount; ++i) - { - SlangResult res = _findMarkup(info, locs[i], otherMarkup); - if (SLANG_SUCCEEDED(res)) - { - // TODO(JS): Warning found markup in another location - } - } - } - - return SLANG_OK; -} - -/* static */DocMarkupExtractor::SearchStyle DocMarkupExtractor::getSearchStyle(Decl* decl) -{ - if (auto enumCaseDecl = as(decl)) - { - return SearchStyle::EnumCase; - } - if (auto paramDecl = as(decl)) - { - return SearchStyle::Param; - } - else if (auto callableDecl = as(decl)) - { - return SearchStyle::Function; - } - else if (as(decl) || as(decl) || as(decl)) - { - return SearchStyle::Variable; - } - else if (auto genericDecl = as(decl)) - { - return getSearchStyle(genericDecl->inner); - } - else if (as(decl) || as(decl)) - { - return SearchStyle::GenericParam; - } - else - { - // If can't determine just allow before - return SearchStyle::Before; - } -} - -SlangResult DocMarkupExtractor::_findMarkup(const FindInfo& info, SearchStyle searchStyle, FoundMarkup& out) -{ - switch (searchStyle) - { - default: - case SearchStyle::None: - { - return SLANG_E_NOT_FOUND; - } - case SearchStyle::EnumCase: - { - Location locs[] = { Location::Before, Location::AfterEnumCase }; - return _findMarkup(info, locs, SLANG_COUNT_OF(locs), out); - } - case SearchStyle::Param: - { - Location locs[] = { Location::Before, Location::AfterParam }; - return _findMarkup(info, locs, SLANG_COUNT_OF(locs), out); - } - case SearchStyle::Before: - { - return _findMarkup(info, Location::Before, out); - } - case SearchStyle::Function: - { - return _findMarkup(info, Location::Before, out); - } - case SearchStyle::Variable: - { - Location locs[] = { Location::Before, Location::AfterSemicolon }; - return _findMarkup(info, locs, SLANG_COUNT_OF(locs), out); - } - case SearchStyle::GenericParam: - { - Location locs[] = { Location::Before, Location::AfterGenericParam }; - return _findMarkup(info, locs, SLANG_COUNT_OF(locs), out); - } - } -} - - -static void _calcLineVisibility(SourceView* sourceView, const TokenList& toks, List& outLineVisibility) -{ - SourceFile* sourceFile = sourceView->getSourceFile(); - const auto& lineOffsets = sourceFile->getLineBreakOffsets(); - - outLineVisibility.setCount(lineOffsets.getCount() + 1); - - MarkupVisibility lastVisibility = MarkupVisibility::Public; - Index lastLine = 0; - - for (const auto& tok : toks) - { - if (tok.type == TokenType::LineComment) - { - UnownedStringSlice contents = tok.getContent(); - - MarkupVisibility newVisibility = lastVisibility; - - // Distinct from other markup - if (contents.startsWith(toSlice("//@"))) - { - UnownedStringSlice access = contents.tail(3).trim(); - if (access == "hidden:" || access == "private:") - { - newVisibility = MarkupVisibility::Hidden; - } - else if (access == "internal:") - { - newVisibility = MarkupVisibility::Internal; - } - else if (access == "public:") - { - newVisibility = MarkupVisibility::Public; - } - } - - if (newVisibility != lastVisibility) - { - // Work up the line it's on - const int offset = sourceView->getRange().getOffset(tok.loc); - Index line = sourceFile->calcLineIndexFromOffset(offset); - - // Fill in the span - for (Index i = lastLine; i < line; ++i) - { - outLineVisibility[i] = lastVisibility; - } - - // Record the new access and where we are up to - lastLine = line; - lastVisibility = newVisibility; - } - } - } - - // Fill in the remaining - for (Index i = lastLine; i < outLineVisibility.getCount(); ++ i) - { - outLineVisibility[i] = lastVisibility; - } -} - - -SlangResult DocMarkupExtractor::extract(const SearchItemInput* inputs, Index inputCount, SourceManager* sourceManager, DiagnosticSink* sink, List& outViews, List& out) -{ - struct Entry - { - typedef Entry ThisType; - - Index viewIndex; ///< The view/file index this loc is found in - SourceLoc::RawValue locOrOffset; ///< Can be a loc or an offset into the file - - SearchStyle searchStyle; ///< The search style when looking for an item - Index inputIndex; ///< The index to this item in the input - }; - - List entries; - - { - entries.setCount(inputCount); - for (Index i = 0; i < inputCount; ++i) - { - const auto& input = inputs[i]; - Entry& entry = entries[i]; - entry.inputIndex = i; - entry.viewIndex = -1; //< We don't know what file/view it's in - entry.locOrOffset = input.sourceLoc.getRaw(); - entry.searchStyle = input.searchStyle; - } - } - - // Sort them into loc order - entries.sort([](const Entry& a, const Entry& b) -> bool { return a.locOrOffset < b.locOrOffset; }); - - { - SourceView* sourceView = nullptr; - Index viewIndex = -1; - - for (auto& entry : entries) - { - const SourceLoc loc = SourceLoc::fromRaw(entry.locOrOffset); - - if (sourceView == nullptr || !sourceView->getRange().contains(loc)) - { - // Find the new view - sourceView = sourceManager->findSourceView(loc); - SLANG_ASSERT(sourceView); - - // We want only one view per SourceFile - SourceFile* sourceFile = sourceView->getSourceFile(); - - // NOTE! The view found might be different than sourceView. - viewIndex = outViews.findFirstIndex([&](SourceView* currentView) -> bool { return currentView->getSourceFile() == sourceFile; }); - - if (viewIndex < 0) - { - viewIndex = outViews.getCount(); - outViews.add(sourceView); - } - } - - SLANG_ASSERT(viewIndex >= 0); - SLANG_ASSERT(sourceView && sourceView->getRange().contains(loc)); - - // Set the file index - entry.viewIndex = viewIndex; - // Set as the offset within the file - entry.locOrOffset = sourceView->getRange().getOffset(loc); - } - - // Sort into view/file and then offset order - entries.sort([](const Entry& a, const Entry& b) -> bool { return (a.viewIndex < b.viewIndex) || ((a.viewIndex == b.viewIndex) && a.locOrOffset < b.locOrOffset); }); - } - - { - TokenList tokens; - List lineVisibility; - - MemoryArena memoryArena(4096); - - RootNamePool rootNamePool; - NamePool namePool; - namePool.setRootNamePool(&rootNamePool); - - Index viewIndex = -1; - SourceView* sourceView = nullptr; - - const Int entryCount = entries.getCount(); - - out.setCount(entryCount); - - for (Index i = 0; i < entryCount; ++i) - { - const auto& entry = entries[i]; - auto& dst = out[i]; - - dst.viewIndex = -1; - dst.inputIndex = entry.inputIndex; - dst.visibilty = MarkupVisibility::Public; - - // If there isn't a mechanism to search with, just move on - if (entry.searchStyle == SearchStyle::None) - { - continue; - } - - if (viewIndex != entry.viewIndex) - { - viewIndex = entry.viewIndex; - sourceView = outViews[viewIndex]; - - // Make all memory free again - memoryArena.reset(); - - // Run the lexer - Lexer lexer; - lexer.initialize(sourceView, sink, &namePool, &memoryArena); - - // Lex everything - tokens = lexer.lexAllMarkupTokens(); - - // Let's work out the access - - _calcLineVisibility(sourceView, tokens, lineVisibility); - } - - dst.viewIndex = viewIndex; - - // Get the offset within the source file - const uint32_t offset = entry.locOrOffset; - - // We need to get the loc in the source views space, so we look up appropriately in the list of tokens (which uses the views loc range) - const SourceLoc loc = sourceView->getRange().getSourceLocFromOffset(offset); - - // Work out the line number - SourceFile* sourceFile = sourceView->getSourceFile(); - const Index lineIndex = sourceFile->calcLineIndexFromOffset(int(offset)); - - dst.visibilty = lineVisibility[lineIndex]; - - // Okay, lets find the token index with a binary chop - Index tokenIndex = _findTokenIndex(loc, tokens.m_tokens.getBuffer(), tokens.m_tokens.getCount()); - if (tokenIndex >= 0 && lineIndex >= 0) - { - FindInfo findInfo; - findInfo.tokenIndex = tokenIndex; - findInfo.lineIndex = lineIndex; - findInfo.tokenList = &tokens; - findInfo.sourceView = sourceView; - - // Okay let's see if we extract some documentation then for this. - FoundMarkup foundMarkup; - SlangResult res = _findMarkup(findInfo, entry.searchStyle, foundMarkup); - - if (SLANG_SUCCEEDED(res)) - { - // We need to extract - StringBuilder buf; - SLANG_RETURN_ON_FAIL(_extractMarkup(findInfo, foundMarkup, buf)); - - // Save the extracted text in the output - dst.text = buf; - - } - else if (res != SLANG_E_NOT_FOUND) - { - return res; - } - } - } - } - - return SLANG_OK; -} - -static void _addDeclRec(Decl* decl, List& outDecls) -{ - if (decl == nullptr) - { - return; - } - - // If we don't have a loc, we have no way of locating documentation. - if (decl->loc.isValid() || decl->nameAndLoc.loc.isValid()) - { - outDecls.add(decl); - } - else - { - SLANG_ASSERT(!"Decl without a location!"); - } - - if (GenericDecl* genericDecl = as(decl)) - { - _addDeclRec(genericDecl->inner, outDecls); - } - - if (ContainerDecl* containerDecl = as(decl)) - { - // Add the container - which could be a class, struct, enum, namespace, extension, generic etc. - // Now add what the container contains - for (Decl* childDecl : containerDecl->members) - { - _addDeclRec(childDecl, outDecls); - } - } -} - -/* static */void DocMarkupExtractor::findDecls(ModuleDecl* moduleDecl, List& outDecls) -{ - for (Decl* decl : moduleDecl->members) - { - _addDeclRec(decl, outDecls); - } -} - -SlangResult DocMarkupExtractor::extract(ModuleDecl* moduleDecl, SourceManager* sourceManager, DiagnosticSink* sink, DocMarkup* outDoc) -{ - List decls; - findDecls(moduleDecl, decls); - - const Index declsCount = decls.getCount(); - - List inputItems; - List outItems; - - { - inputItems.setCount(declsCount); - - for (Index i = 0; i < declsCount; ++i) - { - Decl* decl = decls[i]; - auto& item = inputItems[i]; - - item.sourceLoc = decl->loc.isValid() ? decl->loc : decl->nameAndLoc.loc; - // Has to be valid to be lookupable - SLANG_ASSERT(item.sourceLoc.isValid()); - - item.searchStyle = getSearchStyle(decl); - } - - DocMarkupExtractor extractor; - - List views; - SLANG_RETURN_ON_FAIL(extractor.extract(inputItems.getBuffer(), declsCount, sourceManager, sink, views, outItems)); - } - - // Set back - for (Index i = 0; i < declsCount; ++i) - { - const auto& outputItem = outItems[i]; - const auto& inputItem = inputItems[outputItem.inputIndex]; - - // If we don't know how to search add to the output - if (inputItem.searchStyle != SearchStyle::None) - { - Decl* decl = decls[outputItem.inputIndex]; - - // Add to the documentation - DocMarkup::Entry& docEntry = outDoc->addEntry(decl); - docEntry.m_markup = outputItem.text; - docEntry.m_visibility = outputItem.visibilty; - } - } - - return SLANG_OK; -} - -} // namespace Slang diff --git a/source/slang/slang-doc-extractor.h b/source/slang/slang-doc-extractor.h deleted file mode 100644 index 7a33b390a..000000000 --- a/source/slang/slang-doc-extractor.h +++ /dev/null @@ -1,219 +0,0 @@ -// slang-doc.h -#ifndef SLANG_DOC_EXTRACTOR_H -#define SLANG_DOC_EXTRACTOR_H - -#include "../core/slang-basic.h" -#include "slang-ast-all.h" - -namespace Slang { - -enum class MarkupVisibility : uint8_t -{ - Public, ///< Always available - Internal, ///< Can be available in more verbose 'internal' documentation - Hidden, ///< Not generally available -}; - -/* Holds the documentation markup that is associated with each node (typically a decl) from a module */ -class DocMarkup : public RefObject -{ -public: - struct Entry - { - NodeBase* m_node; ///< The node this documentation is associated with - String m_markup; ///< The raw contents of of markup associated with the decoration - MarkupVisibility m_visibility = MarkupVisibility::Public; ///< How visible this decl is - }; - - /// Adds an entry, returns the reference to pre-existing node if there is one - Entry& addEntry(NodeBase* base); - /// Gets an entry for a node. Returns nullptr if there is no markup. - Entry* getEntry(NodeBase* base); - - /// Get list of all of the entries in source order - const List& getEntries() const { return m_entries; } - -protected: - - /// Map from AST nodes to documentation entries - Dictionary m_entryMap; - /// All of the documentation entries in source order - List m_entries; -}; - -// --------------------------------------------------------------------------- -SLANG_INLINE DocMarkup::Entry& DocMarkup::addEntry(NodeBase* base) -{ - const Index count = m_entries.getCount(); - const Index index = m_entryMap.GetOrAddValue(base, count); - - if (index == count) - { - Entry entry; - entry.m_node = base; - m_entries.add(entry); - } - return m_entries[index]; -} - -// --------------------------------------------------------------------------- -SLANG_INLINE DocMarkup::Entry* DocMarkup::getEntry(NodeBase* base) -{ - Index* indexPtr = m_entryMap.TryGetValue(base); - return (indexPtr) ? &m_entries[*indexPtr] : nullptr; -} - -/* Extracts 'markup' from comments in Slang source core. The comments are extracted and associated in declarations. The association -is held in DocMarkup type. The comment style follows the doxygen style */ -class DocMarkupExtractor -{ -public: - - typedef uint32_t MarkupFlags; - struct MarkupFlag - { - enum Enum : MarkupFlags - { - Before = 0x1, - After = 0x2, - IsMultiToken = 0x4, ///< Can use more than one token - IsBlock = 0x8, ///< - }; - }; - - // NOTE! Don't change order without fixing isBefore and isAfter - enum class MarkupType - { - None, - - BlockBefore, /// /** */ or /*! */. - LineBangBefore, /// //! Can be multiple lines - LineSlashBefore, /// /// Can be multiple lines - - BlockAfter, /// /*!< */ or /**< */ - LineBangAfter, /// //!< Can be multiple lines - LineSlashAfter, /// ///< Can be multiple lines - }; - - static bool isBefore(MarkupType type) { return Index(type) >= Index(MarkupType::BlockBefore) && Index(type) <= Index(MarkupType::LineSlashBefore); } - static bool isAfter(MarkupType type) { return Index(type) >= Index(MarkupType::BlockAfter); } - - struct IndexRange - { - SLANG_FORCE_INLINE Index getCount() const { return end - start; } - - Index start; - Index end; - }; - - enum class Location - { - None, ///< No defined location - Before, - AfterParam, ///< Can have trailing , or ) - AfterSemicolon, ///< Can have a trailing ; - AfterEnumCase, ///< Can have a , or before } - AfterGenericParam, ///< Can have trailing , or > - }; - - static bool isAfter(Location location) { return Index(location) >= Index(Location::AfterParam); } - static bool isBefore(Location location) { return location == Location::Before; } - - struct FoundMarkup - { - void reset() - { - location = Location::None; - type = MarkupType::None; - range = IndexRange{ 0, 0 }; - } - - Location location = Location::None; - MarkupType type = MarkupType::None; - IndexRange range; - }; - - enum SearchStyle - { - None, ///< Cannot be searched for - EnumCase, ///< An enum case - Param, ///< A parameter in a function/method - Variable, ///< A variable-like declaration - Before, ///< Only allows before - Function, ///< Function/method - GenericParam, ///< Generic parameter - }; - - /// An input search item - struct SearchItemInput - { - SourceLoc sourceLoc; - SearchStyle searchStyle; ///< The search style when looking for an item - }; - - /// The items will be in source order - struct SearchItemOutput - { - Index viewIndex; ///< Index into the array of views on the output - Index inputIndex; ///< The index to this item in the input - String text; ///< The found text - MarkupVisibility visibilty; ///< Visibility of the item - }; - - struct FindInfo - { - SourceView* sourceView; ///< The source view the tokens were generated from - TokenList* tokenList; ///< The token list - Index tokenIndex; ///< The token index location (where searches start from) - Index lineIndex; ///< The line number for the decl - }; - - /// Extracts documentation from the nodes held in the module using the source manager. Found documentation is placed - /// in outMarkup - static SlangResult extract(ModuleDecl* moduleDecl, SourceManager* sourceManager, DiagnosticSink* sink, DocMarkup* outMarkup); - - /// Extracts 'markup' doc information for the specified input items - /// The output is placed in out - with the items now in the source order *not* the order of the input items - /// The inputIndex on the output holds the input item index - /// The outViews holds the views specified in viewIndex in the output, which may be useful for determining where the documentation was placed in source - SlangResult extract(const SearchItemInput* inputItems, Index inputCount, SourceManager* sourceManager, DiagnosticSink* sink, List& outViews, List& out); - - /// Given a module finds all the decls, and places in outDecls - static void findDecls(ModuleDecl* moduleDecl, List& outDecls); - - /// Given a decl determines the search style that is appropriate. Returns None if can't determine a suitable style - static SearchStyle getSearchStyle(Decl* decl); - - static MarkupFlags getFlags(MarkupType type); - static MarkupType findMarkupType(const Token& tok); - static UnownedStringSlice removeStart(MarkupType type, const UnownedStringSlice& comment); - -protected: - /// returns SLANG_E_NOT_FOUND if not found, SLANG_OK on success else an error - SlangResult _findMarkup(const FindInfo& info, Location location, FoundMarkup& out); - - /// Locations are processed in order, and the first successful used. If found in another location will issue a warning. - /// returns SLANG_E_NOT_FOUND if not found, SLANG_OK on success else an error - SlangResult _findFirstMarkup(const FindInfo& info, const Location* locs, Index locCount, FoundMarkup& out, Index& outIndex); - - SlangResult _findMarkup(const FindInfo& info, const Location* locs, Index locCount, FoundMarkup& out); - - /// Given the decl, the token stream, and the decls tokenIndex, try to find some associated markup - SlangResult _findMarkup(const FindInfo& info, SearchStyle searchStyle, FoundMarkup& out); - - /// Given a found markup location extracts the contents of the tokens into out - SlangResult _extractMarkup(const FindInfo& info, const FoundMarkup& foundMarkup, StringBuilder& out); - - /// Given a location, try to find the first token index that could potentially be markup - /// Will return -1 if not found - Index _findStartIndex(const FindInfo& info, Location location); - - /// True if the tok is 'on' lineIndex. Interpretation of 'on' depends on the markup type. - static bool _isTokenOnLineIndex(SourceView* sourceView, MarkupType type, const Token& tok, Index lineIndex); - - DiagnosticSink* m_sink; -}; - -} // namespace Slang - -#endif diff --git a/source/slang/slang-doc-markdown-writer.cpp b/source/slang/slang-doc-markdown-writer.cpp index 2a1226bb7..4d8afd763 100644 --- a/source/slang/slang-doc-markdown-writer.cpp +++ b/source/slang/slang-doc-markdown-writer.cpp @@ -128,7 +128,7 @@ String DocMarkdownWriter::_getName(InheritanceDecl* decl) return buf.ProduceString(); } -DocMarkdownWriter::NameAndText DocMarkdownWriter::_getNameAndText(DocMarkup::Entry* entry, Decl* decl) +DocMarkdownWriter::NameAndText DocMarkdownWriter::_getNameAndText(ASTMarkup::Entry* entry, Decl* decl) { NameAndText nameAndText; @@ -146,7 +146,7 @@ DocMarkdownWriter::NameAndText DocMarkdownWriter::_getNameAndText(DocMarkup::Ent DocMarkdownWriter::NameAndText DocMarkdownWriter::_getNameAndText(Decl* decl) { - DocMarkup::Entry* entry = m_markup->getEntry(decl); + ASTMarkup::Entry* entry = m_markup->getEntry(decl); return _getNameAndText(entry, decl); } @@ -242,7 +242,7 @@ void DocMarkdownWriter::_appendCommaList(const List& strings, char wrapC } } -void DocMarkdownWriter::writeVar(const DocMarkup::Entry& entry, VarDecl* varDecl) +void DocMarkdownWriter::writeVar(const ASTMarkup::Entry& entry, VarDecl* varDecl) { writePreamble(entry); auto& out = m_builder; @@ -680,7 +680,7 @@ static bool _isFirstOverridden(Decl* decl) return false; } -void DocMarkdownWriter::writeCallableOverridable(const DocMarkup::Entry& entry, CallableDecl* callableDecl) +void DocMarkdownWriter::writeCallableOverridable(const ASTMarkup::Entry& entry, CallableDecl* callableDecl) { auto& out = m_builder; @@ -843,7 +843,7 @@ void DocMarkdownWriter::writeCallableOverridable(const DocMarkup::Entry& entry, } } -void DocMarkdownWriter::writeEnum(const DocMarkup::Entry& entry, EnumDecl* enumDecl) +void DocMarkdownWriter::writeEnum(const ASTMarkup::Entry& entry, EnumDecl* enumDecl) { writePreamble(entry); @@ -963,7 +963,7 @@ void DocMarkdownWriter::_appendAggTypeName(AggTypeDeclBase* aggTypeDecl) } } -void DocMarkdownWriter::writeAggType(const DocMarkup::Entry& entry, AggTypeDeclBase* aggTypeDecl) +void DocMarkdownWriter::writeAggType(const ASTMarkup::Entry& entry, AggTypeDeclBase* aggTypeDecl) { writePreamble(entry); @@ -1005,7 +1005,7 @@ void DocMarkdownWriter::writeAggType(const DocMarkup::Entry& entry, AggTypeDeclB out << "* _" << assocTypeDecl->getName()->text << "_ "; // Look up markup - DocMarkup::Entry* assocTypeDeclEntry = m_markup->getEntry(assocTypeDecl); + ASTMarkup::Entry* assocTypeDeclEntry = m_markup->getEntry(assocTypeDecl); if (assocTypeDeclEntry) { _appendAsSingleLine(assocTypeDeclEntry->m_markup.getUnownedSlice(), out); @@ -1086,7 +1086,7 @@ void DocMarkdownWriter::writeAggType(const DocMarkup::Entry& entry, AggTypeDeclB } } -void DocMarkdownWriter::writePreamble(const DocMarkup::Entry& entry) +void DocMarkdownWriter::writePreamble(const ASTMarkup::Entry& entry) { SLANG_UNUSED(entry); auto& out = m_builder; @@ -1097,7 +1097,7 @@ void DocMarkdownWriter::writePreamble(const DocMarkup::Entry& entry) out << toSlice("\n"); } -void DocMarkdownWriter::writeDescription(const DocMarkup::Entry& entry) +void DocMarkdownWriter::writeDescription(const ASTMarkup::Entry& entry) { auto& out = m_builder; @@ -1117,7 +1117,7 @@ void DocMarkdownWriter::writeDescription(const DocMarkup::Entry& entry) } } -void DocMarkdownWriter::writeDecl(const DocMarkup::Entry& entry, Decl* decl) +void DocMarkdownWriter::writeDecl(const ASTMarkup::Entry& entry, Decl* decl) { // Skip these they will be output as part of their respective 'containers' if (as(decl) || as(decl) || as(decl) || as(decl)) @@ -1161,7 +1161,7 @@ bool DocMarkdownWriter::isVisible(const Name* name) return name == nullptr || !name->text.startsWith(toSlice("__")); } -bool DocMarkdownWriter::isVisible(const DocMarkup::Entry& entry) +bool DocMarkdownWriter::isVisible(const ASTMarkup::Entry& entry) { // For now if it's not public it's not visible if (entry.m_visibility != MarkupVisibility::Public) diff --git a/source/slang/slang-doc-markdown-writer.h b/source/slang/slang-doc-markdown-writer.h index 7a6c0a7e0..ff4c50759 100644 --- a/source/slang/slang-doc-markdown-writer.h +++ b/source/slang/slang-doc-markdown-writer.h @@ -2,7 +2,8 @@ #ifndef SLANG_DOC_MARKDOWN_WRITER_H #define SLANG_DOC_MARKDOWN_WRITER_H -#include "slang-doc-extractor.h" +#include "slang-doc-ast.h" + #include "slang-ast-print.h" #include "slang-compiler.h" @@ -50,19 +51,19 @@ struct DocMarkdownWriter void writeAll(); /// This will write information about *all* of the overridden versions of a function/method - void writeCallableOverridable(const DocMarkup::Entry& entry, CallableDecl* callable); + void writeCallableOverridable(const ASTMarkup::Entry& entry, CallableDecl* callable); - void writeEnum(const DocMarkup::Entry& entry, EnumDecl* enumDecl); - void writeAggType(const DocMarkup::Entry& entry, AggTypeDeclBase* aggTypeDecl); - void writeDecl(const DocMarkup::Entry& entry, Decl* decl); - void writeVar(const DocMarkup::Entry& entry, VarDecl* varDecl); + void writeEnum(const ASTMarkup::Entry& entry, EnumDecl* enumDecl); + void writeAggType(const ASTMarkup::Entry& entry, AggTypeDeclBase* aggTypeDecl); + void writeDecl(const ASTMarkup::Entry& entry, Decl* decl); + void writeVar(const ASTMarkup::Entry& entry, VarDecl* varDecl); - void writePreamble(const DocMarkup::Entry& entry); - void writeDescription(const DocMarkup::Entry& entry); + void writePreamble(const ASTMarkup::Entry& entry); + void writeDescription(const ASTMarkup::Entry& entry); void writeSignature(CallableDecl* callableDecl); - bool isVisible(const DocMarkup::Entry& entry); + bool isVisible(const ASTMarkup::Entry& entry); bool isVisible(Decl* decl); bool isVisible(const Name* name); @@ -70,7 +71,7 @@ struct DocMarkdownWriter const StringBuilder& getOutput() const { return m_builder; } /// Ctor. - DocMarkdownWriter(DocMarkup* markup, ASTBuilder* astBuilder) : + DocMarkdownWriter(ASTMarkup* markup, ASTBuilder* astBuilder) : m_markup(markup), m_astBuilder(astBuilder) { @@ -92,7 +93,7 @@ struct DocMarkdownWriter String _getName(Decl* decl); String _getName(InheritanceDecl* decl); - NameAndText _getNameAndText(DocMarkup::Entry* entry, Decl* decl); + NameAndText _getNameAndText(ASTMarkup::Entry* entry, Decl* decl); NameAndText _getNameAndText(Decl* decl); template @@ -134,7 +135,7 @@ struct DocMarkdownWriter void _appendAggTypeName(AggTypeDeclBase* aggTypeDecl); - DocMarkup* m_markup; + ASTMarkup* m_markup; ASTBuilder* m_astBuilder; StringBuilder m_builder; }; diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp index 67908841f..e0b11604c 100644 --- a/source/slang/slang.cpp +++ b/source/slang/slang.cpp @@ -31,7 +31,7 @@ #include "slang-serialize-ir.h" #include "slang-serialize-container.h" -#include "slang-doc-extractor.h" +#include "slang-doc-ast.h" #include "slang-doc-markdown-writer.h" #include "slang-check-impl.h" @@ -300,8 +300,8 @@ SlangResult Session::compileStdLib(slang::CompileStdLibFlags compileFlags) // For all the modules add their doc output to docStrings for (Module* stdlibModule : stdlibModules) { - RefPtr markup(new DocMarkup); - DocMarkupExtractor::extract(stdlibModule->getModuleDecl(), sourceManager, &sink, markup); + RefPtr markup(new ASTMarkup); + ASTMarkupUtil::extract(stdlibModule->getModuleDecl(), sourceManager, &sink, markup); DocMarkdownWriter writer(markup, astBuilder); writer.writeAll(); @@ -2150,8 +2150,8 @@ SlangResult FrontEndCompileRequest::executeActionsInner() { for (TranslationUnitRequest* translationUnit : translationUnits) { - RefPtr markup(new DocMarkup); - DocMarkupExtractor::extract(translationUnit->getModuleDecl(), getSourceManager(), getSink(), markup); + RefPtr markup(new ASTMarkup); + ASTMarkupUtil::extract(translationUnit->getModuleDecl(), getSourceManager(), getSink(), markup); // Convert to markdown DocMarkdownWriter markdownWriter(markup, astBuilder); diff --git a/tools/slang-cpp-extractor/cpp-extractor-main.cpp b/tools/slang-cpp-extractor/cpp-extractor-main.cpp index d1ec47e69..9db61f8e0 100644 --- a/tools/slang-cpp-extractor/cpp-extractor-main.cpp +++ b/tools/slang-cpp-extractor/cpp-extractor-main.cpp @@ -20,6 +20,7 @@ #include "../../source/compiler-core/slang-diagnostic-sink.h" #include "../../source/compiler-core/slang-name.h" #include "../../source/compiler-core/slang-name-convention-util.h" +#include "../../source/compiler-core/slang-doc-extractor.h" #include "node.h" #include "diagnostics.h" @@ -63,6 +64,8 @@ public: protected: + SlangResult _extractDoc(NodeTree* nodeTree); + NamePool m_namePool; Options m_options; @@ -73,6 +76,116 @@ protected: }; +// Work out an appropriate search type for a node type. +// +// TODO(JS): +// NOTE! Currently extractor doesn't extract callable types and so doesn't extract callable types parameters +static DocMarkupExtractor::SearchStyle _getSearchStyle(Node* node) +{ + typedef DocMarkupExtractor::SearchStyle SearchStyle; + + if (!node->getSourceLoc().isValid()) + { + return SearchStyle::None; + } + + switch (node->m_kind) + { + case Node::Kind::Invalid: + { + return SearchStyle::None; + } + case Node::Kind::Field: + { + return SearchStyle::Variable; + } + case Node::Kind::EnumCase: + { + return SearchStyle::EnumCase; + } + case Node::Kind::TypeDef: + { + return SearchStyle::Variable; + } + default: break; + } + + // Default is to only allow before. + return SearchStyle::Before; +} + +SlangResult App::_extractDoc(NodeTree* nodeTree) +{ + // Find all of the nodes + List nodes; + // Add the root + nodes.add(nodeTree->getRootNode()); + + // Traverse all nodes + for (Index startIndex = 0; startIndex < nodes.getCount(); ++startIndex) + { + Node* node = nodes[startIndex]; + + ScopeNode* scopeNode = as(node); + + if (scopeNode) + { + for (Node* child : scopeNode->m_children) + { + nodes.add(child); + } + } + } + + // Find out what to find + + List inputItems; + + for (Node* node : nodes) + { + auto searchStyle = _getSearchStyle(node); + + DocMarkupExtractor::SearchItemInput inputItem; + inputItem.searchStyle = searchStyle; + inputItem.sourceLoc = node->getSourceLoc(); + + inputItems.add(inputItem); + } + + List outputItems; + + List views; + + DocMarkupExtractor extractor; + + SLANG_RETURN_ON_FAIL(extractor.extract(inputItems.getBuffer(), inputItems.getCount(), m_sourceManager, m_sink, views, outputItems)); + + // Put what was extracted into the nodes + { + const Index count = inputItems.getCount(); + SLANG_ASSERT(count == outputItems.getCount() && count == nodes.getCount()); + + for (Index i = 0; i < count; ++i) + { + const auto& outputItem = outputItems[i]; + + // We need to use the index used for input, because in output they can be reordered. + const auto inputIndex = outputItem.inputIndex; + const auto& inputItem = inputItems[inputIndex]; + + if (inputItem.searchStyle != DocMarkupExtractor::SearchStyle::None && outputItem.text.getLength()) + { + Node* node = nodes[inputIndex]; + + node->m_markup = outputItem.text; + node->m_markupVisibility = outputItem.visibilty; + } + } + } + + return SLANG_OK; +} + SlangResult App::execute(const Options& options) { m_options = options; @@ -143,6 +256,11 @@ SlangResult App::execute(const Options& options) } } + if (options.m_extractDoc) + { + SLANG_RETURN_ON_FAIL(_extractDoc(&tree)); + } + // Dump out the tree if (options.m_dump) { diff --git a/tools/slang-cpp-extractor/macro-writer.cpp b/tools/slang-cpp-extractor/macro-writer.cpp index 2259f1800..c4245a646 100644 --- a/tools/slang-cpp-extractor/macro-writer.cpp +++ b/tools/slang-cpp-extractor/macro-writer.cpp @@ -214,7 +214,7 @@ SlangResult MacroWriter::calcTypeHeader(NodeTree* tree, TypeSet* typeSet, String for (Node* scopeNode : baseScopePath) { - SLANG_ASSERT(scopeNode->m_type == Node::Type::Namespace); + SLANG_ASSERT(scopeNode->m_kind == Node::Kind::Namespace); out << "namespace " << scopeNode->m_name.getContent() << " {\n"; } @@ -262,7 +262,7 @@ SlangResult MacroWriter::calcTypeHeader(NodeTree* tree, TypeSet* typeSet, String // If it's not reflected we don't output, in the enum list if (node->isReflected()) { - const char* type = (node->m_type == Node::Type::ClassType) ? "class" : "struct"; + const char* type = (node->m_kind == Node::Kind::ClassType) ? "class" : "struct"; out << type << " " << node->m_name.getContent() << ";\n"; } } diff --git a/tools/slang-cpp-extractor/node-tree.cpp b/tools/slang-cpp-extractor/node-tree.cpp index 3002a7abd..05360e0de 100644 --- a/tools/slang-cpp-extractor/node-tree.cpp +++ b/tools/slang-cpp-extractor/node-tree.cpp @@ -18,7 +18,7 @@ NodeTree::NodeTree(StringSlicePool* typePool, NamePool* namePool, IdentifierLook m_identifierLookup(identifierLookup), m_typeSetPool(StringSlicePool::Style::Empty) { - m_rootNode = new ScopeNode(Node::Type::Namespace); + m_rootNode = new ScopeNode(Node::Kind::Namespace); m_rootNode->m_reflectionType = ReflectionType::Reflected; } diff --git a/tools/slang-cpp-extractor/node.cpp b/tools/slang-cpp-extractor/node.cpp index 3b2403816..2074b7c41 100644 --- a/tools/slang-cpp-extractor/node.cpp +++ b/tools/slang-cpp-extractor/node.cpp @@ -3,6 +3,7 @@ #include "file-util.h" #include "../../source/core/slang-string-util.h" +#include "../../source/core/slang-string-escape-util.h" namespace CppExtract { @@ -10,6 +11,40 @@ namespace CppExtract { SLANG_FORCE_INLINE static void _indent(Index indentCount, StringBuilder& out) { FileUtil::indent(indentCount, out); } +void Node::dumpMarkup(int indentCount, StringBuilder& out) +{ + if (m_markup.getLength() <= 0) + { + return; + } + + List lines; + StringUtil::calcLines(m_markup.getUnownedSlice(), lines); + + // Remove empty lines from the end + while (lines.getCount()) + { + auto lastLine = lines.getLast(); + if (lastLine.trim().getLength() == 0) + { + lines.removeLast(); + continue; + } + break; + } + + if (lines.getCount() == 0) + { + return; + } + + for (auto line : lines) + { + _indent(indentCount, out); + out << "// " << line << "\n"; + } +} + ScopeNode* Node::getRootScope() { if (m_parentScope) @@ -47,7 +82,7 @@ void Node::calcAbsoluteName(StringBuilder& outName) const outName << "::"; } - if (node->m_type == Type::AnonymousNamespace) + if (node->m_kind == Kind::AnonymousNamespace) { outName << "{Anonymous}"; } @@ -119,7 +154,7 @@ void Node::calcAbsoluteName(StringBuilder& outName) const for (Node* node : scope->m_children) { EnumNode* enumNode = as(node); - if (enumNode && enumNode->m_type == Node::Type::Enum) + if (enumNode && enumNode->m_kind == Node::Kind::Enum) { Node** nodePtr = enumNode->m_childMap.TryGetValue(name); if (nodePtr) @@ -251,7 +286,7 @@ ScopeNode* ScopeNode::getAnonymousNamespace() { if (!m_anonymousNamespace) { - m_anonymousNamespace = new ScopeNode(Type::AnonymousNamespace); + m_anonymousNamespace = new ScopeNode(Kind::AnonymousNamespace); m_anonymousNamespace->m_parentScope = this; m_children.add(m_anonymousNamespace); } @@ -263,7 +298,7 @@ void ScopeNode::addChild(Node* child) { SLANG_ASSERT(child->m_parentScope == nullptr); // Can't add anonymous namespace this way - should be added via getAnonymousNamespace - SLANG_ASSERT(child->m_type != Type::AnonymousNamespace); + SLANG_ASSERT(child->m_kind != Kind::AnonymousNamespace); child->m_parentScope = this; m_children.add(child); @@ -298,15 +333,17 @@ void ScopeNode::calcScopeDepthFirst(List& outNodes) void ScopeNode::dump(int indentCount, StringBuilder& out) { + dumpMarkup(indentCount, out); + _indent(indentCount, out); - switch (m_type) + switch (m_kind) { - case Type::AnonymousNamespace: + case Kind::AnonymousNamespace: { out << "namespace {\n"; } - case Type::Namespace: + case Kind::Namespace: { if (m_name.hasContent()) { @@ -331,17 +368,66 @@ void ScopeNode::dump(int indentCount, StringBuilder& out) /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! EnumCaseNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ +/* Returns true if needs space between the tokens. +It determines this based on the locs, and if they contain something between them. +*/ +static bool _needsSpace(const Token& prevTok, const Token& tok) +{ + auto prevLoc = prevTok.getLoc(); + auto loc = tok.getLoc(); + + auto prevContent = prevTok.getContent(); + auto content = tok.getContent(); + + if (prevLoc + prevContent.getLength() == loc) + { + return false; + } + + return true; +} + + +static void _dumpTokens(const Token* toks, Index count, StringBuilder& out) +{ + if (count > 0) + { + out << toks[0].getContent(); + + for (Index i = 1; i < count; ++i) + { + const auto& prevToken = toks[i - 1]; + const auto& token = toks[i]; + + if (_needsSpace(prevToken, token)) + { + out << " "; + } + + out << token.getContent(); + } + } +} + +static void _dumpTokens(const List& toks, StringBuilder& out) +{ + _dumpTokens(toks.getBuffer(), toks.getCount(), out); +} + + void EnumCaseNode::dump(int indent, StringBuilder& out) { if (isReflected()) { + dumpMarkup(indent, out); + _indent(indent, out); out << m_name.getContent(); - if (m_value.type != TokenType::Invalid) + if (m_valueTokens.getCount()) { out << " = "; - out << m_value.getContent(); + _dumpTokens(m_valueTokens, out); } out << ",\n"; @@ -354,16 +440,13 @@ void TypeDefNode::dump(int indent, StringBuilder& out) { if (isReflected()) { + dumpMarkup(indent, out); + _indent(indent, out); out << "typedef "; - - for (auto& tok : m_targetTypeTokens) - { - out << tok.getContent() << " "; - } - - out << m_name.getContent() << ";\n"; + _dumpTokens(m_targetTypeTokens, out); + out << " " << m_name.getContent() << ";\n"; } } @@ -376,11 +459,13 @@ void EnumNode::dump(int indent, StringBuilder& out) return; } + dumpMarkup(indent, out); + _indent(indent, out); out << "enum "; - if (m_type == Type::EnumClass) + if (m_kind == Kind::EnumClass) { out << "class "; } @@ -390,9 +475,10 @@ void EnumNode::dump(int indent, StringBuilder& out) out << m_name.getContent(); } - if (m_backingToken.type != TokenType::Invalid) - { - out << " : " << m_backingToken.getContent(); + if (m_backingTokens.getCount() > 0) + { + out << " : "; + _dumpTokens(m_backingTokens, out); } out << "\n"; @@ -414,6 +500,8 @@ void FieldNode::dump(int indent, StringBuilder& out) { if (isReflected()) { + dumpMarkup(indent, out); + _indent(indent, out); out << m_fieldType << " " << m_name.getContent() << "\n"; } @@ -506,9 +594,11 @@ void ClassLikeNode::getReflectedDerivedTypes(List& out) const void ClassLikeNode::dump(int indentCount, StringBuilder& out) { + dumpMarkup(indentCount, out); + _indent(indentCount, out); - const char* typeName = (m_type == Type::StructType) ? "struct" : "class"; + const char* typeName = (m_kind == Kind::StructType) ? "struct" : "class"; out << typeName << " "; diff --git a/tools/slang-cpp-extractor/node.h b/tools/slang-cpp-extractor/node.h index c741024e4..f649f1adb 100644 --- a/tools/slang-cpp-extractor/node.h +++ b/tools/slang-cpp-extractor/node.h @@ -3,6 +3,8 @@ #include "diagnostics.h" +#include "../../source/compiler-core/slang-doc-extractor.h" + namespace CppExtract { using namespace Slang; @@ -21,7 +23,7 @@ struct ScopeNode; class Node : public RefObject { public: - enum class Type : uint8_t + enum class Kind : uint8_t { Invalid, @@ -42,29 +44,49 @@ public: CountOf, }; - enum class TypeRange + enum class KindRange { - ScopeStart = int(Type::StructType), - ScopeEnd = int(Type::AnonymousNamespace), + ScopeStart = int(Kind::StructType), + ScopeEnd = int(Kind::AnonymousNamespace), + + ClassLikeStart = int(Kind::StructType), + ClassLikeEnd = int(Kind::ClassType), - ClassLikeStart = int(Type::StructType), - ClassLikeEnd = int(Type::ClassType), + ScopeTypeStart = int(Kind::StructType), + ScopeTypeEnd = int(Kind::EnumClass), - EnumStart = int(Type::Enum), - EnumEnd = int(Type::EnumClass), + OtherTypeStart = int(Kind::TypeDef), + OtherTypeEnd = int(Kind::TypeDef), + + EnumStart = int(Kind::Enum), + EnumEnd = int(Kind::EnumClass), }; - static bool isScopeType(Type type) { return int(type) >= int(TypeRange::ScopeStart) && int(type) <= int(TypeRange::ScopeEnd); } - static bool isClassLikeType(Type type) { return int(type) >= int(TypeRange::ClassLikeStart) && int(type) <= int(TypeRange::ClassLikeEnd); } - static bool isEnumLikeType(Type type) { return int(type) >= int(TypeRange::EnumStart) && int(type) <= int(TypeRange::EnumEnd); } - static bool canAcceptTypes(Type type) + /// Returns true if kind can cast to this type + /// Used for implementing as casting + static bool isOfKind(Kind type) { return true; } + + static bool isKindScope(Kind kind) { return int(kind) >= int(KindRange::ScopeStart) && int(kind) <= int(KindRange::ScopeEnd); } + static bool isKindClassLike(Kind kind) { return int(kind) >= int(KindRange::ClassLikeStart) && int(kind) <= int(KindRange::ClassLikeEnd); } + static bool isKindEnumLike(Kind kind) { return int(kind) >= int(KindRange::EnumStart) && int(kind) <= int(KindRange::EnumEnd); } + + /// It a type, but doesn't have a scope + static bool isKindOtherType(Kind kind) { return int(kind) >= int(KindRange::OtherTypeStart) && int(kind) <= int(KindRange::OtherTypeEnd); } + /// Is a type and has a scope + static bool isKindScopeType(Kind kind) { return int(kind) >= int(KindRange::ScopeTypeStart) && int(kind) <= int(KindRange::ScopeTypeEnd); } + + /// True if the kind is any type + static bool isKindType(Kind kind) { return isKindOtherType(kind) || isKindScopeType(kind); } + + /// True if the kind can accept contained types + static bool canKindContainTypes(Kind type) { switch (type) { - case Type::StructType: - case Type::ClassType: - case Type::Namespace: - case Type::AnonymousNamespace: + case Kind::StructType: + case Kind::ClassType: + case Kind::Namespace: + case Kind::AnonymousNamespace: { return true; } @@ -73,9 +95,11 @@ public: return false; } - static bool isType(Type type) { return true; } + bool isClassLike() const { return isKindClassLike(m_kind); } - bool isClassLike() const { return isClassLikeType(m_type); } + /// These are useful for the filter + static bool isClassLikeAndReflected(Node* node) { return node->isClassLike() && node->isReflected(); } + static bool isClassLike(Node* node) { return isKindClassLike(node->m_kind); } virtual void dump(int indent, StringBuilder& out) = 0; @@ -94,13 +118,12 @@ public: /// True if reflected bool isReflected() const { return m_reflectionType == ReflectionType::Reflected; } + SourceLoc getSourceLoc() const { return m_name.getLoc(); } + ScopeNode* getRootScope(); typedef bool(*Filter)(Node* node); - - static bool isClassLikeAndReflected(Node* node) { return node->isClassLike() && node->isReflected(); } - static bool isClassLike(Node* node) { return isClassLikeType(node->m_type); } - + template static void filter(Filter filter, List& io) { const Node* _isNodeDerived = (T*)nullptr; SLANG_UNUSED(_isNodeDerived); filterImpl(filter, reinterpret_cast&>(io)); } @@ -122,15 +145,21 @@ public: static void splitPath(const UnownedStringSlice& slice, List& outSplitPath); - Node(Type type) : - m_type(type), + /// If markup is specified dump it + void dumpMarkup(int indent, StringBuilder& out); + + Node(Kind type) : + m_kind(type), m_parentScope(nullptr), m_reflectionType(ReflectionType::NotReflected) { } - Type m_type; ///< The type of node this is - ReflectionType m_reflectionType; /// Classes can be traversed, but not reflected. To be reflected they have to contain the marker + Kind m_kind; ///< The kind of node this is + ReflectionType m_reflectionType; ///< Classes can be traversed, but not reflected. To be reflected they have to contain the marker + + MarkupVisibility m_markupVisibility = MarkupVisibility::Public; ///< The visibility of the markup + String m_markup; ///< Documentation associated with this node Token m_name; ///< The name of this scope/type @@ -141,15 +170,15 @@ struct ScopeNode : public Node { typedef Node Super; - static bool isType(Type type) { return isScopeType(type); } + static bool isOfKind(Kind kind) { return isKindScope(kind); } virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; virtual void calcScopeDepthFirst(List& outNodes) SLANG_OVERRIDE; /// True if can accept fields (class like types can) - bool acceptsFields() const { return isClassLike(); } + bool canContainFields() const { return isClassLike(); } /// True if the scope can accept types - bool acceptsTypes() const { return canAcceptTypes(m_type); } + bool canContainTypes() const { return canKindContainTypes(m_kind); } /// Gets the reflection for any contained types ReflectionType getContainedReflectionType() const { return m_reflectionType == ReflectionType::NotReflected ? ReflectionType::NotReflected : m_reflectionOverride; } @@ -163,8 +192,8 @@ struct ScopeNode : public Node /// Gets the anonymous namespace associated with this scope ScopeNode* getAnonymousNamespace(); - ScopeNode(Type type) : - Super(type), + ScopeNode(Kind kind) : + Super(kind), m_reflectionOverride(ReflectionType::Reflected), m_anonymousNamespace(nullptr) { @@ -187,12 +216,12 @@ struct FieldNode : public Node { typedef Node Super; - static bool isType(Type type) { return type == Type::Field; } + static bool isOfKind(Kind type) { return type == Kind::Field; } virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; FieldNode() : - Super(Type::Field) + Super(Kind::Field) { } @@ -205,7 +234,7 @@ struct ClassLikeNode : public ScopeNode { typedef ScopeNode Super; - static bool isType(Type type) { return isClassLikeType(type); } + static bool isOfKind(Kind kind) { return isKindClassLike(kind); } /// Add a node that is derived from this void addDerived(ClassLikeNode* derived); @@ -231,13 +260,13 @@ struct ClassLikeNode : public ScopeNode // Node Impl virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; - ClassLikeNode(Type type) : - Super(type), + ClassLikeNode(Kind kind) : + Super(kind), m_origin(nullptr), m_typeSet(nullptr), m_superNode(nullptr) { - SLANG_ASSERT(type == Type::ClassType || type == Type::StructType); + SLANG_ASSERT(kind == Kind::ClassType || kind == Kind::StructType); } SourceOrigin* m_origin; ///< Defines where this was uniquely defined. @@ -256,43 +285,44 @@ struct EnumCaseNode : public Node { typedef Node Super; - static bool isType(Type type) { return type == Type::EnumCase; } + static bool isOfKind(Kind kind) { return kind == Kind::EnumCase; } virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; EnumCaseNode(): - Super(Type::EnumCase) + Super(Kind::EnumCase) { } - Token m_value; ///< If not defined will be invalid + // Tokens that make up the value. If not defined will be empty + List m_valueTokens; }; struct EnumNode : public ScopeNode { typedef ScopeNode Super; - static bool isType(Type type) { return isEnumLikeType(type); } + static bool isOfKind(Kind kind) { return isKindEnumLike(kind); } virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; - EnumNode(Type type): - Super(type) + EnumNode(Kind kind): + Super(kind) { - SLANG_ASSERT(isEnumLikeType(type)); + SLANG_ASSERT(isKindEnumLike(kind)); } - Token m_backingToken; + List m_backingTokens; }; struct TypeDefNode : public Node { typedef Node Super; - static bool isType(Type type) { return type == Type::TypeDef; } + static bool isOfKind(Kind kind) { return kind == Kind::TypeDef; } virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; TypeDefNode(): - Super(Type::TypeDef) + Super(Kind::TypeDef) { } @@ -300,7 +330,7 @@ struct TypeDefNode : public Node }; template -T* as(Node* node) { return (node && T::isType(node->m_type)) ? static_cast(node) : nullptr; } +T* as(Node* node) { return (node && T::isOfKind(node->m_kind)) ? static_cast(node) : nullptr; } } // CppExtract diff --git a/tools/slang-cpp-extractor/options.h b/tools/slang-cpp-extractor/options.h index 14ef9ecfc..186ac2fbb 100644 --- a/tools/slang-cpp-extractor/options.h +++ b/tools/slang-cpp-extractor/options.h @@ -25,6 +25,7 @@ struct Options bool m_defs = false; ///< If set will output a '-defs.h' file for each of the input files, that corresponds to previous defs files (although doesn't have fields/RAW) bool m_dump = false; ///< If true will dump to stderr the types/fields and hierarchy it extracted bool m_runUnitTests = false; ///< If true will run internal unit tests + bool m_extractDoc = true; ///< If set will try to extract documentation associated with nodes bool m_outputFields = false; ///< When dumping macros also dump field definitions diff --git a/tools/slang-cpp-extractor/parser.cpp b/tools/slang-cpp-extractor/parser.cpp index 97c156f2f..29bc517f5 100644 --- a/tools/slang-cpp-extractor/parser.cpp +++ b/tools/slang-cpp-extractor/parser.cpp @@ -11,7 +11,7 @@ namespace CppExtract { using namespace Slang; // If fails then we need more bits to identify types -SLANG_COMPILE_TIME_ASSERT(int(Node::Type::CountOf) <= 8 * sizeof(uint32_t)); +SLANG_COMPILE_TIME_ASSERT(int(Node::Kind::CountOf) <= 8 * sizeof(uint32_t)); // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Parser !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -21,34 +21,39 @@ Parser::Parser(NodeTree* nodeTree, DiagnosticSink* sink) : m_nodeTypeEnabled(0) { // Enable types by default - const Node::Type defaultEnabled[] = - { - Node::Type::ClassType, - Node::Type::StructType, - Node::Type::Namespace, - Node::Type::AnonymousNamespace, - Node::Type::Field, + const Node::Kind defaultEnabled[] = + { + Node::Kind::ClassType, + Node::Kind::StructType, + Node::Kind::Namespace, + Node::Kind::AnonymousNamespace, + Node::Kind::Field, + + // These are disabled by default because AST uses macro magic to build up the types + // Node::Type::TypeDef, + // Node::Type::Enum, + // Node::Type::EnumClass, }; - setTypesEnabled(defaultEnabled, SLANG_COUNT_OF(defaultEnabled)); + setKindsEnabled(defaultEnabled, SLANG_COUNT_OF(defaultEnabled)); } -void Parser::setTypeEnabled(Node::Type type, bool isEnabled ) +void Parser::setKindEnabled(Node::Kind kind, bool isEnabled) { if (isEnabled) { - m_nodeTypeEnabled |= (NodeTypeBitType(1) << int(type)); + m_nodeTypeEnabled |= (NodeTypeBitType(1) << int(kind)); } else { - m_nodeTypeEnabled &= ~(NodeTypeBitType(1) << int(type)); + m_nodeTypeEnabled &= ~(NodeTypeBitType(1) << int(kind)); } } -void Parser::setTypesEnabled(const Node::Type* types, Index typesCount, bool isEnabled) +void Parser::setKindsEnabled(const Node::Kind* kinds, Index kindsCount, bool isEnabled) { - for (Index i = 0; i < typesCount; ++i) + for (Index i = 0; i < kindsCount; ++i) { - setTypeEnabled(types[i], isEnabled); + setKindEnabled(kinds[i], isEnabled); } } @@ -157,9 +162,9 @@ SlangResult Parser::pushScope(ScopeNode* scopeNode) return SLANG_FAIL; } - if (foundNode->m_type == Node::Type::Namespace) + if (foundNode->m_kind == Node::Kind::Namespace) { - if (foundNode->m_type != scopeNode->m_type) + if (foundNode->m_kind != scopeNode->m_kind) { // Different types can't work m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::typeAlreadyDeclared, scopeNode->m_name.getContent()); @@ -274,13 +279,13 @@ SlangResult Parser::_parseEnum() // consume enum SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &enumToken)); - if (!m_currentScope->acceptsTypes()) + if (!m_currentScope->canContainTypes()) { m_sink->diagnose(enumToken.loc, CPPDiagnostics::cannotDeclareTypeInScope); return SLANG_FAIL; } - Node::Type type = Node::Type::Enum; + Node::Kind kind = Node::Kind::Enum; Token nameToken; if (advanceIfToken(TokenType::Identifier, &nameToken)) @@ -289,7 +294,7 @@ SlangResult Parser::_parseEnum() if (style == IdentifierStyle::Class) { - type = Node::Type::EnumClass; + kind = Node::Kind::EnumClass; SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &nameToken)); } else if (style == IdentifierStyle::None) @@ -303,7 +308,7 @@ SlangResult Parser::_parseEnum() } } - RefPtr node = new EnumNode(type); + RefPtr node = new EnumNode(kind); node->m_name = nameToken; if (advanceIfToken(TokenType::Colon)) @@ -328,9 +333,9 @@ SlangResult Parser::_parseEnum() // for our uses here. // If we can't find the type, we could assume it's size is undefined - if (backingTokens.getCount() == 1) + if (backingTokens.getCount() > 0) { - node->m_backingToken = backingTokens[0]; + node->m_backingTokens.swapWith(backingTokens); } } @@ -345,7 +350,7 @@ SlangResult Parser::_parseEnum() { // Strictly speaking we should check the backing type etc, match, but for now ignore and assume it's ok - if (node->m_type == type) + if (node->m_kind == kind) { return SLANG_OK; } @@ -400,9 +405,9 @@ SlangResult Parser::_parseEnum() valueTokens.add(m_reader.advanceToken()); } - if (valueTokens.getCount() == 1) + if (valueTokens.getCount() > 0) { - caseNode->m_value = valueTokens[0]; + caseNode->m_valueTokens.swapWith(valueTokens); } } @@ -422,12 +427,12 @@ SlangResult Parser::_parseEnum() return popScope(); } -SlangResult Parser::_maybeParseNode(Node::Type type) +SlangResult Parser::_maybeParseNode(Node::Kind kind) { // We are looking for // struct/class identifier [: [public|private|protected] Identifier ] { [public|private|proctected:]* marker ( identifier ); - if (type == Node::Type::Namespace) + if (kind == Node::Kind::Namespace) { // consume namespace SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier)); @@ -442,7 +447,7 @@ SlangResult Parser::_maybeParseNode(Node::Type type) if (advanceIfToken(TokenType::LBrace)) { // Okay looks like we are opening a namespace - RefPtr node(new ScopeNode(Node::Type::Namespace)); + RefPtr node(new ScopeNode(Node::Kind::Namespace)); node->m_name = name; // Push the node return pushScope(node); @@ -452,14 +457,14 @@ SlangResult Parser::_maybeParseNode(Node::Type type) // Just ignore it then return SLANG_OK; } - else if (Node::isEnumLikeType(type)) + else if (Node::isKindEnumLike(kind)) { return _parseEnum(); } // Must be class | struct - SLANG_ASSERT(type == Node::Type::ClassType || type == Node::Type::StructType); + SLANG_ASSERT(kind == Node::Kind::ClassType || kind == Node::Kind::StructType); Token name; @@ -474,7 +479,7 @@ SlangResult Parser::_maybeParseNode(Node::Type type) return SLANG_OK; } - RefPtr node(new ClassLikeNode(type)); + RefPtr node(new ClassLikeNode(kind)); node->m_name = name; // Defaults to not reflected @@ -962,7 +967,7 @@ SlangResult Parser::_parseBalanced(DiagnosticSink* sink) SlangResult Parser::_parseTypeDef() { - if (!m_currentScope->acceptsTypes()) + if (!m_currentScope->canContainTypes()) { m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::cannotDeclareTypeInScope); return SLANG_FAIL; @@ -1078,16 +1083,16 @@ SlangResult Parser::_maybeParseField() return SLANG_OK; } -/* static */Node::Type Parser::_toNodeType(IdentifierStyle style) +/* static */Node::Kind Parser::_toNodeKind(IdentifierStyle style) { switch (style) { - case IdentifierStyle::Class: return Node::Type::ClassType; - case IdentifierStyle::Struct: return Node::Type::StructType; - case IdentifierStyle::Namespace: return Node::Type::Namespace; - case IdentifierStyle::Enum: return Node::Type::Enum; - case IdentifierStyle::TypeDef: return Node::Type::TypeDef; - default: return Node::Type::Invalid; + case IdentifierStyle::Class: return Node::Kind::ClassType; + case IdentifierStyle::Struct: return Node::Kind::StructType; + case IdentifierStyle::Namespace: return Node::Kind::Namespace; + case IdentifierStyle::Enum: return Node::Kind::Enum; + case IdentifierStyle::TypeDef: return Node::Kind::TypeDef; + default: return Node::Kind::Invalid; } } @@ -1118,7 +1123,7 @@ SlangResult Parser::_parsePreDeclare() SLANG_RETURN_ON_FAIL(expect(TokenType::Comma)); // Get the type of type - Node::Type nodeType; + Node::Kind nodeKind; { Token typeToken; SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &typeToken)); @@ -1130,7 +1135,7 @@ SlangResult Parser::_parsePreDeclare() m_sink->diagnose(typeToken, CPPDiagnostics::expectingTypeKeyword, typeToken.getContent()); return SLANG_FAIL; } - nodeType = _toNodeType(style); + nodeKind = _toNodeKind(style); } Token name; @@ -1145,12 +1150,12 @@ SlangResult Parser::_parsePreDeclare() SLANG_RETURN_ON_FAIL(expect(TokenType::RParent)); - switch (nodeType) + switch (nodeKind) { - case Node::Type::ClassType: - case Node::Type::StructType: + case Node::Kind::ClassType: + case Node::Kind::StructType: { - RefPtr node(new ClassLikeNode(nodeType)); + RefPtr node(new ClassLikeNode(nodeKind)); node->m_name = name; node->m_super = super; @@ -1274,7 +1279,7 @@ SlangResult Parser::parse(SourceOrigin* sourceOrigin, const Options* options) } case IdentifierStyle::TypeDef: { - if (isTypeEnabled(Node::Type::TypeDef)) + if (isTypeEnabled(Node::Kind::TypeDef)) { SLANG_RETURN_ON_FAIL(_parseTypeDef()); } @@ -1291,12 +1296,12 @@ SlangResult Parser::parse(SourceOrigin* sourceOrigin, const Options* options) if (flags & IdentifierFlag::StartScope) { - Node::Type type = _toNodeType(style); - SLANG_ASSERT(type != Node::Type::Invalid); + Node::Kind kind = _toNodeKind(style); + SLANG_ASSERT(kind != Node::Kind::Invalid); - if (isTypeEnabled(type)) + if (isTypeEnabled(kind)) { - SLANG_RETURN_ON_FAIL(_maybeParseNode(type)); + SLANG_RETURN_ON_FAIL(_maybeParseNode(kind)); } else { @@ -1307,7 +1312,7 @@ SlangResult Parser::parse(SourceOrigin* sourceOrigin, const Options* options) { // Special case the node that's the root of the hierarchy (as far as reflection is concerned) // This could be a field - if (m_currentScope->acceptsFields()) + if (m_currentScope->canContainFields()) { SLANG_RETURN_ON_FAIL(_maybeParseField()); } diff --git a/tools/slang-cpp-extractor/parser.h b/tools/slang-cpp-extractor/parser.h index f0febb728..5ed862d65 100644 --- a/tools/slang-cpp-extractor/parser.h +++ b/tools/slang-cpp-extractor/parser.h @@ -31,14 +31,14 @@ public: /// Parse the contents of the source file SlangResult parse(SourceOrigin* sourceOrigin, const Options* options); - void setTypeEnabled(Node::Type type, bool isEnabled = true); - bool isTypeEnabled(Node::Type type) { return (m_nodeTypeEnabled & (NodeTypeBitType(1) << int(type))) != 0; } - void setTypesEnabled(const Node::Type* types, Index typesCount, bool isEnabled = true); + void setKindEnabled(Node::Kind kind, bool isEnabled = true); + bool isTypeEnabled(Node::Kind kind) { return (m_nodeTypeEnabled & (NodeTypeBitType(1) << int(kind))) != 0; } + void setKindsEnabled(const Node::Kind* kinds, Index kindsCount, bool isEnabled = true); Parser(NodeTree* nodeTree, DiagnosticSink* sink); protected: - static Node::Type _toNodeType(IdentifierStyle style); + static Node::Kind _toNodeKind(IdentifierStyle style); bool _isMarker(const UnownedStringSlice& name); @@ -47,7 +47,7 @@ protected: SlangResult _parsePreDeclare(); SlangResult _parseTypeSet(); - SlangResult _maybeParseNode(Node::Type type); + SlangResult _maybeParseNode(Node::Kind kind); SlangResult _maybeParseField(); SlangResult _parseTypeDef(); diff --git a/tools/slang-cpp-extractor/unit-test.cpp b/tools/slang-cpp-extractor/unit-test.cpp index 7648e4e50..a30a3caa7 100644 --- a/tools/slang-cpp-extractor/unit-test.cpp +++ b/tools/slang-cpp-extractor/unit-test.cpp @@ -67,8 +67,8 @@ static const char someSource[] = Parser parser(&tree, &state.m_sink); { - const Node::Type enableTypes[] = { Node::Type::Enum, Node::Type::EnumClass, Node::Type::EnumCase, Node::Type::TypeDef }; - parser.setTypesEnabled(enableTypes, SLANG_COUNT_OF(enableTypes)); + const Node::Kind enableKinds[] = { Node::Kind::Enum, Node::Kind::EnumClass, Node::Kind::EnumCase, Node::Kind::TypeDef }; + parser.setKindsEnabled(enableKinds, SLANG_COUNT_OF(enableKinds)); } SLANG_RETURN_ON_FAIL(parser.parse(sourceOrigin, &state.m_options)); -- cgit v1.2.3