stdlib documentation (#1745)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Split out AST 'printing'.

* Replace listener with List<Section>

* Section -> Part.

* Kind -> Type Flags -> Kind for ASTPrinter::Part

* Improve comments around ASTPrinter.

* toString -> toText on Val derived types. toText appends to a StringBuilder.

* Added toSlice free function.
Added operator<< for Val derived types.
Use << where appropriate in doing toText.

* More work at mark down output.

* Fill in sourceloc for enum case.
Add more sophisticated location determination for EnumCase.
Refactored documentation output into DocMarkdownWriter.

* Improvements for sig output.

* Split up slang-doc into extractor and writer.

* WIP generic support for doc support.

* Some refactoring to make DocExtractor have potential to be used without Decls.

* Made doc extraction work without Decls.

* Output generic parameters.

* Add generic parameter extraction.

* Added writing variables.

* Add an interface test.

* Fix toArray.

* Support for extensions, and inheritance.

* Disable the doc test.

* Added flags to compileStdLib.

* More work around handling generics in markdown output.

* More improvements around associated type handling.

* List method names only once.
Output in/out/inout/const

* Fix namespace printing.

* WIP summarizing doc output.

* Small fixes and improvements for doc output.

* Output all stdlib in single doc file.

* Remove compile flags from addBuiltinSource.

* Find only unique signatures.
First pass at trying to get requirements.

* First pass at requirements for stdlib docs.

* Remove __ function/methods

* Added Target Availability

* Add markup access.
Make sections of stdlib hidden.

* MarkdownAccess -> Visibility
Add isVisible methods
Use ASTPrinter to print decl name.

* Add current stdlib doc output.

* Disable doc test for now.

* Fix clang issue.

* Don't use bullets and numbering , just use numbering.

* Put methods in source order.

* Fix bad-operator-call.slang test that fails because it now outputs out parameters as such.

* Refactor MarkDownWriter to separate 'extraction' from output.

* Fix typo around @ lines.

* Fix issue with extracting 'before' when preceeded by complex attributes/modifiers.

* Fix handling of generics with the same name.

* Work around for having overloading with generics - we don't want to output generic params as part of name.

* Remove generic paramters from name.

* Simplify handling of outputting overridable names.

1 files changed, 83 insertions, 5 deletions

diff --git a/source/slang/slang-doc-extractor.cpp b/source/slang/slang-doc-extractor.cpp
index f44257877..3951e4977 100644
--- a/source/slang/slang-doc-extractor.cpp
+++ b/source/slang/slang-doc-extractor.cpp
@@ -322,6 +322,7 @@ Index DocMarkupExtractor::_findStartIndex(const FindInfo& info, Location locatio
 
         switch (tok.type)
         {
+            case TokenType::LBrace:
             case TokenType::LBracket:
             case TokenType::LParent:
             case TokenType::OpLess:
@@ -396,11 +397,11 @@ Index DocMarkupExtractor::_findStartIndex(const FindInfo& info, Location locatio
                     {
                         return i + 1;
                     }
-                }
 
-                if (location == Location::Before)
-                {
-                    return -1;
+                    if (location == Location::Before)
+                    {
+                        return -1;
+                    }
                 }
 
                 break;
@@ -636,6 +637,9 @@ SlangResult DocMarkupExtractor::_findMarkup(const FindInfo& info, SearchStyle se
             return _findMarkup(info, locs, SLANG_COUNT_OF(locs), out);
         }
         case SearchStyle::Before:
+        {
+            return _findMarkup(info, Location::Before, out);
+        }
         case SearchStyle::Function:
         {
             return _findMarkup(info, Location::Before, out);
@@ -653,6 +657,70 @@ SlangResult DocMarkupExtractor::_findMarkup(const FindInfo& info, SearchStyle se
     }
 }
 
+
+static void _calcLineVisibility(SourceView* sourceView, const TokenList& toks, List<MarkupVisibility>& outLineVisibility)
+{
+    SourceFile* sourceFile = sourceView->getSourceFile();
+    const auto& lineOffsets = sourceFile->getLineBreakOffsets();
+
+    outLineVisibility.setCount(lineOffsets.getCount() + 1);
+
+    MarkupVisibility lastVisibility = MarkupVisibility::Public;
+    Index lastLine = 0;
+
+    for (const auto& tok : toks)
+    {
+        if (tok.type == TokenType::LineComment)
+        {
+            UnownedStringSlice contents = tok.getContent();
+
+            MarkupVisibility newVisibility = lastVisibility;
+
+            // Distinct from other markup
+            if (contents.startsWith(toSlice("//@")))
+            {
+                UnownedStringSlice access = contents.tail(3).trim();
+                if (access == "hidden:" || access == "private:")
+                {
+                    newVisibility = MarkupVisibility::Hidden;
+                }
+                else if (access == "internal:")
+                {
+                    newVisibility = MarkupVisibility::Internal;
+                }
+                else if (access == "public:")
+                {
+                    newVisibility = MarkupVisibility::Public;
+                }
+            }
+
+            if (newVisibility != lastVisibility)
+            {
+                // Work up the line it's on
+                const int offset = sourceView->getRange().getOffset(tok.loc);
+                Index line = sourceFile->calcLineIndexFromOffset(offset);
+
+                // Fill in the span
+                for (Index i = lastLine; i < line; ++i)
+                {
+                    outLineVisibility[i] = lastVisibility;
+                }
+
+                // Record the new access and where we are up to
+                lastLine = line;
+                lastVisibility = newVisibility;
+            }
+        }
+    }
+
+    // Fill in the remaining
+    for (Index i = lastLine; i < outLineVisibility.getCount(); ++ i)
+    {
+        outLineVisibility[i] = lastVisibility;
+    }
+}
+
+
 SlangResult DocMarkupExtractor::extract(const SearchItemInput* inputs, Index inputCount, SourceManager* sourceManager, DiagnosticSink* sink, List<SourceView*>& outViews, List<SearchItemOutput>& out)
 {
     struct Entry
@@ -726,8 +794,10 @@ SlangResult DocMarkupExtractor::extract(const SearchItemInput* inputs, Index inp
 
     {
         TokenList tokens;
+        List<MarkupVisibility> lineVisibility;
+
+        MemoryArena memoryArena(4096);
 
-        MemoryArena memoryArena;
         RootNamePool rootNamePool;
         NamePool namePool;
         namePool.setRootNamePool(&rootNamePool);
@@ -746,6 +816,7 @@ SlangResult DocMarkupExtractor::extract(const SearchItemInput* inputs, Index inp
 
             dst.viewIndex = -1;
             dst.inputIndex = entry.inputIndex;
+            dst.visibilty = MarkupVisibility::Public;
 
             // If there isn't a mechanism to search with, just move on
             if (entry.searchStyle == SearchStyle::None)
@@ -767,6 +838,10 @@ SlangResult DocMarkupExtractor::extract(const SearchItemInput* inputs, Index inp
 
                 // Lex everything
                 tokens = lexer.lexAllTokens();
+
+                // Let's work out the access
+
+                _calcLineVisibility(sourceView, tokens, lineVisibility);
             }
 
             dst.viewIndex = viewIndex;
@@ -781,6 +856,8 @@ SlangResult DocMarkupExtractor::extract(const SearchItemInput* inputs, Index inp
             SourceFile* sourceFile = sourceView->getSourceFile();
             const Index lineIndex = sourceFile->calcLineIndexFromOffset(int(offset));
 
+            dst.visibilty = lineVisibility[lineIndex];
+
             // Okay, lets find the token index with a binary chop
             Index tokenIndex = _findTokenIndex(loc, tokens.m_tokens.getBuffer(), tokens.m_tokens.getCount());
             if (tokenIndex >= 0 && lineIndex >= 0)
@@ -902,6 +979,7 @@ SlangResult DocMarkupExtractor::extract(ModuleDecl* moduleDecl, SourceManager* s
             // Add to the documentation
             DocMarkup::Entry& docEntry = outDoc->addEntry(decl);
             docEntry.m_markup = outputItem.text;
+            docEntry.m_visibility = outputItem.visibilty;
         }
     }