Documentation markup extraction (#1724)

* #include an absolute path didn't work - because paths were taken to always be relative.

* WIP extracting source documentation.

* WIP doc extraction.

* More stuff around doc markup extraction.

* More WIP around doc extraction.

* Fix some indexing issues.

* Initial doc extraction working.

* Renaming of types in markup extraction process.

* Extracting markup content.
Removing indenting.
Other fixes and improvements around document tools.

* WIP support for documentation system.

* Remove some commented out sections.

* Remove some comments that no longer apply.

* Improvements around SourceFile - such that more granularity around line ops.
Made some functionality explicitly work without source.
Improved Doc types nameing.

1 files changed, 34 insertions, 12 deletions

diff --git a/source/slang/slang-lexer.cpp b/source/slang/slang-lexer.cpp
index b0146c5b0..6c8e9474a 100644
--- a/source/slang/slang-lexer.cpp
+++ b/source/slang/slang-lexer.cpp
@@ -75,27 +75,29 @@ namespace Slang
     // Lexer
 
     void Lexer::initialize(
-        SourceView*     inSourceView,
-        DiagnosticSink* inSink,
-        NamePool*       inNamePool,
-        MemoryArena*    inMemoryArena)
+        SourceView*     sourceView,
+        DiagnosticSink* sink,
+        NamePool*       namePool,
+        MemoryArena*    memoryArena,
+        OptionFlags     optionFlags)
     {
-        m_sourceView  = inSourceView;
-        m_sink        = inSink;
-        m_namePool    = inNamePool;
-        m_memoryArena = inMemoryArena;
+        m_sourceView  = sourceView;
+        m_sink        = sink;
+        m_namePool    = namePool;
+        m_memoryArena = memoryArena;
 
-        auto content = inSourceView->getContent();
+        auto content = sourceView->getContent();
         
         m_begin   = content.begin();
         m_cursor  = content.begin();
         m_end     = content.end();
 
         // Set the start location
-        m_startLoc = inSourceView->getRange().begin;
+        m_startLoc = sourceView->getRange().begin;
 
         m_tokenFlags = TokenFlag::AtStartOfLine | TokenFlag::AfterWhitespace;
         m_lexerFlags = 0;
+        m_optionFlags = optionFlags;
     }
 
     Lexer::~Lexer()
@@ -1231,11 +1233,31 @@ namespace Slang
                 continue;
 
             case TokenType::WhiteSpace:
-            case TokenType::LineComment:
-            case TokenType::BlockComment:
+            {
                 flags |= TokenFlag::AfterWhitespace;
                 continue;
+            }
+            case TokenType::BlockComment:
+            case TokenType::LineComment:
+            {
+                flags |= TokenFlag::AfterWhitespace;
+                if (m_optionFlags & OptionFlag::TokenizeComments)
+                {
+                    // We don't break here, and use the normal token adding logic
+                    // because we want the behavior to be identical (in terms of flags etc)
+                    // as if TokenizeComments is not enabled
+                    char const* textEnd = m_cursor;
+
+                    token.type =  tokenType;
+                    token.flags = m_tokenFlags;
+                    token.setContent(UnownedStringSlice(textBegin, textEnd));
 
+                    return token;
+                }
+
+                continue;
+            }
+            
             // We don't want to skip the end-of-file token, but we *do*
             // want to make sure it has appropriate flags to make our life easier
             case TokenType::EndOfFile: