From 55a5ccc559b34b8d2eb9c7b7a2d9efbae40619c2 Mon Sep 17 00:00:00 2001
From: jsmall-nvidia <jsmall@nvidia.com>
Date: Tue, 23 Feb 2021 12:36:46 -0500
Subject: Documentation markup extraction (#1724)

* #include an absolute path didn't work - because paths were taken to always be relative.

* WIP extracting source documentation.

* WIP doc extraction.

* More stuff around doc markup extraction.

* More WIP around doc extraction.

* Fix some indexing issues.

* Initial doc extraction working.

* Renaming of types in markup extraction process.

* Extracting markup content.
Removing indenting.
Other fixes and improvements around document tools.

* WIP support for documentation system.

* Remove some commented out sections.

* Remove some comments that no longer apply.

* Improvements around SourceFile - such that more granularity around line ops.
Made some functionality explicitly work without source.
Improved Doc types nameing.
---
 source/slang/slang-lexer.cpp | 46 ++++++++++++++++++++++++++++++++------------
 1 file changed, 34 insertions(+), 12 deletions(-)

(limited to 'source/slang/slang-lexer.cpp')

diff --git a/source/slang/slang-lexer.cpp b/source/slang/slang-lexer.cpp
index b0146c5b0..6c8e9474a 100644
--- a/source/slang/slang-lexer.cpp
+++ b/source/slang/slang-lexer.cpp
@@ -75,27 +75,29 @@ namespace Slang
     // Lexer
 
     void Lexer::initialize(
-        SourceView*     inSourceView,
-        DiagnosticSink* inSink,
-        NamePool*       inNamePool,
-        MemoryArena*    inMemoryArena)
+        SourceView*     sourceView,
+        DiagnosticSink* sink,
+        NamePool*       namePool,
+        MemoryArena*    memoryArena,
+        OptionFlags     optionFlags)
     {
-        m_sourceView  = inSourceView;
-        m_sink        = inSink;
-        m_namePool    = inNamePool;
-        m_memoryArena = inMemoryArena;
+        m_sourceView  = sourceView;
+        m_sink        = sink;
+        m_namePool    = namePool;
+        m_memoryArena = memoryArena;
 
-        auto content = inSourceView->getContent();
+        auto content = sourceView->getContent();
         
         m_begin   = content.begin();
         m_cursor  = content.begin();
         m_end     = content.end();
 
         // Set the start location
-        m_startLoc = inSourceView->getRange().begin;
+        m_startLoc = sourceView->getRange().begin;
 
         m_tokenFlags = TokenFlag::AtStartOfLine | TokenFlag::AfterWhitespace;
         m_lexerFlags = 0;
+        m_optionFlags = optionFlags;
     }
 
     Lexer::~Lexer()
@@ -1231,11 +1233,31 @@ namespace Slang
                 continue;
 
             case TokenType::WhiteSpace:
-            case TokenType::LineComment:
-            case TokenType::BlockComment:
+            {
                 flags |= TokenFlag::AfterWhitespace;
                 continue;
+            }
+            case TokenType::BlockComment:
+            case TokenType::LineComment:
+            {
+                flags |= TokenFlag::AfterWhitespace;
+                if (m_optionFlags & OptionFlag::TokenizeComments)
+                {
+                    // We don't break here, and use the normal token adding logic
+                    // because we want the behavior to be identical (in terms of flags etc)
+                    // as if TokenizeComments is not enabled
+                    char const* textEnd = m_cursor;
+
+                    token.type =  tokenType;
+                    token.flags = m_tokenFlags;
+                    token.setContent(UnownedStringSlice(textBegin, textEnd));
 
+                    return token;
+                }
+
+                continue;
+            }
+            
             // We don't want to skip the end-of-file token, but we *do*
             // want to make sure it has appropriate flags to make our life easier
             case TokenType::EndOfFile:
-- 
cgit v1.2.3