Language server: extract documentation from ordinary comments (#2308)

* Language server: improved documentation formatting.

* Extend doc extractor to search in ordinary comments.
Reuse language server instance between tests.

* Fix test case.

* Fix comment.

* Fix crash.

* Fix enum case doc extraction.

* Doc extractor fixes.

* Fix.

* Fix.

Co-authored-by: Yong He <yhe@nvidia.com>

2 files changed, 46 insertions, 6 deletions

diff --git a/source/compiler-core/slang-doc-extractor.cpp b/source/compiler-core/slang-doc-extractor.cpp
index 363108548..9885e29e1 100644
--- a/source/compiler-core/slang-doc-extractor.cpp
+++ b/source/compiler-core/slang-doc-extractor.cpp
@@ -40,7 +40,15 @@ namespace Slang {
             }
             return comment;
         }
-
+        case MarkupType::OrdinaryBlockBefore:
+        {
+            if (comment.startsWith(UnownedStringSlice::fromLiteral("/*")))
+            {
+                /// ordinary /* */ block.
+                return comment.tail(2);
+            }
+            return comment;
+        }
         case MarkupType::LineBangBefore:
         {
             return comment.startsWith(UnownedStringSlice::fromLiteral("//!")) ? comment.tail(3) : comment;
@@ -49,7 +57,11 @@ namespace Slang {
         {
             return comment.startsWith(UnownedStringSlice::fromLiteral("///")) ? comment.tail(3) : comment;
         }
-
+        case MarkupType::OrdinaryLineBefore:
+        case MarkupType::OrdinaryLineAfter:
+        {
+            return comment.startsWith(UnownedStringSlice::fromLiteral("//")) ? comment.tail(2) : comment;
+        }
         case MarkupType::LineBangAfter:
         {
             /// //!< Can be multiple lines
@@ -102,12 +114,16 @@ static Index _findTokenIndex(SourceLoc loc, const Token* toks, Index numToks)
         case MarkupType::None:              return 0;
         case MarkupType::BlockBefore:      return MarkupFlag::Before | MarkupFlag::IsBlock; 
         case MarkupType::BlockAfter:       return MarkupFlag::After  | MarkupFlag::IsBlock;
+        case MarkupType::OrdinaryBlockBefore: return MarkupFlag::Before | MarkupFlag::IsBlock;
 
         case MarkupType::LineBangBefore:     return MarkupFlag::Before | MarkupFlag::IsMultiToken; 
         case MarkupType::LineSlashBefore:    return MarkupFlag::Before | MarkupFlag::IsMultiToken; 
+        case MarkupType::OrdinaryLineBefore: return MarkupFlag::Before | MarkupFlag::IsMultiToken;
 
         case MarkupType::LineBangAfter:      return MarkupFlag::After | MarkupFlag::IsMultiToken; 
         case MarkupType::LineSlashAfter:     return MarkupFlag::After | MarkupFlag::IsMultiToken;
+        case MarkupType::OrdinaryLineAfter: return MarkupFlag::After | MarkupFlag::IsMultiToken;
+
     }
 }
 
@@ -122,6 +138,10 @@ static Index _findTokenIndex(SourceLoc loc, const Token* toks, Index numToks)
             {
                 return (slice.getLength() >= 4 && slice[3] == '<') ? MarkupType::BlockAfter : MarkupType::BlockBefore;
             }
+            else
+            {
+                return MarkupType::OrdinaryBlockBefore;
+            }
             break;
         }
         case TokenType::LineComment:
@@ -138,6 +158,8 @@ static Index _findTokenIndex(SourceLoc loc, const Token* toks, Index numToks)
                     return (slice.getLength() >= 4 && slice[3] == '<') ? MarkupType::LineSlashAfter : MarkupType::LineSlashBefore;
                 }
             }
+            return (tok.flags & TokenFlag::AtStartOfLine) != 0 ? MarkupType::OrdinaryLineBefore
+                                                               : MarkupType::OrdinaryLineAfter;
             break;
         }
         default: break;
@@ -186,6 +208,7 @@ SlangResult DocMarkupExtractor::_extractMarkup(const FindInfo& info, const Found
     {
         case MarkupType::BlockBefore:
         case MarkupType::BlockAfter:
+        case MarkupType::OrdinaryBlockBefore:
         {
             // We should only have a single line
             SLANG_ASSERT(foundMarkup.range.getCount() == 1);
@@ -253,6 +276,8 @@ SlangResult DocMarkupExtractor::_extractMarkup(const FindInfo& info, const Found
 
             break;
         }
+        case MarkupType::OrdinaryLineBefore:
+        case MarkupType::OrdinaryLineAfter:
         case MarkupType::LineBangBefore:      
         case MarkupType::LineSlashBefore:
         case MarkupType::LineBangAfter:
@@ -373,6 +398,10 @@ Index DocMarkupExtractor::_findStartIndex(const FindInfo& info, Location locatio
                 {
                     // Determine the markup type
                     const MarkupType markupType = findMarkupType(tok);
+                    if (!m_searchInOrindaryComments &&
+                        (markupType == MarkupType::OrdinaryBlockBefore ||
+                         markupType == MarkupType::OrdinaryLineBefore))
+                        break;
                     // If the location wanted is before and the markup is, we'll assume this is it
                     if (isBefore(location) && isBefore(markupType))
                     {
@@ -459,7 +488,7 @@ SlangResult DocMarkupExtractor::_findMarkup(const FindInfo& info, Location locat
     SourceFile* sourceFile = sourceView->getSourceFile();
 
     // Let's lookup the line index where this occurred
-    const int startOffset = sourceView->getRange().getOffset(toks[startIndex - 1].loc);
+    const int startOffset = sourceView->getRange().getOffset(toks[startIndex].loc);
 
     // The line index that the markoff starts from 
     Index lineIndex = sourceFile->calcLineIndexFromOffset(startOffset);
@@ -503,7 +532,8 @@ SlangResult DocMarkupExtractor::_findMarkup(const FindInfo& info, Location locat
         {
             endIndex += searchDirection;
             expectedLineIndex += searchDirection;
-
+            if (expectedLineIndex < 0)
+                break;
             if (endIndex < 0 || endIndex >= toks.getCount())
             {
                 break;
@@ -516,7 +546,7 @@ SlangResult DocMarkupExtractor::_findMarkup(const FindInfo& info, Location locat
             }
 
             // Is it on the right line?
-            if (_isTokenOnLineIndex(info.sourceView, type, toks[startIndex], expectedLineIndex))
+            if (!_isTokenOnLineIndex(info.sourceView, type, toks[endIndex], expectedLineIndex))
             {
                 break;
             }
diff --git a/source/compiler-core/slang-doc-extractor.h b/source/compiler-core/slang-doc-extractor.h
index 5b4c67782..fc486250e 100644
--- a/source/compiler-core/slang-doc-extractor.h
+++ b/source/compiler-core/slang-doc-extractor.h
@@ -42,13 +42,17 @@ public:
         BlockBefore,        /// /**  */ or /*!  */.
         LineBangBefore,      /// //! Can be multiple lines
         LineSlashBefore,     /// /// Can be multiple lines
+        OrdinaryBlockBefore,
+        OrdinaryLineBefore,
 
         BlockAfter,         /// /*!< */ or /**< */
         LineBangAfter,       /// //!< Can be multiple lines
         LineSlashAfter,      /// ///< Can be multiple lines
+        OrdinaryLineAfter,
+
     };
 
-    static bool isBefore(MarkupType type) { return Index(type) >= Index(MarkupType::BlockBefore) && Index(type) <= Index(MarkupType::LineSlashBefore); }
+    static bool isBefore(MarkupType type) { return Index(type) >= Index(MarkupType::BlockBefore) && Index(type) <= Index(MarkupType::OrdinaryLineBefore); }
     static bool isAfter(MarkupType type) { return Index(type) >= Index(MarkupType::BlockAfter); }
 
     struct IndexRange
@@ -121,6 +125,10 @@ public:
         Index lineIndex;                ///< The line number for the decl
     };
 
+    void setSearchInOrdinaryComments(bool val)
+    {
+        m_searchInOrindaryComments = val;
+    }
     
         /// Extracts 'markup' doc information for the specified input items
         /// The output is placed in out - with the items now in the source order *not* the order of the input items
@@ -156,6 +164,8 @@ protected:
     static bool _isTokenOnLineIndex(SourceView* sourceView, MarkupType type, const Token& tok, Index lineIndex);
 
     DiagnosticSink* m_sink;
+
+    bool m_searchInOrindaryComments = false;
 };
 
 } // namespace Slang