Major language server features. (#2264)

* Major language server features.

* Include slangd in binary release.

* Fix compiler issues.

* Fix compiler error.

* Completion resolve.

* Various improvements.

* Update diagnostic test expected output.

* Bug fix for source locations.

* Adjust diagnostic update frequency.

* Update github actions to store artifacts.

* Fix infinite parser loop.

* Fix parser recovery.

* Fix parser recovery.

* Update test.

* Fix test.

* Disable IR gen for language server.

* Allow commit characters in auto completion.

* Fix lookup for invoke exprs.

* More parser robustness fixes.

* update solution file

Co-authored-by: Yong He <yhe@nvidia.com>

1 files changed, 74 insertions, 27 deletions

diff --git a/source/slang/slang-parser.cpp b/source/slang/slang-parser.cpp
index 2604ffd9b..1c32499ef 100644
--- a/source/slang/slang-parser.cpp
+++ b/source/slang/slang-parser.cpp
@@ -91,6 +91,10 @@ namespace Slang
 
         int anonymousCounter = 0;
 
+        // Numbers of times we are peeking the same token at `ReadToken` without advancing in
+        // recover mode.
+        int sameTokenPeekedTimes = 0;
+
         Scope* outerScope = nullptr;
         Scope* currentScope = nullptr;
 
@@ -530,6 +534,7 @@ namespace Slang
         if (tokenReader.peekTokenType() == expected)
         {
             isRecovering = false;
+            sameTokenPeekedTimes = 0;
             return tokenReader.advanceToken();
         }
 
@@ -546,8 +551,22 @@ namespace Slang
                 isRecovering = false;
                 return tokenReader.advanceToken();
             }
-
-            return tokenReader.peekToken();
+            // This could be dangerous: if `ReadToken()` is being called
+            // in a loop we may never make forward progress, so we use
+            // a counter to limit the maximum amount of times we are allowed
+            // to peek the same token. If the outter parsing logic is
+            // correct, we will pop back to the right level. If there are
+            // erroneous parsing logic, this counter is to prevent us
+            // looping infinitely.
+            static const int kMaxTokenPeekCount = 64;
+            sameTokenPeekedTimes++;
+            if (sameTokenPeekedTimes < kMaxTokenPeekCount)
+                return tokenReader.peekToken();
+            else
+            {
+                sameTokenPeekedTimes = 0;
+                return tokenReader.advanceToken();
+            }
         }
     }
 
@@ -1238,7 +1257,7 @@ namespace Slang
     // parent link is set up correctly.
     static void AddMember(ContainerDecl* container, Decl* member)
     {
-        if (container)
+        if (container && member)
         {
             member->parentDecl = container;
             container->members.add(member);
@@ -1325,12 +1344,19 @@ namespace Slang
                 break;
             }
 
+            auto currentCursor = parser->tokenReader.getCursor();
+
             AddMember(decl, ParseGenericParamDecl(parser, decl));
 
+            // Make sure we make forward progress.
+            if (parser->tokenReader.getCursor() == currentCursor)
+                advanceToken(parser);
+
             if (parser->LookAheadToken(TokenType::OpGreater))
                 break;
 
-            parser->ReadToken(TokenType::Comma);
+            if (!AdvanceIf(parser, TokenType::Comma))
+                break;
         }
         parser->genericDepth--;
         parser->ReadToken(TokenType::OpGreater);
@@ -1863,7 +1889,7 @@ namespace Slang
     {
         GenericAppExpr* genericApp = parser->astBuilder->create<GenericAppExpr>();
 
-        parser->FillPosition(genericApp); // set up scope for lookup
+        genericApp->loc = base->loc;
         genericApp->functionExpr = base;
         parser->ReadToken(TokenType::OpLess);
         parser->genericDepth++;
@@ -1947,12 +1973,12 @@ namespace Slang
         }
         return base;
     }
-    static Expr* parseMemberType(Parser * parser, Expr* base)
+    static Expr* parseMemberType(Parser * parser, Expr* base, SourceLoc opLoc)
     {
         // When called the :: or . have been consumed, so don't need to consume here.
 
         MemberExpr* memberExpr = parser->astBuilder->create<MemberExpr>();
-
+        memberExpr->memberOperatorLoc = opLoc;
         parser->FillPosition(memberExpr);
         memberExpr->baseExpression = base;
         memberExpr->name = expectIdentifier(parser).name;
@@ -2258,13 +2284,17 @@ namespace Slang
                 typeExpr = parseGenericApp(parser, typeExpr);
                 break;
             case TokenType::Scope:
-                parser->ReadToken(TokenType::Scope);
-                typeExpr = parseMemberType(parser, typeExpr);
-                break;
+                {
+                    auto opToken = parser->ReadToken(TokenType::Scope);
+                    typeExpr = parseMemberType(parser, typeExpr, opToken.loc);
+                    break;
+                }
             case TokenType::Dot:
-                parser->ReadToken(TokenType::Dot);
-                typeExpr = parseMemberType(parser, typeExpr);
-                break;
+                {
+                    auto opToken = parser->ReadToken(TokenType::Dot);
+                    typeExpr = parseMemberType(parser, typeExpr, opToken.loc);
+                    break;
+                }
             default:
                 shouldLoop = false;
             }
@@ -3186,6 +3216,10 @@ namespace Slang
         {
             decl->returnType = parser->ParseTypeExp();
         }
+        else
+        {
+            decl->returnType.exp = parser->astBuilder->create<IncompleteExpr>();
+        }
 
         parseStorageDeclBody(parser, decl);
 
@@ -3218,7 +3252,6 @@ namespace Slang
     static NodeBase* parsePropertyDecl(Parser* parser, void* /*userData*/)
     {
         PropertyDecl* decl = parser->astBuilder->create<PropertyDecl>();
-        parser->FillPosition(decl);
         parser->PushScope(decl);
 
         // We want to support property declarations with two
@@ -3244,6 +3277,7 @@ namespace Slang
         //
         if(_peekModernStyleVarDecl(parser))
         {
+            parser->FillPosition(decl);
             decl->nameAndLoc = expectIdentifier(parser);
             expect(parser, TokenType::Colon);
             decl->type = parser->ParseTypeExp();
@@ -3768,10 +3802,12 @@ namespace Slang
         ContainerDecl*      containerDecl,
         MatchedTokenType    matchType)
     {
-        while(!AdvanceIfMatch(parser, matchType))
+        Token closingBraceToken;
+        while (!AdvanceIfMatch(parser, matchType, &closingBraceToken))
         {
             ParseDecl(parser, containerDecl);
         }
+        containerDecl->closingSourceLoc = closingBraceToken.loc;
     }
 
     static void parseDeclBody(
@@ -3823,8 +3859,8 @@ namespace Slang
     Decl* Parser::ParseStruct()
     {
         StructDecl* rs = astBuilder->create<StructDecl>();
-        FillPosition(rs);
         ReadToken("struct");
+        FillPosition(rs);
 
         // The `struct` keyword may optionally be followed by
         // attributes that appertain to the struct declaration
@@ -3857,8 +3893,8 @@ namespace Slang
     ClassDecl* Parser::ParseClass()
     {
         ClassDecl* rs = astBuilder->create<ClassDecl>();
-        FillPosition(rs);
         ReadToken("class");
+        FillPosition(rs);
         rs->nameAndLoc = expectIdentifier(this);
 
         parseOptionalInheritanceClause(this, rs);
@@ -3885,7 +3921,6 @@ namespace Slang
     static Decl* parseEnumDecl(Parser* parser)
     {
         EnumDecl* decl = parser->astBuilder->create<EnumDecl>();
-        parser->FillPosition(decl);
 
         parser->ReadToken("enum");
 
@@ -3897,6 +3932,8 @@ namespace Slang
         //
         AdvanceIf(parser, "class");
 
+        parser->FillPosition(decl);
+
         decl->nameAndLoc = expectIdentifier(parser);
 
 
@@ -4178,6 +4215,10 @@ namespace Slang
         {
             statement = parseCompileTimeStmt(this);
         }
+        else if (LookAheadToken("try"))
+        {
+            statement = ParseExpressionStatement();
+        }
         else if (LookAheadToken(TokenType::Identifier))
         {
             // We might be looking at a local declaration, or an
@@ -5184,7 +5225,7 @@ namespace Slang
         default:
             // TODO: should this return an error expression instead of NULL?
             parser->sink->diagnose(parser->tokenReader.peekLoc(), Diagnostics::syntaxError);
-            return nullptr;
+            return parser->astBuilder->create<IncompleteExpr>();
 
         // Either:
         // - parenthesized expression `(exp)`
@@ -5577,6 +5618,10 @@ namespace Slang
                     {
                         indexExpr->indexExpression = parser->ParseExpression();
                     }
+                    else
+                    {
+                        indexExpr->indexExpression = parser->astBuilder->create<IncompleteExpr>();
+                    }
                     parser->ReadToken(TokenType::RBracket);
 
                     expr = indexExpr;
@@ -5589,7 +5634,8 @@ namespace Slang
                     InvokeExpr* invokeExpr = parser->astBuilder->create<InvokeExpr>();
                     invokeExpr->functionExpr = expr;
                     parser->FillPosition(invokeExpr);
-                    parser->ReadToken(TokenType::LParent);
+                    auto lParen = parser->ReadToken(TokenType::LParent);
+                    invokeExpr->argumentDelimeterLocs.add(lParen.loc);
                     while (!parser->tokenReader.isAtEnd())
                     {
                         if (!parser->LookAheadToken(TokenType::RParent))
@@ -5600,10 +5646,11 @@ namespace Slang
                         }
                         if (!parser->LookAheadToken(TokenType::Comma))
                             break;
-                        parser->ReadToken(TokenType::Comma);
+                        auto comma = parser->ReadToken(TokenType::Comma);
+                        invokeExpr->argumentDelimeterLocs.add(comma.loc);
                     }
-                    parser->ReadToken(TokenType::RParent);
-
+                    auto rParen = parser->ReadToken(TokenType::RParent);
+                    invokeExpr->argumentDelimeterLocs.add(rParen.loc);
                     expr = invokeExpr;
                 }
                 break;
@@ -5615,10 +5662,10 @@ namespace Slang
 
                 // TODO(tfoley): why would a member expression need this?
                 staticMemberExpr->scope = parser->currentScope;
-
-                parser->FillPosition(staticMemberExpr);
+                staticMemberExpr->memberOperatorLoc = parser->tokenReader.peekLoc();
                 staticMemberExpr->baseExpression = expr;
                 parser->ReadToken(TokenType::Scope);
+                parser->FillPosition(staticMemberExpr);
                 staticMemberExpr->name = expectIdentifier(parser).name;
 
                 if (peekTokenType(parser) == TokenType::OpLess)
@@ -5635,10 +5682,10 @@ namespace Slang
 
                     // TODO(tfoley): why would a member expression need this?
                     memberExpr->scope = parser->currentScope;
-
-                    parser->FillPosition(memberExpr);
+                    memberExpr->memberOperatorLoc = parser->tokenReader.peekLoc();
                     memberExpr->baseExpression = expr;
                     parser->ReadToken(TokenType::Dot);
+                    parser->FillPosition(memberExpr);
                     memberExpr->name = expectIdentifier(parser).name;
 
                     if (peekTokenType(parser) == TokenType::OpLess)