Follow up on Language Server Improvement (#2275)

* Fix typo and improve parser recovery.

* Add search path configuration.

Co-authored-by: Yong He <yhe@nvidia.com>

8 files changed, 157 insertions, 17 deletions

diff --git a/source/compiler-core/slang-diagnostic-sink.cpp b/source/compiler-core/slang-diagnostic-sink.cpp
index c934157d2..0110b16d7 100644
--- a/source/compiler-core/slang-diagnostic-sink.cpp
+++ b/source/compiler-core/slang-diagnostic-sink.cpp
@@ -570,7 +570,7 @@ void DiagnosticSink::diagnoseImpl(DiagnosticInfo const& info, const UnownedStrin
         m_parentSink->diagnoseImpl(info, formattedMessage);
     }
 
-    if (getEffectiveMessageSeverity(info) >= Severity::Fatal)
+    if (info.severity >= Severity::Fatal)
     {
         // TODO: figure out a better policy for aborting compilation
         SLANG_ABORT_COMPILATION("");
diff --git a/source/slang/slang-language-server.cpp b/source/slang/slang-language-server.cpp
index 9f7329e2c..5aa8d0e14 100644
--- a/source/slang/slang-language-server.cpp
+++ b/source/slang/slang-language-server.cpp
@@ -150,9 +150,11 @@ SlangResult LanguageServer::parseNextMessage()
                 if (response.result.getKind() == JSONValue::Kind::Array)
                 {
                     auto arr = m_connection->getContainer()->getArray(response.result);
-                    if (arr.getCount() > 0)
+                    if (arr.getCount() == 3)
                     {
                         updatePredefinedMacros(arr[0]);
+                        updateSearchPaths(arr[1]);
+                        updateSearchInWorkspace(arr[2]);
                     }
                 }
                 break;
@@ -791,14 +793,56 @@ void LanguageServer::updatePredefinedMacros(const JSONValue& macros)
     }
 }
 
+void LanguageServer::updateSearchPaths(const JSONValue& macros)
+{
+    if (macros.isValid())
+    {
+        auto container = m_connection->getContainer();
+        JSONToNativeConverter converter(container, m_connection->getSink());
+        List<String> searchPaths;
+        if (SLANG_SUCCEEDED(converter.convert(macros, &searchPaths)))
+        {
+            if (m_workspace->updateSearchPaths(searchPaths))
+            {
+                m_connection->sendCall(
+                    UnownedStringSlice("workspace/semanticTokens/refresh"), JSONValue::makeInt(0));
+            }
+        }
+    }
+}
+
+void LanguageServer::updateSearchInWorkspace(const JSONValue& macros)
+{
+    if (macros.isValid())
+    {
+        auto container = m_connection->getContainer();
+        JSONToNativeConverter converter(container, m_connection->getSink());
+        bool searchPaths;
+        if (SLANG_SUCCEEDED(converter.convert(macros, &searchPaths)))
+        {
+            if (m_workspace->updateSearchInWorkspace(searchPaths))
+            {
+                m_connection->sendCall(
+                    UnownedStringSlice("workspace/semanticTokens/refresh"), JSONValue::makeInt(0));
+            }
+        }
+    }
+}
+
 void LanguageServer::sendConfigRequest()
 {
     ConfigurationParams args;
     ConfigurationItem item;
     item.section = "slang.predefinedMacros";
     args.items.add(item);
+    item.section = "slang.additionalSearchPaths";
+    args.items.add(item);
+    item.section = "slang.searchInAllWorkspaceDirectories";
+    args.items.add(item);
     m_connection->sendCall(
-        ConfigurationParams::methodName, &args, JSONValue::makeInt(kConfigResponseId));
+        ConfigurationParams::methodName,
+        &args,
+        JSONValue::makeInt(kConfigResponseId));
 }
 
 void LanguageServer::registerCapability(const char* methodName)
diff --git a/source/slang/slang-language-server.h b/source/slang/slang-language-server.h
index f9ba2ddfc..898037144 100644
--- a/source/slang/slang-language-server.h
+++ b/source/slang/slang-language-server.h
@@ -110,6 +110,9 @@ private:
     void resetDiagnosticUpdateTime();
     void publishDiagnostics();
     void updatePredefinedMacros(const JSONValue& macros);
+    void updateSearchPaths(const JSONValue& macros);
+    void updateSearchInWorkspace(const JSONValue& macros);
+
     void sendConfigRequest();
     void registerCapability(const char* methodName);
     void logMessage(int type, String message);
diff --git a/source/slang/slang-parser.cpp b/source/slang/slang-parser.cpp
index 38317009d..aaf175812 100644
--- a/source/slang/slang-parser.cpp
+++ b/source/slang/slang-parser.cpp
@@ -100,6 +100,8 @@ namespace Slang
 
         TokenReader tokenReader;
         DiagnosticSink* sink;
+        SourceLoc lastErrorLoc;
+
         int genericDepth = 0;
 
         // Have we seen any `import` declarations? If so, we need
@@ -192,6 +194,18 @@ namespace Slang
         TypeExp										ParseTypeExp();
 
         Parser & operator = (const Parser &) = delete;
+
+        // Helper to issue diagnose message that filters out errors for the same token.
+        template <typename P, typename... Args>
+        void diagnose(P const& pos, DiagnosticInfo const& info, Args const&... args)
+        {
+            auto loc = getDiagnosticPos(pos);
+            if (loc != lastErrorLoc)
+            {
+                sink->diagnose(pos, info, args...);
+                lastErrorLoc = loc;
+            }
+        }
     };
 
     // Forward Declarations
@@ -248,7 +262,7 @@ namespace Slang
         // Don't emit "unexpected token" errors if we are in recovering mode
         if (!parser->isRecovering)
         {
-            parser->sink->diagnose(parser->tokenReader.peekLoc(), Diagnostics::unexpectedToken,
+            parser->diagnose(parser->tokenReader.peekLoc(), Diagnostics::unexpectedToken,
                 parser->tokenReader.peekTokenType());
 
             // Switch into recovery mode, to suppress additional errors
@@ -279,10 +293,15 @@ namespace Slang
         // Don't emit "unexpected token" errors if we are in recovering mode
         if (!parser->isRecovering)
         {
-            parser->sink->diagnose(parser->tokenReader.peekLoc(), Diagnostics::unexpectedTokenExpectedTokenType,
-                parser->tokenReader.peekTokenType(),
-                expected);
-
+            if (parser->lastErrorLoc != parser->tokenReader.peekLoc())
+            {
+                parser->sink->diagnose(
+                    parser->tokenReader.peekLoc(),
+                    Diagnostics::unexpectedTokenExpectedTokenType,
+                    parser->tokenReader.peekTokenType(),
+                    expected);
+                parser->lastErrorLoc = parser->tokenReader.peekLoc();
+            }
             // Switch into recovery mode, to suppress additional errors
             parser->isRecovering = true;
         }
@@ -2515,6 +2534,9 @@ namespace Slang
             if (!AdvanceIf(parser, TokenType::Comma))
             {
                 parser->ReadToken(TokenType::Semicolon);
+                // We don't need to enter recovering mode if next token isn't semicolon.
+                // In this case we just continue parsing the token as the next decl.
+                parser->isRecovering = false;
                 return declGroupBuilder.getResult();
             }
 
@@ -5231,7 +5253,7 @@ namespace Slang
         {
         default:
             // TODO: should this return an error expression instead of NULL?
-            parser->sink->diagnose(parser->tokenReader.peekLoc(), Diagnostics::syntaxError);
+            parser->diagnose(parser->tokenReader.peekLoc(), Diagnostics::syntaxError);
             return parser->astBuilder->create<IncompleteExpr>();
 
         // Either:
diff --git a/source/slang/slang-workspace-version.cpp b/source/slang/slang-workspace-version.cpp
index d93be86b7..2742c083c 100644
--- a/source/slang/slang-workspace-version.cpp
+++ b/source/slang/slang-workspace-version.cpp
@@ -32,7 +32,7 @@ DocumentVersion* Workspace::openDoc(String path, String text)
     doc->setText(text.getUnownedSlice());
     doc->setURI(URI::fromLocalFilePath(path.getUnownedSlice()));
     openedDocuments[path] = doc;
-    searchPaths.Add(Path::getParentDirectory(path));
+    workspaceSearchPaths.Add(Path::getParentDirectory(path));
     moduleCache.invalidate(path);
     invalidate();
     return doc.Ptr();
@@ -66,11 +66,11 @@ void Workspace::closeDoc(const String& path)
 
 bool Workspace::updatePredefinedMacros(List<String> macros)
 {
-    List<OnwedPreprocessorMacroDefinition> newDefs;
+    List<OwnedPreprocessorMacroDefinition> newDefs;
     for (auto macro : macros)
     {
         auto index = macro.indexOf('=');
-        OnwedPreprocessorMacroDefinition def;
+        OwnedPreprocessorMacroDefinition def;
         def.name = macro.getUnownedSlice().head(index).trim();
         if (index != -1)
         {
@@ -102,6 +102,41 @@ bool Workspace::updatePredefinedMacros(List<String> macros)
     return changed;
 }
 
+bool Workspace::updateSearchPaths(List<String> paths)
+{
+    bool changed = false;
+    if (paths.getCount() != additionalSearchPaths.getCount())
+        changed = true;
+    else
+    {
+        for (Index i = 0; i < paths.getCount(); i++)
+        {
+            if (paths[i] != additionalSearchPaths[i])
+            {
+                changed = true;
+                break;
+            }
+        }
+    }
+    if (changed)
+    {
+        additionalSearchPaths = _Move(paths);
+        invalidate();
+    }
+    return changed;
+}
+
+bool Workspace::updateSearchInWorkspace(bool value)
+{
+    bool changed = searchInWorkspace != value;
+    searchInWorkspace = value;
+    if (changed)
+    {
+        invalidate();
+    }
+    return changed;
+}
+
 void Workspace::init(List<URI> rootDirURI, slang::IGlobalSession* globalSession)
 {
     for (auto uri : rootDirURI)
@@ -132,7 +167,7 @@ void Workspace::init(List<URI> rootDirURI, slang::IGlobalSession* globalSession)
                 },
                 &context);
         }
-        searchPaths = _Move(context.paths);
+        workspaceSearchPaths = _Move(context.paths);
     }
     slangGlobalSession = globalSession;
 }
@@ -239,8 +274,23 @@ RefPtr<WorkspaceVersion> Workspace::createWorkspaceVersion()
     targetDesc.profile = slangGlobalSession->findProfile("sm_6_6");
     desc.targets = &targetDesc;
     List<const char*> searchPathsRaw;
-    for (auto path : searchPaths)
+    for (auto& path : additionalSearchPaths)
         searchPathsRaw.add(path.getBuffer());
+    if (searchInWorkspace)
+    {
+        for (auto& path : workspaceSearchPaths)
+            searchPathsRaw.add(path.getBuffer());
+    }
+    else
+    {
+        HashSet<String> set;
+        for (auto& p : openedDocuments)
+        {
+            auto dir = Path::getParentDirectory(p.Key.getBuffer());
+            if (set.Add(dir))
+                searchPathsRaw.add(dir.getBuffer());
+        }
+    }
     desc.searchPaths = searchPathsRaw.getBuffer();
     desc.searchPathCount = searchPathsRaw.getCount();
 
diff --git a/source/slang/slang-workspace-version.h b/source/slang/slang-workspace-version.h
index ee4d51896..dea914bb5 100644
--- a/source/slang/slang-workspace-version.h
+++ b/source/slang/slang-workspace-version.h
@@ -124,7 +124,7 @@ namespace Slang
         Module* getOrLoadModule(String path);
     };
 
-    struct OnwedPreprocessorMacroDefinition
+    struct OwnedPreprocessorMacroDefinition
     {
         String name;
         String value;
@@ -138,9 +138,11 @@ namespace Slang
         RefPtr<WorkspaceVersion> createWorkspaceVersion();
     public:
         List<String> rootDirectories;
-        OrderedHashSet<String> searchPaths;
-        List<OnwedPreprocessorMacroDefinition> predefinedMacros;
+        List<String> additionalSearchPaths;
+        OrderedHashSet<String> workspaceSearchPaths;
+        List<OwnedPreprocessorMacroDefinition> predefinedMacros;
         SerializedModuleCache moduleCache;
+        bool searchInWorkspace = true;
 
         slang::IGlobalSession* slangGlobalSession;
         Dictionary<String, RefPtr<DocumentVersion>> openedDocuments;
@@ -150,6 +152,8 @@ namespace Slang
 
         // Update predefined macro settings. Returns true if the new settings are different from existing ones.
         bool updatePredefinedMacros(List<String> predefinedMacros);
+        bool updateSearchPaths(List<String> searchPaths);
+        bool updateSearchInWorkspace(bool value);
 
         void init(List<URI> rootDirURI, slang::IGlobalSession* globalSession);
         void invalidate();
diff --git a/tests/language-server/member-completion-broken-syntax-2.slang b/tests/language-server/member-completion-broken-syntax-2.slang
new file mode 100644
index 000000000..adafceaa6
--- /dev/null
+++ b/tests/language-server/member-completion-broken-syntax-2.slang
@@ -0,0 +1,14 @@
+//TEST:LANG_SERVER:
+//COMPLETE:13,7
+UNDEFINED_MACRO(myGlobalParameterDefinedWithMacro)
+UNDEFINED_MACRO(myGlobalParameterDefinedWithMacro2)
+
+struct MyType
+{
+    int getSum() { return 0; }
+}
+
+void m(MyType t)
+{
+    t.
+}
diff --git a/tests/language-server/member-completion-broken-syntax-2.slang.expected.txt b/tests/language-server/member-completion-broken-syntax-2.slang.expected.txt
new file mode 100644
index 000000000..5faa1d323
--- /dev/null
+++ b/tests/language-server/member-completion-broken-syntax-2.slang.expected.txt
@@ -0,0 +1,3 @@
+--------
+getSum: 2  ,.;:()[]<>{}*&^%!-=+|/? 
+