Language Server improvements. (#2269)

* Language Server improvements.
- Improve parser robustness around `attribute_syntax`.
- Exclude instance members in a static query.
- Coloring accessors
- Improved signature help cursor range check.

* Add expected test result.

* Language server: support configuring predefined macros.

* Fix constructor highlighting.

* Improving performance by supporting incremental text change notifications.

* Fix UTF16 positions and highlighting of constructor calls.

* Add completion suggestions for HLSL semantics.

* Fix tests.

* Fix: don't skip static variables in a static query.

* Include literal init expr value in hover text.

* Fix scenarios where completion failed to trigger.

* Fixing language server protocol field initializations.

Co-authored-by: Yong He <yhe@nvidia.com>

1 files changed, 242 insertions, 11 deletions

diff --git a/source/slang/slang-workspace-version.cpp b/source/slang/slang-workspace-version.cpp
index 1e92c31ca..d93be86b7 100644
--- a/source/slang/slang-workspace-version.cpp
+++ b/source/slang/slang-workspace-version.cpp
@@ -2,6 +2,7 @@
 #include "../core/slang-io.h"
 #include "../core/slang-file-system.h"
 #include "../compiler-core/slang-lexer.h"
+#include "slang-serialize-container.h"
 
 namespace Slang
 {
@@ -32,10 +33,75 @@ DocumentVersion* Workspace::openDoc(String path, String text)
     doc->setURI(URI::fromLocalFilePath(path.getUnownedSlice()));
     openedDocuments[path] = doc;
     searchPaths.Add(Path::getParentDirectory(path));
+    moduleCache.invalidate(path);
     invalidate();
     return doc.Ptr();
 }
 
+void Workspace::changeDoc(const String& path, LanguageServerProtocol::Range range, const String& text)
+{
+    RefPtr<DocumentVersion> doc;
+    if (openedDocuments.TryGetValue(path, doc))
+    {
+        Index line, col;
+        doc->zeroBasedUTF16LocToOneBasedUTF8Loc(range.start.line, range.start.character, line, col);
+        auto startOffset = doc->getOffset(line, col);
+        doc->zeroBasedUTF16LocToOneBasedUTF8Loc(range.end.line, range.end.character, line, col);
+        auto endOffset = doc->getOffset(line, col);
+        auto originalText = doc->getText().getUnownedSlice();
+        StringBuilder newText;
+        newText << originalText.head(startOffset) << text << originalText.tail(endOffset);
+        doc->setText(newText.ProduceString());
+    }
+    moduleCache.invalidate(path);
+    invalidate();
+}
+
+void Workspace::closeDoc(const String& path)
+{
+    moduleCache.invalidate(path);
+    openedDocuments.Remove(path);
+    invalidate();
+}
+
+bool Workspace::updatePredefinedMacros(List<String> macros)
+{
+    List<OnwedPreprocessorMacroDefinition> newDefs;
+    for (auto macro : macros)
+    {
+        auto index = macro.indexOf('=');
+        OnwedPreprocessorMacroDefinition def;
+        def.name = macro.getUnownedSlice().head(index).trim();
+        if (index != -1)
+        {
+            def.value = macro.getUnownedSlice().tail(index + 1).trim();
+        }
+        newDefs.add(def);
+    }
+    
+    bool changed = false;
+    if (newDefs.getCount() != predefinedMacros.getCount())
+        changed = true;
+    else
+    {
+        for (Index i = 0; i < newDefs.getCount(); i++)
+        {
+            if (newDefs[i].name != predefinedMacros[i].name ||
+                newDefs[i].value != predefinedMacros[i].value)
+            {
+                changed = true;
+                break;
+            }
+        }
+    }
+    if (changed)
+    {
+        predefinedMacros = _Move(newDefs);
+        invalidate();
+    }
+    return changed;
+}
+
 void Workspace::init(List<URI> rootDirURI, slang::IGlobalSession* globalSession)
 {
     for (auto uri : rootDirURI)
@@ -73,7 +139,7 @@ void Workspace::init(List<URI> rootDirURI, slang::IGlobalSession* globalSession)
 
 void Workspace::invalidate() { currentVersion = nullptr; }
 
-void parseDiagnostics(Dictionary<String, DocumentDiagnostics>& diagnostics, String compilerOutput)
+void WorkspaceVersion::parseDiagnostics(String compilerOutput)
 {
     List<UnownedStringSlice> lines;
     StringUtil::calcLines(compilerOutput.getUnownedSlice(), lines);
@@ -95,12 +161,12 @@ void parseDiagnostics(Dictionary<String, DocumentDiagnostics>& diagnostics, Stri
         int lineLoc = StringUtil::parseIntAndAdvancePos(line, pos);
         if (lineLoc == 0)
             lineLoc = 1;
-        diagnostic.range.end.line = diagnostic.range.start.line = lineLoc - 1;
+        diagnostic.range.end.line = diagnostic.range.start.line = lineLoc;
         pos++;
         int colLoc = StringUtil::parseIntAndAdvancePos(line, pos);
         if (colLoc == 0)
             colLoc = 1;
-        diagnostic.range.end.character = diagnostic.range.start.character = colLoc - 1;
+        diagnostic.range.end.character = diagnostic.range.start.character = colLoc;
         if (pos >= line.getLength())
             continue;
         line = line.tail(colonIndex + 3);
@@ -133,7 +199,31 @@ void parseDiagnostics(Dictionary<String, DocumentDiagnostics>& diagnostics, Stri
             auto tokenLength = StringUtil::parseIntAndAdvancePos(line, pos);
             diagnostic.range.end.character += tokenLength;
         }
+
+        if (auto doc = workspace->openedDocuments.TryGetValue(fileName))
+        {
+            // If the file is open, translate to UTF16 positions using the document.
+            Index lineUTF16, colUTF16;
+            doc->Ptr()->oneBasedUTF8LocToZeroBasedUTF16Loc(
+                diagnostic.range.start.line, diagnostic.range.start.character, lineUTF16, colUTF16);
+            diagnostic.range.start.line = (int)lineUTF16;
+            diagnostic.range.start.character = (int)colUTF16;
+            doc->Ptr()->oneBasedUTF8LocToZeroBasedUTF16Loc(
+                diagnostic.range.end.line, diagnostic.range.end.character, lineUTF16, colUTF16);
+            diagnostic.range.end.line = (int)lineUTF16;
+            diagnostic.range.end.character = (int)colUTF16;
+        }
+        else
+        {
+            // Otherwise, just return an 0-based position.
+            diagnostic.range.start.line--;
+            diagnostic.range.start.character--;
+            diagnostic.range.end.line--;
+            diagnostic.range.end.character--;
+        }
         diagnosticList.messages.Add(diagnostic);
+        if (diagnosticList.messages.Count() >= 1000)
+            break;
     }
 }
 
@@ -148,17 +238,33 @@ RefPtr<WorkspaceVersion> Workspace::createWorkspaceVersion()
     slang::TargetDesc targetDesc = {};
     targetDesc.profile = slangGlobalSession->findProfile("sm_6_6");
     desc.targets = &targetDesc;
-
     List<const char*> searchPathsRaw;
-    
     for (auto path : searchPaths)
         searchPathsRaw.add(path.getBuffer());
     desc.searchPaths = searchPathsRaw.getBuffer();
     desc.searchPathCount = searchPathsRaw.getCount();
 
+    desc.preprocessorMacroCount = predefinedMacros.getCount();
+    List<slang::PreprocessorMacroDesc> macroDescs;
+    for (auto& macro : predefinedMacros)
+    {
+        slang::PreprocessorMacroDesc macroDesc;
+        macroDesc.name = macro.name.getBuffer();
+        macroDesc.value = macro.value.getBuffer();
+        macroDescs.add(macroDesc);
+    }
+    desc.preprocessorMacros = macroDescs.getBuffer();
+
     ComPtr<slang::ISession> session;
     slangGlobalSession->createSession(desc, session.writeRef());
     version->linkage = static_cast<Linkage*>(session.get());
+    // TODO(yong): module cache does improves performance by 30%. However there are some issues
+    // that prevents the deserialization to resolve the imported decls from the correct module.
+    // This doesn't lead to crash, but may cause problems. We can enable this when the issues
+    // are fixed.
+#if 0
+    version->linkage->setModuleCache(&moduleCache);
+#endif
     return version;
 }
 
@@ -194,14 +300,79 @@ void* Workspace::getInterface(const Guid& uuid)
 void DocumentVersion::setText(const String& newText)
 {
     text = newText;
-    lineBreaks.clear();
-    for (Index i = 0; i < newText.getLength(); i++)
+    StringUtil::calcLines(text.getUnownedSlice(), lines);
+    utf16CharStarts.clear();
+}
+ArrayView<Index> DocumentVersion::getUTF16Boundaries(Index line)
+{
+    if (!utf16CharStarts.getCount())
+    {
+        for (auto slice : lines)
+        {
+            List<Index> bounds;
+            Index index = 0;
+            while (index < slice.getLength())
+            {
+                auto startIndex = index;
+                const Char32 codePoint = getUnicodePointFromUTF8(
+                    [&]() -> Byte
+                    {
+                        if (index < slice.getLength())
+                            return slice[index++];
+                        else
+                            return '\0';
+                    });
+                if (!codePoint)
+                    break;
+                Char16 buffer[2];
+                int count = encodeUnicodePointToUTF16Reversed(codePoint, buffer);
+                for (int i = 0; i < count; i++)
+                    bounds.add(startIndex);
+            }
+            bounds.add(slice.getLength());
+            utf16CharStarts.add(_Move(bounds));
+        }
+    }
+    return line >= 1 && line <= utf16CharStarts.getCount() ? utf16CharStarts[line - 1].getArrayView()
+                                                           : ArrayView<Index>();
+}
+
+void DocumentVersion::oneBasedUTF8LocToZeroBasedUTF16Loc(
+    Index inLine, Index inCol, Index& outLine, Index& outCol)
+{
+    Index rsLine = inLine - 1;
+    auto line = lines[rsLine];
+    auto bounds = getUTF16Boundaries(inLine);
+    outLine = rsLine;
+    outCol = std::lower_bound(bounds.begin(), bounds.end(), inCol - 1) - bounds.begin();
+}
+
+void DocumentVersion::zeroBasedUTF16LocToOneBasedUTF8Loc(
+    Index inLine, Index inCol, Index& outLine, Index& outCol)
+{
+    outLine = inLine + 1;
+    auto bounds = getUTF16Boundaries(inLine + 1);
+    outCol = inCol >=0 && inCol < bounds.getCount()? bounds[inCol] + 1 : 0;
+}
+
+static bool _isIdentifierChar(char ch)
+{
+    return ch >= 'a' && ch <= 'z' || ch >= 'A' && ch <= 'Z' || ch >= '0' && ch <= '9' || ch == '_';
+}
+
+int DocumentVersion::getTokenLength(Index line, Index col)
+{
+    auto offset = getOffset(line, col);
+    if (offset >= 0)
     {
-        if (newText[i] == '\n')
-            lineBreaks.add(i);
+        Index pos = offset;
+        for (; pos < text.getLength() && _isIdentifierChar(text[pos]); ++pos)
+        {}
+        return (int)(pos - offset);
     }
-    lineBreaks.add(newText.getLength());
+    return 0;
 }
+
 ASTMarkup* WorkspaceVersion::getOrCreateMarkupAST(ModuleDecl* module)
 {
     RefPtr<ASTMarkup> astMarkup;
@@ -209,6 +380,7 @@ ASTMarkup* WorkspaceVersion::getOrCreateMarkupAST(ModuleDecl* module)
         return astMarkup.Ptr();
     DiagnosticSink sink;
     astMarkup = new ASTMarkup();
+    sink.setSourceManager(linkage->getSourceManager());
     ASTMarkupUtil::extract(module, linkage->getSourceManager(), &sink, astMarkup.Ptr());
     markupASTs[module] = astMarkup;
     return astMarkup.Ptr();
@@ -238,7 +410,7 @@ Module* WorkspaceVersion::getOrLoadModule(String path)
     if (diagnosticBlob)
     {
         auto diagnosticString = String((const char*)diagnosticBlob->getBufferPointer());
-        parseDiagnostics(diagnostics, diagnosticString);
+        parseDiagnostics(diagnosticString);
         auto docDiagnostic = diagnostics.TryGetValue(path);
         if (docDiagnostic)
             docDiagnostic->originalOutput = diagnosticString;
@@ -246,4 +418,63 @@ Module* WorkspaceVersion::getOrLoadModule(String path)
     return static_cast<Module*>(parsedModule);
 }
 
+RefPtr<Module> SerializedModuleCache::tryLoadModule(
+    Linkage* linkage, String filePath)
+{
+    Path::getCanonical(filePath, filePath);
+    if (List<uint8_t>* rawData = serializedModules.TryGetValue(filePath))
+    {
+        RefPtr<MemoryStreamBase> memStream =
+            new MemoryStreamBase(FileAccess::Read, rawData->getBuffer(), rawData->getCount());
+        RiffContainer riffContainer;
+        RiffUtil::read(memStream.Ptr(), riffContainer);
+        SerialContainerData outData;
+        SerialContainerUtil::ReadOptions options;
+        options.linkage = linkage;
+        options.namePool = linkage->getNamePool();
+        options.session = linkage->getSessionImpl();
+        options.sharedASTBuilder = linkage->getASTBuilder()->getSharedASTBuilder();
+        options.astBuilder = linkage->getASTBuilder();
+        DiagnosticSink sink(linkage->getSourceManager(), Lexer::sourceLocationLexer);
+        options.sink = &sink;
+        options.sourceManager = linkage->getSourceManager();
+        SLANG_RETURN_NULL_ON_FAIL(SerialContainerUtil::read(&riffContainer, options, outData));
+        if (outData.modules.getCount() == 1)
+        {
+            RefPtr<Module> module = new Module(linkage, linkage->getASTBuilder());
+            auto moduleDecl = as<ModuleDecl>(outData.modules[0].astRootNode);
+            if (moduleDecl)
+            {
+                moduleDecl->module = module.Ptr();
+                module->setModuleDecl(moduleDecl);
+                return module;
+            }
+        }
+    }
+    return nullptr;
+}
+
+void SerializedModuleCache::storeModule(
+    Linkage* linkage, String filePath, RefPtr<Module> module)
+{
+    Path::getCanonical(filePath, filePath);
+    RiffContainer container;
+    SerialContainerUtil::WriteOptions options;
+    options.sourceManager = linkage->getSourceManager();
+    options.compressionType = SerialCompressionType::None;
+    options.optionFlags = SerialOptionFlag::SourceLocation | SerialOptionFlag::ASTModule;
+    SerialContainerData data;
+    SerialContainerData::Module moduleData;
+    moduleData.astBuilder = linkage->getASTBuilder();
+    moduleData.astRootNode = module->getModuleDecl();
+    moduleData.irModule = nullptr;
+    data.modules.add(moduleData);
+    SerialContainerUtil::write(data, options, &container);
+    RefPtr<OwnedMemoryStream> memStream = new OwnedMemoryStream(FileAccess::Write);
+    RiffUtil::write(&container, memStream);
+    List<uint8_t> rawData;
+    memStream->swapContents(rawData);
+    serializedModules[filePath] = _Move(rawData);
+}
+
 } // namespace Slang