Major language server features. (#2264)

* Major language server features.

* Include slangd in binary release.

* Fix compiler issues.

* Fix compiler error.

* Completion resolve.

* Various improvements.

* Update diagnostic test expected output.

* Bug fix for source locations.

* Adjust diagnostic update frequency.

* Update github actions to store artifacts.

* Fix infinite parser loop.

* Fix parser recovery.

* Fix parser recovery.

* Update test.

* Fix test.

* Disable IR gen for language server.

* Allow commit characters in auto completion.

* Fix lookup for invoke exprs.

* More parser robustness fixes.

* update solution file

Co-authored-by: Yong He <yhe@nvidia.com>

5 files changed, 25 insertions, 558 deletions

diff --git a/tools/slangd/language-server-protocol.cpp b/tools/slangd/language-server-protocol.cpp
deleted file mode 100644
index eca89fd86..000000000
--- a/tools/slangd/language-server-protocol.cpp
+++ /dev/null
@@ -1,191 +0,0 @@
-#include "language-server-protocol.h"
-
-namespace Slang
-{
-namespace LanguageServerProtocol
-{
-static const StructRttiInfo _makeTextDocumentSyncOptionsRtti()
-{
-    TextDocumentSyncOptions obj;
-    StructRttiBuilder builder(&obj, "LanguageServerProtocol::TextDocumentSyncOptions", nullptr);
-    builder.addField("change", &obj.change);
-    builder.addField("openClose", &obj.openClose);
-    return builder.make();
-}
-const StructRttiInfo TextDocumentSyncOptions::g_rttiInfo = _makeTextDocumentSyncOptionsRtti();
-
-static const StructRttiInfo _makeTextDocumentItemRtti()
-{
-    TextDocumentItem obj;
-    StructRttiBuilder builder(&obj, "LanguageServerProtocol::TextDocumentItem", nullptr);
-    builder.addField("uri", &obj.uri);
-    builder.addField("version", &obj.version);
-    builder.addField("languageId", &obj.languageId);
-    builder.addField("text", &obj.text);
-    return builder.make();
-}
-const StructRttiInfo TextDocumentItem::g_rttiInfo = _makeTextDocumentItemRtti();
-
-static const StructRttiInfo _makeTextDocumentIdentifierRtti()
-{
-    TextDocumentIdentifier obj;
-    StructRttiBuilder builder(&obj, "LanguageServerProtocol::TextDocumentIdentifier", nullptr);
-    builder.addField("uri", &obj.uri);
-    return builder.make();
-}
-const StructRttiInfo TextDocumentIdentifier::g_rttiInfo = _makeTextDocumentIdentifierRtti();
-
-static const StructRttiInfo _makeVersionedTextDocumentIdentifierRtti()
-{
-    VersionedTextDocumentIdentifier obj;
-    StructRttiBuilder builder(&obj, "LanguageServerProtocol::VersionedTextDocumentIdentifier", nullptr);
-    builder.addField("uri", &obj.uri);
-    builder.addField("version", &obj.version);
-    return builder.make();
-}
-const StructRttiInfo VersionedTextDocumentIdentifier::g_rttiInfo =
-    _makeVersionedTextDocumentIdentifierRtti();
-
-static const StructRttiInfo _makePositionRtti()
-{
-    Position obj;
-    StructRttiBuilder builder(
-        &obj, "LanguageServerProtocol::Position", nullptr);
-    builder.addField("line", &obj.line);
-    builder.addField("character", &obj.character);
-    return builder.make();
-}
-const StructRttiInfo Position::g_rttiInfo = _makePositionRtti();
-
-static const StructRttiInfo _makeRangeRtti()
-{
-    Range obj;
-    StructRttiBuilder builder(&obj, "LanguageServerProtocol::Range", nullptr);
-    builder.addField("start", &obj.start);
-    builder.addField("end", &obj.end);
-    return builder.make();
-}
-const StructRttiInfo Range::g_rttiInfo = _makeRangeRtti();
-
-static const StructRttiInfo _makeDidOpenTextDocumentRtti()
-{
-    DidOpenTextDocumentParams obj;
-    StructRttiBuilder builder(&obj, "LanguageServerProtocol::DidOpenTextDocumentParams", nullptr);
-    builder.addField("textDocument", &obj.textDocument);
-    return builder.make();
-}
-const StructRttiInfo DidOpenTextDocumentParams::g_rttiInfo = _makeDidOpenTextDocumentRtti();
-const UnownedStringSlice DidOpenTextDocumentParams::methodName =
-    UnownedStringSlice::fromLiteral("textDocument/didOpen");
-
-static const StructRttiInfo _makeTextDocumentContentChangeEventRtti()
-{
-    TextDocumentContentChangeEvent obj;
-    StructRttiBuilder builder(&obj, "LanguageServerProtocol::TextDocumentContentChangeEvent", nullptr);
-    builder.addField("range", &obj.range, StructRttiInfo::Flag::Optional);
-    builder.addField("text", &obj.text);
-    return builder.make();
-}
-const StructRttiInfo TextDocumentContentChangeEvent::g_rttiInfo =
-    _makeTextDocumentContentChangeEventRtti();
-
-static const StructRttiInfo _makeDidChangeTextDocumentParamsRtti()
-{
-    DidChangeTextDocumentParams obj;
-    StructRttiBuilder builder(
-        &obj, "LanguageServerProtocol::DidChangeTextDocumentParams", nullptr);
-    builder.addField("textDocument", &obj.textDocument);
-    builder.addField("contentChanges", &obj.contentChanges);
-    return builder.make();
-}
-const StructRttiInfo DidChangeTextDocumentParams::g_rttiInfo =
-    _makeDidChangeTextDocumentParamsRtti();
-const UnownedStringSlice DidChangeTextDocumentParams::methodName =
-    UnownedStringSlice::fromLiteral("textDocument/didChange");
-
-
-static const StructRttiInfo _makeDidCloseTextDocumentParamsRtti()
-{
-    DidCloseTextDocumentParams obj;
-    StructRttiBuilder builder(&obj, "LanguageServerProtocol::DidCloseTextDocumentParams", nullptr);
-    builder.addField("textDocument", &obj.textDocument);
-    return builder.make();
-}
-const StructRttiInfo DidCloseTextDocumentParams::g_rttiInfo = _makeDidCloseTextDocumentParamsRtti();
-const UnownedStringSlice DidCloseTextDocumentParams::methodName =
-    UnownedStringSlice::fromLiteral("textDocument/didClose");
-
-static const StructRttiInfo _makeServerCapabilitiesRtti()
-{
-    ServerCapabilities obj;
-    StructRttiBuilder builder(&obj, "LanguageServerProtocol::ServerCapabilities", nullptr);
-    builder.addField("positionEncoding", &obj.positionEncoding);
-    builder.addField("textDocumentSync", &obj.textDocumentSync);
-    return builder.make();
-}
-const StructRttiInfo ServerCapabilities::g_rttiInfo = _makeServerCapabilitiesRtti();
-
-static const StructRttiInfo _makeServerInfoRtti()
-{
-    ServerInfo obj;
-    StructRttiBuilder builder(&obj, "LanguageServerProtocol::ServerInfo", nullptr);
-    builder.addField("name", &obj.name);
-    builder.addField("version", &obj.version);
-    return builder.make();
-}
-const StructRttiInfo ServerInfo::g_rttiInfo = _makeServerInfoRtti();
-
-
-static const StructRttiInfo _makeInitializeResultRtti()
-{
-    InitializeResult obj;
-    StructRttiBuilder builder(&obj, "LanguageServerProtocol::InitializeResult", nullptr);
-    builder.addField("capabilities", &obj.capabilities);
-    builder.addField("serverInfo", &obj.serverInfo);
-    return builder.make();
-}
-const StructRttiInfo InitializeResult::g_rttiInfo = _makeInitializeResultRtti();
-
-const UnownedStringSlice InitializeParams::methodName =
-    UnownedStringSlice::fromLiteral("initialize");
-
-const UnownedStringSlice ShutdownParams::methodName = UnownedStringSlice::fromLiteral("shutdown");
-
-const UnownedStringSlice ExitParams::methodName = UnownedStringSlice::fromLiteral("exit");
-
-static const StructRttiInfo _makeWorkspaceFolderRtti()
-{
-    WorkspaceFolder obj;
-    StructRttiBuilder builder(&obj, "LanguageServerProtocol::WorkspaceFolder", nullptr);
-    builder.addField("uri", &obj.uri);
-    builder.addField("name", &obj.name);
-    return builder.make();
-}
-const StructRttiInfo WorkspaceFolder::g_rttiInfo = _makeWorkspaceFolderRtti();
-
-static const StructRttiInfo _makeInitializeParamsRtti()
-{
-    InitializeParams obj;
-    StructRttiBuilder builder(&obj, "LanguageServerProtocol::InitializeParams", nullptr);
-    builder.addField("workspaceFolders", &obj.workspaceFolders, StructRttiInfo::Flag::Optional);
-    return builder.make();
-}
-const StructRttiInfo InitializeParams::g_rttiInfo = _makeInitializeParamsRtti();
-
-static const StructRttiInfo _makeNullResponseRtti()
-{
-    NullResponse obj;
-    StructRttiBuilder builder(&obj, "LanguageServerProtocol::NullResponse", nullptr);
-    return builder.make();
-}
-const StructRttiInfo NullResponse::g_rttiInfo = _makeNullResponseRtti();
-
-NullResponse* NullResponse::get()
-{
-    static NullResponse result = {};
-    return &result;
-}
-
-} // namespace LanguageServerProtocol
-
-}
diff --git a/tools/slangd/language-server-protocol.h b/tools/slangd/language-server-protocol.h
deleted file mode 100644
index c76f91d5c..000000000
--- a/tools/slangd/language-server-protocol.h
+++ /dev/null
@@ -1,148 +0,0 @@
-#pragma once
-
-#include "../../slang-com-helper.h"
-#include "../../slang-com-ptr.h"
-#include "../../slang.h"
-
-#include "../../source/core/slang-rtti-info.h"
-#include "../../source/compiler-core/slang-json-value.h"
-
-namespace Slang
-{
-namespace LanguageServerProtocol
-{
-    struct ServerInfo
-    {
-        String name;
-        String version;
-
-        static const StructRttiInfo g_rttiInfo;
-    };
-
-    enum class TextDocumentSyncKind
-    {
-        None = 0,
-        Full = 1,
-        Incremental = 2
-    };
-
-    struct TextDocumentSyncOptions
-    {
-        bool openClose;
-        int32_t change; // TextDocumentSyncKind 
-        static const StructRttiInfo g_rttiInfo;
-    };
-
-    struct TextDocumentItem
-    {
-        String uri;
-        String languageId;
-        int version;
-        String text;
-        static const StructRttiInfo g_rttiInfo;
-    };
-
-    struct TextDocumentIdentifier
-    {
-        String uri;
-        static const StructRttiInfo g_rttiInfo;
-    };
-
-    struct VersionedTextDocumentIdentifier
-    {
-        String uri;
-        int version;
-        static const StructRttiInfo g_rttiInfo;
-    };
-
-    struct Position
-    {
-        int line = -1;
-        int character = -1;
-        static const StructRttiInfo g_rttiInfo;
-    };
-
-    struct Range
-    {
-        Position start;
-        Position end;
-        static const StructRttiInfo g_rttiInfo;
-    };
-
-    struct DidOpenTextDocumentParams
-    {
-        TextDocumentItem textDocument;
-        static const StructRttiInfo g_rttiInfo;
-        static const UnownedStringSlice methodName;
-    };
-
-    struct TextDocumentContentChangeEvent
-    {
-        Range range; // optional
-        String text;
-        static const StructRttiInfo g_rttiInfo;
-    };
-
-    struct DidChangeTextDocumentParams
-    {
-        VersionedTextDocumentIdentifier textDocument;
-        List<TextDocumentContentChangeEvent> contentChanges;
-        static const StructRttiInfo g_rttiInfo;
-        static const UnownedStringSlice methodName;
-
-    };
-
-    struct DidCloseTextDocumentParams
-    {
-        TextDocumentIdentifier textDocument;
-        static const StructRttiInfo g_rttiInfo;
-        static const UnownedStringSlice methodName;
-    };
-
-    struct ServerCapabilities
-    {
-        String positionEncoding;
-        TextDocumentSyncOptions textDocumentSync;
-        static const StructRttiInfo g_rttiInfo;
-    };
-
-    struct WorkspaceFolder
-    {
-        String uri;
-        String name;
-        static const StructRttiInfo g_rttiInfo;
-    };
-
-    struct InitializeParams
-    {
-        List<WorkspaceFolder> workspaceFolders;
-        static const UnownedStringSlice methodName;
-        static const StructRttiInfo g_rttiInfo;
-    };
-
-    struct NullResponse
-    {
-        static const StructRttiInfo g_rttiInfo;
-        static NullResponse* get();
-    };
-
-    struct InitializeResult
-    {
-        ServerCapabilities capabilities;
-        ServerInfo serverInfo;
-
-        static const StructRttiInfo g_rttiInfo;
-    };
-
-    struct ShutdownParams
-    {
-        static const UnownedStringSlice methodName;
-    };
-
-    struct ExitParams
-    {
-        static const UnownedStringSlice methodName;
-    };
-
-}
-} // namespace LanguageServerProtocol
diff --git a/tools/slangd/language-server.cpp b/tools/slangd/language-server.cpp
deleted file mode 100644
index 7cb744ecd..000000000
--- a/tools/slangd/language-server.cpp
+++ /dev/null
@@ -1,184 +0,0 @@
-// language-server.cpp
-
-// This file implements the language server for Slang, conforming to the Language Server Protocol.
-// https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/
-
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <thread>
-
-#include "../../source/core/slang-secure-crt.h"
-#include "../../slang-com-helper.h"
-
-#include "language-server-protocol.h"
-#include "language-server.h"
-
-namespace Slang
-{
-using namespace LanguageServerProtocol;
-
-SlangResult LanguageServer::init(const InitializeParams& args)
-{
-    SLANG_RETURN_ON_FAIL(m_connection->initWithStdStreams());
-    m_workspaceFolders = args.workspaceFolders;
-    return SLANG_OK;
-}
-
-slang::IGlobalSession* LanguageServer::getOrCreateGlobalSession()
-{
-    if (!m_session)
-    {
-        // Just create the global session in the regular way if there isn't one set
-        if (SLANG_FAILED(slang_createGlobalSession(SLANG_API_VERSION, m_session.writeRef())))
-        {
-            return nullptr;
-        }
-    }
-
-    return m_session;
-}
-
-SlangResult LanguageServer::_executeSingle()
-{
-    // If we don't have a message, we can quit for now
-    if (!m_connection->hasMessage())
-    {
-        return SLANG_OK;
-    }
-
-    const JSONRPCMessageType msgType = m_connection->getMessageType();
-
-    switch (msgType)
-    {
-    case JSONRPCMessageType::Call:
-        {
-            JSONRPCCall call;
-            SLANG_RETURN_ON_FAIL(m_connection->getRPCOrSendError(&call));
-
-            // Do different things
-            if (call.method == ExitParams::methodName)
-            {
-                m_quit = true;
-                return SLANG_OK;
-            }
-            else if (call.method == ShutdownParams::methodName)
-            {
-                m_connection->sendResult(NullResponse::get(), call.id);
-                return SLANG_OK;
-            }
-            else if (call.method == InitializeParams::methodName)
-            {
-                InitializeParams args = {};
-                SLANG_RETURN_ON_FAIL(
-                    m_connection->toNativeArgsOrSendError(call.params, &args, call.id));
-
-                init(args);
-
-                InitializeResult result = {};
-                result.serverInfo.name = "SlangLanguageServer";
-                result.serverInfo.version = "1.0";
-                result.capabilities.positionEncoding = "utf-8";
-                result.capabilities.textDocumentSync.openClose = true;
-                result.capabilities.textDocumentSync.change = (int)TextDocumentSyncKind::Full;
-                m_connection->sendResult(&result, call.id);
-                return SLANG_OK;
-            }
-            else if (call.method == DidOpenTextDocumentParams::methodName)
-            {
-                DidOpenTextDocumentParams args;
-                SLANG_RETURN_ON_FAIL(
-                    m_connection->toNativeArgsOrSendError(call.params, &args, call.id));
-                return didOpenTextDocument(args);
-            }
-            else if (call.method == DidCloseTextDocumentParams::methodName)
-            {
-                DidCloseTextDocumentParams args;
-                SLANG_RETURN_ON_FAIL(
-                    m_connection->toNativeArgsOrSendError(call.params, &args, call.id));
-                return didCloseTextDocument(args);
-            }
-            else if (call.method == DidChangeTextDocumentParams::methodName)
-            {
-                DidChangeTextDocumentParams args;
-                SLANG_RETURN_ON_FAIL(
-                    m_connection->toNativeArgsOrSendError(call.params, &args, call.id));
-                return didChangeTextDocument(args);
-            }
-            else if (call.method == "initialized")
-            {
-                return SLANG_OK;
-            }
-            else
-            {
-                return m_connection->sendError(JSONRPC::ErrorCode::MethodNotFound, call.id);
-            }
-        }
-    default:
-        {
-            return m_connection->sendError(
-                JSONRPC::ErrorCode::InvalidRequest, m_connection->getCurrentMessageId());
-        }
-    }
-
-    return SLANG_OK;
-}
-
-SlangResult LanguageServer::didOpenTextDocument(const DidOpenTextDocumentParams& args)
-{
-    return SLANG_OK;
-}
-SlangResult LanguageServer::didCloseTextDocument(const DidCloseTextDocumentParams& args)
-{
-    return SLANG_OK;
-}
-SlangResult LanguageServer::didChangeTextDocument(const DidChangeTextDocumentParams& args)
-{
-    return SLANG_OK;
-}
-
-void LanguageServer::update()
-{
-
-}
-
-SlangResult LanguageServer::execute()
-{
-    m_connection = new JSONRPCConnection();
-    m_connection->initWithStdStreams();
-    while (m_connection->isActive() && !m_quit)
-    {
-        // Consume all messages first.
-        while (m_connection->tryReadMessage() == SLANG_OK)
-        {
-            const SlangResult res = _executeSingle();
-        }
-
-        // Now we can use this time to reparse user's code, report diagnostics, etc.
-        update();
-    }
-
-    return SLANG_OK;
-}
-
-} // namespace LanguageServer
-
-int main(int argc, const char* const* argv)
-{
-    bool isDebug = false;
-    for (auto i = 1; i < argc; i++)
-    {
-        if (Slang::UnownedStringSlice(argv[i]) == "--debug")
-        {
-            isDebug = true;
-        }
-    }
-    if (isDebug)
-    {
-        std::this_thread::sleep_for(std::chrono::seconds(10));
-    }
-    Slang::LanguageServer server;
-    SLANG_RETURN_ON_FAIL(server.execute());
-    return 0;
-}
diff --git a/tools/slangd/language-server.h b/tools/slangd/language-server.h
deleted file mode 100644
index 829a629b1..000000000
--- a/tools/slangd/language-server.h
+++ /dev/null
@@ -1,35 +0,0 @@
-#pragma once
-
-#include "../../source/core/slang-io.h"
-#include "../../source/core/slang-process-util.h"
-#include "../../source/core/slang-string-util.h"
-#include "../../source/core/slang-string.h"
-#include "../../source/core/slang-writer.h"
-#include "../../source/compiler-core/slang-json-rpc-connection.h"
-#include "language-server-protocol.h"
-
-namespace Slang
-{
-    class LanguageServer
-    {
-    public:
-        RefPtr<JSONRPCConnection> m_connection;
-        ComPtr<slang::IGlobalSession> m_session;
-        bool m_quit = false;
-        List<LanguageServerProtocol::WorkspaceFolder> m_workspaceFolders;
-
-        SlangResult init(const LanguageServerProtocol::InitializeParams& args);
-        SlangResult execute();
-        void update();
-        SlangResult didOpenTextDocument(
-            const LanguageServerProtocol::DidOpenTextDocumentParams& args);
-        SlangResult didCloseTextDocument(
-            const LanguageServerProtocol::DidCloseTextDocumentParams& args);
-        SlangResult didChangeTextDocument(
-            const LanguageServerProtocol::DidChangeTextDocumentParams& args);
-
-    private:
-        SlangResult _executeSingle();
-        slang::IGlobalSession* getOrCreateGlobalSession();
-    };
-}
diff --git a/tools/slangd/main.cpp b/tools/slangd/main.cpp
new file mode 100644
index 000000000..4e3bfd029
--- /dev/null
+++ b/tools/slangd/main.cpp
@@ -0,0 +1,25 @@
+// main.cpp
+
+// This file implements the entry point for `slangd`, the daemon process of Slang's language server.
+
+#include <thread>
+
+#include "../../source/core/slang-basic.h"
+#include "../../source/slang/slang-language-server.h"
+
+int main(int argc, const char* const* argv)
+{
+    bool isDebug = false;
+    for (auto i = 1; i < argc; i++)
+    {
+        if (Slang::UnownedStringSlice(argv[i]) == "--debug")
+        {
+            isDebug = true;
+        }
+    }
+    if (isDebug)
+    {
+        std::this_thread::sleep_for(std::chrono::seconds(10));
+    }
+    return Slang::runLanguageServer();
+}