Support unicode identifier names. (#4772)

* Support unicode identifier names.

* Fix.

* Fix language server.

* Fix build errors.

* Fix.

* Fix offset translation in language server.

4 files changed, 95 insertions, 43 deletions

diff --git a/source/slang/slang-language-server-document-symbols.cpp b/source/slang/slang-language-server-document-symbols.cpp
index ec9b434eb..45a76e97b 100644
--- a/source/slang/slang-language-server-document-symbols.cpp
+++ b/source/slang/slang-language-server-document-symbols.cpp
@@ -167,7 +167,7 @@ namespace Slang
                 sym.selectionRange.start.line = (int)line;
                 sym.selectionRange.start.character = (int)col;
                 sym.selectionRange.end.line = (int)line;
-                sym.selectionRange.end.character = (int)(col + nameLoc.name->text.getLength());
+                sym.selectionRange.end.character = (int)(col + (int)UTF8Util::calcUTF16CharCount(nameLoc.name->text.getUnownedSlice()));
                 sym.range.start.line = (int)line;
                 sym.range.start.character = 0;
                 sym.range.end.line = (int)line;
diff --git a/source/slang/slang-language-server.cpp b/source/slang/slang-language-server.cpp
index 09b14932c..ed03a5dc4 100644
--- a/source/slang/slang-language-server.cpp
+++ b/source/slang/slang-language-server.cpp
@@ -654,9 +654,12 @@ SlangResult LanguageServer::hover(
             maybeAppendAdditionalOverloadsHint();
             auto nodeHumaneLoc =
                 version->linkage->getSourceManager()->getHumaneLoc(leafNode->loc);
-            hover.range.start.line = int(nodeHumaneLoc.line - 1);
-            hover.range.end.line = int(nodeHumaneLoc.line - 1);
-            hover.range.start.character = int(nodeHumaneLoc.column - 1);
+            doc->oneBasedUTF8LocToZeroBasedUTF16Loc(
+                nodeHumaneLoc.line,
+                nodeHumaneLoc.column,
+                hover.range.start.line,
+                hover.range.start.character);
+            hover.range.end = hover.range.start;
             auto name = declRef.getName();
             if (auto ctorDecl = declRef.as<ConstructorDecl>())
             {
@@ -668,17 +671,19 @@ SlangResult LanguageServer::hover(
             }
             if (name)
             {
-                hover.range.end.character = int(nodeHumaneLoc.column + name->text.getLength() - 1);
+                hover.range.end.character = hover.range.start.character + (int)UTF8Util::calcUTF16CharCount(name->text.getUnownedSlice());
             }
         }
     };
     auto fillLoc = [&](SourceLoc loc)
     {
         auto humaneLoc = version->linkage->getSourceManager()->getHumaneLoc(loc, SourceLocType::Actual);
-        hover.range.start.line = int(humaneLoc.line - 1);
-        hover.range.end.line = int(humaneLoc.line - 1);
-        hover.range.start.character = int(humaneLoc.column - 1);
-        hover.range.end.character = hover.range.start.character + int(doc->getTokenLength(humaneLoc.line, humaneLoc.column));
+        doc->oneBasedUTF8LocToZeroBasedUTF16Loc(humaneLoc.line, humaneLoc.column, hover.range.start.line, hover.range.start.character);
+        doc->oneBasedUTF8LocToZeroBasedUTF16Loc(
+            humaneLoc.line,
+            humaneLoc.column + doc->getTokenLength(humaneLoc.line, humaneLoc.column),
+            hover.range.end.line,
+            hover.range.end.character);
     };
     auto fillExprHoverInfo = [&](Expr* expr)
     {
@@ -851,7 +856,7 @@ SlangResult LanguageServer::gotoDefinition(
                                                                 : declRefExpr->declRef.getLoc(),
                     SourceLocType::Actual);
             auto name = declRefExpr->declRef.getName();
-            locations.add(LocationResult{location, name ? (int)name->text.getLength() : 0});
+            locations.add(LocationResult{location, name ? (int)UTF8Util::calcUTF16CharCount(name->text.getUnownedSlice()) : 0});
         }
     }
     else if (auto overloadedExpr = as<OverloadedExpr>(leafNode))
@@ -863,7 +868,7 @@ SlangResult LanguageServer::gotoDefinition(
                 auto location = version->linkage->getSourceManager()->getHumaneLoc(
                     item.declRef.getNameLoc(), SourceLocType::Actual);
                 auto name = item.declRef.getName();
-                locations.add(LocationResult{location, name ? (int)name->text.getLength() : 0});
+                locations.add(LocationResult{location, name ? (int)UTF8Util::calcUTF16CharCount(name->text.getUnownedSlice()) : 0});
             }
         }
         else 
@@ -874,7 +879,7 @@ SlangResult LanguageServer::gotoDefinition(
                 auto location = version->linkage->getSourceManager()->getHumaneLoc(
                     item.declRef.getNameLoc(), SourceLocType::Actual);
                 auto name = item.declRef.getName();
-                locations.add(LocationResult{location, name ? (int)name->text.getLength() : 0});
+                locations.add(LocationResult{location, name ? (int)UTF8Util::calcUTF16CharCount(name->text.getUnownedSlice()) : 0});
             }
         }
     }
@@ -909,8 +914,11 @@ SlangResult LanguageServer::gotoDefinition(
             {
                 result.uri =
                     URI::fromLocalFilePath(loc.loc.pathInfo.foundPath.getUnownedSlice()).uri;
-                result.range.start.line = int(loc.loc.line - 1);
-                result.range.start.character = int(loc.loc.column - 1);
+                doc->oneBasedUTF8LocToZeroBasedUTF16Loc(
+                    loc.loc.line,
+                    loc.loc.column,
+                    result.range.start.line,
+                    result.range.start.character);
                 result.range.end = result.range.start;
                 result.range.end.character += loc.length;
                 results.add(result);
diff --git a/source/slang/slang-workspace-version.cpp b/source/slang/slang-workspace-version.cpp
index a07ef75cc..d85724328 100644
--- a/source/slang/slang-workspace-version.cpp
+++ b/source/slang/slang-workspace-version.cpp
@@ -396,42 +396,66 @@ void DocumentVersion::setText(const String& newText)
 {
     text = newText;
     StringUtil::calcLines(text.getUnownedSlice(), lines);
-    utf16CharStarts.clear();
+    mapUTF16CharIndexToCodePointIndex.clear();
+    mapCodePointIndexToUTF8ByteOffset.clear();
 }
-ArrayView<Index> DocumentVersion::getUTF16Boundaries(Index line)
+
+void DocumentVersion::ensureUTFBoundsAvailable()
 {
-    if (!utf16CharStarts.getCount())
+    for (auto slice : lines)
     {
-        for (auto slice : lines)
+        List<Index> bounds;
+        List<Index> utf8Bounds;
+        Index index = 0;
+        Index codePointIndex = 0;
+        while (index < slice.getLength())
         {
-            List<Index> bounds;
-            Index index = 0;
-            while (index < slice.getLength())
-            {
-                auto startIndex = index;
-                const Char32 codePoint = getUnicodePointFromUTF8(
-                    [&]() -> Byte
-                    {
-                        if (index < slice.getLength())
-                            return slice[index++];
-                        else
-                            return '\0';
-                    });
-                if (!codePoint)
-                    break;
-                Char16 buffer[2];
-                int count = encodeUnicodePointToUTF16Reversed(codePoint, buffer);
-                for (int i = 0; i < count; i++)
-                    bounds.add(startIndex);
-            }
-            bounds.add(slice.getLength());
-            utf16CharStarts.add(_Move(bounds));
+            auto startIndex = index;
+            const Char32 codePoint = getUnicodePointFromUTF8(
+                [&]() -> Byte
+                {
+                    if (index < slice.getLength())
+                        return slice[index++];
+                    else
+                        return '\0';
+                });
+            if (!codePoint)
+                break;
+
+            Char16 buffer[2];
+            int count = encodeUnicodePointToUTF16Reversed(codePoint, buffer);
+            for (int i = 0; i < count; i++)
+                bounds.add(codePointIndex);
+            utf8Bounds.add(startIndex);
+            codePointIndex++;
         }
+        bounds.add(slice.getLength());
+        utf8Bounds.add(slice.getLength());
+        mapUTF16CharIndexToCodePointIndex.add(_Move(bounds));
+        mapCodePointIndexToUTF8ByteOffset.add(_Move(utf8Bounds));
     }
-    return line >= 1 && line <= utf16CharStarts.getCount() ? utf16CharStarts[line - 1].getArrayView()
+}
+
+ArrayView<Index> DocumentVersion::getUTF16Boundaries(Index line)
+{
+    if (!mapUTF16CharIndexToCodePointIndex.getCount())
+    {
+        ensureUTFBoundsAvailable();
+    }
+    return line >= 1 && line <= mapUTF16CharIndexToCodePointIndex.getCount() ? mapUTF16CharIndexToCodePointIndex[line - 1].getArrayView()
                                                            : ArrayView<Index>();
 }
 
+ArrayView<Index> DocumentVersion::getUTF8Boundaries(Index line)
+{
+    if (!mapCodePointIndexToUTF8ByteOffset.getCount())
+    {
+        ensureUTFBoundsAvailable();
+    }
+    return line >= 1 && line <= mapCodePointIndexToUTF8ByteOffset.getCount() ? mapCodePointIndexToUTF8ByteOffset[line - 1].getArrayView()
+        : ArrayView<Index>();
+}
+
 void DocumentVersion::oneBasedUTF8LocToZeroBasedUTF16Loc(
     Index inLine, Index inCol, Index& outLine, Index& outCol)
 {
@@ -447,6 +471,15 @@ void DocumentVersion::oneBasedUTF8LocToZeroBasedUTF16Loc(
     outCol = std::lower_bound(bounds.begin(), bounds.end(), inCol - 1) - bounds.begin();
 }
 
+void DocumentVersion::oneBasedUTF8LocToZeroBasedUTF16Loc(
+    Index inLine, Index inCol, int& outLine, int& outCol)
+{
+    Index ioutLine, ioutCol;
+    oneBasedUTF8LocToZeroBasedUTF16Loc(inLine, inCol, ioutLine, ioutCol);
+    outLine = (int)ioutLine;
+    outCol = (int)ioutCol;
+}
+
 void DocumentVersion::zeroBasedUTF16LocToOneBasedUTF8Loc(
     Index inLine, Index inCol, Index& outLine, Index& outCol)
 {
diff --git a/source/slang/slang-workspace-version.h b/source/slang/slang-workspace-version.h
index 44ab6b43c..d6cbfe6c5 100644
--- a/source/slang/slang-workspace-version.h
+++ b/source/slang/slang-workspace-version.h
@@ -20,7 +20,8 @@ namespace Slang
         String path;
         String text;
         List<UnownedStringSlice> lines;
-        List<List<Index>> utf16CharStarts;
+        List<List<Index>> mapUTF16CharIndexToCodePointIndex;
+        List<List<Index>> mapCodePointIndexToUTF8ByteOffset;
     public:
         void setPath(String filePath)
         {
@@ -32,10 +33,14 @@ namespace Slang
         const String& getText() { return text; }
         void setText(const String& newText);
 
+        void ensureUTFBoundsAvailable();
         ArrayView<Index> getUTF16Boundaries(Index line);
+        ArrayView<Index> getUTF8Boundaries(Index line);
 
         void oneBasedUTF8LocToZeroBasedUTF16Loc(
             Index inLine, Index inCol, Index& outLine, Index& outCol);
+        void oneBasedUTF8LocToZeroBasedUTF16Loc(
+            Index inLine, Index inCol, int& outLine, int& outCol);
         void zeroBasedUTF16LocToOneBasedUTF8Loc(
             Index inLine, Index inCol, Index& outLine, Index& outCol);
 
@@ -60,7 +65,11 @@ namespace Slang
                 return -1;
 
             Index lineStart = lineIndex >= 1 ? getLineStart(lines[lineIndex - 1]) : 0;
-            return lineStart + colIndex - 1;
+            auto boundaries = getUTF8Boundaries(lineIndex);
+            Index byteOffset = 0;
+            if (colIndex > 0 && colIndex <= boundaries.getCount())
+                byteOffset = boundaries[colIndex - 1];
+            return lineStart + byteOffset;
         }
 
         // Get 1-based, utf-8 encoding location from offset.
@@ -81,6 +90,8 @@ namespace Slang
             {
                 col = Index(offset - getLineStart(lines[line-1])) + 1;
             }
+            if (line > 0 && line <= lines.getCount())
+                col = UTF8Util::calcCodePointCount(lines[line-1].head(col));
         }
 
         // Get line from 1-based index.