Support unicode identifier names. (#4772)

* Support unicode identifier names.

* Fix.

* Fix language server.

* Fix build errors.

* Fix.

* Fix offset translation in language server.

4 files changed, 36 insertions, 2 deletions

diff --git a/source/core/slang-char-encode.cpp b/source/core/slang-char-encode.cpp
index 105cfac7f..526c6c923 100644
--- a/source/core/slang-char-encode.cpp
+++ b/source/core/slang-char-encode.cpp
@@ -211,4 +211,27 @@ CharEncoding* CharEncoding::UTF32 = &_utf32Encoding;
     return count;
 }
 
+Index UTF8Util::calcUTF16CharCount(const UnownedStringSlice& in)
+{
+    Index count = 0;
+    Index readPtr = 0;
+    for (;;)
+    {
+        int c = getUnicodePointFromUTF8([&]() -> Byte
+            {
+                if (readPtr < in.getLength())
+                    return in[readPtr++];
+                else
+                    return 0;
+            });
+        if (c == 0)
+            break;
+        Char16 buffer[2];
+        count += encodeUnicodePointToUTF16(c, buffer);
+        if (readPtr >= in.getLength())
+            break;
+    }
+    return count;
+}
+
 } // namespace Slang
diff --git a/source/core/slang-char-encode.h b/source/core/slang-char-encode.h
index 2bb4cba29..a7cd501ab 100644
--- a/source/core/slang-char-encode.h
+++ b/source/core/slang-char-encode.h
@@ -203,6 +203,10 @@ struct UTF8Util
         /// Non valid utf8 input or ending starting in partial characters, will produce 
         /// undefined results without error.
     static Index calcCodePointCount(const UnownedStringSlice& in);
+
+
+        /// Given a slice in UTF8, calculate the number of UTF16 characters needed to represent the string.
+    static Index calcUTF16CharCount(const UnownedStringSlice& in);
 };
 
 }
diff --git a/source/core/slang-char-util.h b/source/core/slang-char-util.h
index c65f676c4..88af24426 100644
--- a/source/core/slang-char-util.h
+++ b/source/core/slang-char-util.h
@@ -61,7 +61,7 @@ struct CharUtil
         /// Returns the value if c interpretted as a octal digit
         /// If c is not a valid octal returns -1
     inline static int getOctalDigitValue(char c) { return isOctalDigit(c) ? (c - '0') : -1; }
-    
+
     struct CharFlagMap
     {
         Flags flags[0x100];
diff --git a/source/core/slang-std-writers.cpp b/source/core/slang-std-writers.cpp
index a23d878fb..264f37c98 100644
--- a/source/core/slang-std-writers.cpp
+++ b/source/core/slang-std-writers.cpp
@@ -1,6 +1,10 @@
 
 #include "slang-std-writers.h"
 
+#if SLANG_WINDOWS_FAMILY
+#include <Windows.h>
+#endif
+
 namespace Slang
 {
 
@@ -8,8 +12,11 @@ namespace Slang
 
 /* static */RefPtr<StdWriters> StdWriters::createDefault()
 {
+#if SLANG_WINDOWS_FAMILY
+    SetConsoleCP(CP_UTF8);
+    SetConsoleOutputCP(CP_UTF8);
+#endif
     RefPtr<StdWriters> stdWriters(new StdWriters);
-
     RefPtr<FileWriter> stdError(new FileWriter(stderr, WriterFlag::AutoFlush | WriterFlag::IsUnowned));
     RefPtr<FileWriter> stdOut(new FileWriter(stdout, WriterFlag::AutoFlush | WriterFlag::IsUnowned));