From b390566b55700582321b09b72c726b8dff9bd819 Mon Sep 17 00:00:00 2001
From: Yong He <yonghe@outlook.com>
Date: Mon, 12 Aug 2024 20:53:03 -0700
Subject: Support unicode identifier names. (#4772)

* Support unicode identifier names.

* Fix.

* Fix language server.

* Fix build errors.

* Fix.

* Fix offset translation in language server.
---
 source/core/slang-char-encode.cpp | 23 +++++++++++++++++++++++
 source/core/slang-char-encode.h   |  4 ++++
 source/core/slang-char-util.h     |  2 +-
 source/core/slang-std-writers.cpp |  9 ++++++++-
 4 files changed, 36 insertions(+), 2 deletions(-)

(limited to 'source/core')

diff --git a/source/core/slang-char-encode.cpp b/source/core/slang-char-encode.cpp
index 105cfac7f..526c6c923 100644
--- a/source/core/slang-char-encode.cpp
+++ b/source/core/slang-char-encode.cpp
@@ -211,4 +211,27 @@ CharEncoding* CharEncoding::UTF32 = &_utf32Encoding;
     return count;
 }
 
+Index UTF8Util::calcUTF16CharCount(const UnownedStringSlice& in)
+{
+    Index count = 0;
+    Index readPtr = 0;
+    for (;;)
+    {
+        int c = getUnicodePointFromUTF8([&]() -> Byte
+            {
+                if (readPtr < in.getLength())
+                    return in[readPtr++];
+                else
+                    return 0;
+            });
+        if (c == 0)
+            break;
+        Char16 buffer[2];
+        count += encodeUnicodePointToUTF16(c, buffer);
+        if (readPtr >= in.getLength())
+            break;
+    }
+    return count;
+}
+
 } // namespace Slang
diff --git a/source/core/slang-char-encode.h b/source/core/slang-char-encode.h
index 2bb4cba29..a7cd501ab 100644
--- a/source/core/slang-char-encode.h
+++ b/source/core/slang-char-encode.h
@@ -203,6 +203,10 @@ struct UTF8Util
         /// Non valid utf8 input or ending starting in partial characters, will produce 
         /// undefined results without error.
     static Index calcCodePointCount(const UnownedStringSlice& in);
+
+
+        /// Given a slice in UTF8, calculate the number of UTF16 characters needed to represent the string.
+    static Index calcUTF16CharCount(const UnownedStringSlice& in);
 };
 
 }
diff --git a/source/core/slang-char-util.h b/source/core/slang-char-util.h
index c65f676c4..88af24426 100644
--- a/source/core/slang-char-util.h
+++ b/source/core/slang-char-util.h
@@ -61,7 +61,7 @@ struct CharUtil
         /// Returns the value if c interpretted as a octal digit
         /// If c is not a valid octal returns -1
     inline static int getOctalDigitValue(char c) { return isOctalDigit(c) ? (c - '0') : -1; }
-    
+
     struct CharFlagMap
     {
         Flags flags[0x100];
diff --git a/source/core/slang-std-writers.cpp b/source/core/slang-std-writers.cpp
index a23d878fb..264f37c98 100644
--- a/source/core/slang-std-writers.cpp
+++ b/source/core/slang-std-writers.cpp
@@ -1,6 +1,10 @@
 
 #include "slang-std-writers.h"
 
+#if SLANG_WINDOWS_FAMILY
+#include <Windows.h>
+#endif
+
 namespace Slang
 {
 
@@ -8,8 +12,11 @@ namespace Slang
 
 /* static */RefPtr<StdWriters> StdWriters::createDefault()
 {
+#if SLANG_WINDOWS_FAMILY
+    SetConsoleCP(CP_UTF8);
+    SetConsoleOutputCP(CP_UTF8);
+#endif
     RefPtr<StdWriters> stdWriters(new StdWriters);
-
     RefPtr<FileWriter> stdError(new FileWriter(stderr, WriterFlag::AutoFlush | WriterFlag::IsUnowned));
     RefPtr<FileWriter> stdOut(new FileWriter(stdout, WriterFlag::AutoFlush | WriterFlag::IsUnowned));
 
-- 
cgit v1.2.3