From 3d90678c8f54b9696c12185481fba27d146b0820 Mon Sep 17 00:00:00 2001 From: Yong He Date: Sat, 4 Nov 2017 15:17:08 -0400 Subject: Fix encoding detection when reading text file. Win32 API could mistakenly report UTF16 when the file is actually UTF8. --- source/core/text-io.cpp | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) (limited to 'source/core/text-io.cpp') diff --git a/source/core/text-io.cpp b/source/core/text-io.cpp index ec41b8d7d..3e8301fa8 100644 --- a/source/core/text-io.cpp +++ b/source/core/text-io.cpp @@ -194,6 +194,14 @@ namespace Slang this->encoding = determinedEncoding; } + bool HasNullBytes(char * str, int len) + { + for (int i = 0; i < len; i++) + if (str[len] == 0) + return true; + return false; + } + Encoding * StreamReader::DetermineEncoding() { if (buffer.Count() >= 3 && (unsigned char)(buffer[0]) == 0xEF && (unsigned char)(buffer[1]) == 0xBB && (unsigned char)(buffer[2]) == 0xBF) @@ -213,19 +221,9 @@ namespace Slang } else { -#ifdef _WIN32 - int flag = IS_TEXT_UNICODE_SIGNATURE | IS_TEXT_UNICODE_REVERSE_SIGNATURE | IS_TEXT_UNICODE_STATISTICS | IS_TEXT_UNICODE_ASCII16; - int rs = IsTextUnicode(buffer.Buffer(), (int) buffer.Count(), &flag); - if (rs) - { - if (flag & (IS_TEXT_UNICODE_SIGNATURE | IS_TEXT_UNICODE_STATISTICS)) - return Encoding::UTF16; - else if (flag & (IS_TEXT_UNICODE_SIGNATURE | IS_TEXT_UNICODE_STATISTICS)) - return Encoding::UTF16Reversed; - else if (flag & IS_TEXT_UNICODE_ASCII16) - return Encoding::UTF8; - } -#endif + // find null bytes + if (HasNullBytes(buffer.Buffer(), (int)buffer.Count())) + return Encoding::UTF16; return Encoding::UTF8; } } -- cgit v1.2.3