#include "text-io.h" #ifdef _WIN32 #define WIN32_LEAN_AND_MEAN #define NOMINMAX #include #undef WIN32_LEAN_AND_MEAN #undef NOMINMAX #define CONVERT_END_OF_LINE #endif namespace Slang { class Utf8Encoding : public Encoding { public: virtual void GetBytes(List & result, const String & str) override { result.AddRange(str.Buffer(), str.Length()); } virtual String ToString(const char * bytes, int /*length*/) override { return String(bytes); } }; class Utf32Encoding : public Encoding { public: virtual void GetBytes(List & result, const String & str) override { UInt ptr = 0; while (ptr < str.Length()) { int codePoint = GetUnicodePointFromUTF8([&](int) { if (ptr < str.Length()) return str[ptr++]; else return '\0'; }); result.AddRange((char*)&codePoint, 4); } } virtual String ToString(const char * bytes, int length) override { StringBuilder sb; int * content = (int*)bytes; for (int i = 0; i < (length >> 2); i++) { char buf[5]; int count = EncodeUnicodePointToUTF8(buf, content[i]); for (int j = 0; j < count; j++) sb.Append(buf[j]); } return sb.ProduceString(); } }; class Utf16Encoding : public Encoding //UTF16 { private: bool reverseOrder = false; public: Utf16Encoding(bool pReverseOrder) : reverseOrder(pReverseOrder) {} virtual void GetBytes(List & result, const String & str) override { UInt ptr = 0; while (ptr < str.Length()) { int codePoint = GetUnicodePointFromUTF8([&](int) { if (ptr < str.Length()) return str[ptr++]; else return '\0'; }); unsigned short buffer[2]; int count; if (!reverseOrder) count = EncodeUnicodePointToUTF16(buffer, codePoint); else count = EncodeUnicodePointToUTF16Reversed(buffer, codePoint); result.AddRange((char*)buffer, count * 2); } } virtual String ToString(const char * bytes, int length) override { int ptr = 0; StringBuilder sb; while (ptr < length) { int codePoint = GetUnicodePointFromUTF16([&](int) { if (ptr < length) return bytes[ptr++]; else return '\0'; }); char buf[5]; int count = EncodeUnicodePointToUTF8(buf, codePoint); for (int i = 0; i < count; i++) sb.Append(buf[i]); } return sb.ProduceString(); } }; Utf8Encoding __utf8Encoding; Utf16Encoding __utf16Encoding(false); Utf16Encoding __utf16EncodingReversed(true); Utf32Encoding __utf32Encoding; Encoding * Encoding::UTF8 = &__utf8Encoding; Encoding * Encoding::UTF16 = &__utf16Encoding; Encoding * Encoding::UTF16Reversed = &__utf16EncodingReversed; Encoding * Encoding::UTF32 = &__utf32Encoding; const unsigned short Utf16Header = 0xFEFF; const unsigned short Utf16ReversedHeader = 0xFFFE; StreamWriter::StreamWriter(const String & path, Encoding * encoding) { this->stream = new FileStream(path, FileMode::Create); this->encoding = encoding; if (encoding == Encoding::UTF16) { this->stream->Write(&Utf16Header, 2); } else if (encoding == Encoding::UTF16Reversed) { this->stream->Write(&Utf16ReversedHeader, 2); } } StreamWriter::StreamWriter(RefPtr stream, Encoding * encoding) { this->stream = stream; this->encoding = encoding; if (encoding == Encoding::UTF16) { this->stream->Write(&Utf16Header, 2); } else if (encoding == Encoding::UTF16Reversed) { this->stream->Write(&Utf16ReversedHeader, 2); } } void StreamWriter::Write(const String & str) { encodingBuffer.Clear(); StringBuilder sb; String newLine; #ifdef _WIN32 newLine = "\r\n"; #else newLine = "\n"; #endif for (UInt i = 0; i < str.Length(); i++) { if (str[i] == '\r') sb << newLine; else if (str[i] == '\n') { if (i > 0 && str[i - 1] != '\r') sb << newLine; } else sb << str[i]; } encoding->GetBytes(encodingBuffer, sb.ProduceString()); stream->Write(encodingBuffer.Buffer(), encodingBuffer.Count()); } void StreamWriter::Write(const char * str) { Write(String(str)); } StreamReader::StreamReader(const String & path) { stream = new FileStream(path, FileMode::Open); ReadBuffer(); encoding = DetermineEncoding(); if (encoding == 0) encoding = Encoding::UTF8; } StreamReader::StreamReader(RefPtr stream, Encoding * encoding) { this->stream = stream; this->encoding = encoding; ReadBuffer(); auto determinedEncoding = DetermineEncoding(); if (this->encoding == nullptr) this->encoding = determinedEncoding; } bool HasNullBytes(char * str, int len) { bool hasSeenNull = false; for (int i = 0; i < len - 1; i++) if (str[i] == 0) hasSeenNull = true; else if (hasSeenNull) return true; return false; } Encoding * StreamReader::DetermineEncoding() { if (buffer.Count() >= 3 && (unsigned char)(buffer[0]) == 0xEF && (unsigned char)(buffer[1]) == 0xBB && (unsigned char)(buffer[2]) == 0xBF) { ptr += 3; return Encoding::UTF8; } else if (*((unsigned short*)(buffer.Buffer())) == 0xFEFF) { ptr += 2; return Encoding::UTF16; } else if (*((unsigned short*)(buffer.Buffer())) == 0xFFFE) { ptr += 2; return Encoding::UTF16Reversed; } else { // find null bytes if (HasNullBytes(buffer.Buffer(), (int)buffer.Count())) { return Encoding::UTF16; } return Encoding::UTF8; } } void StreamReader::ReadBuffer() { buffer.SetSize(4096); memset(buffer.Buffer(), 0, buffer.Count() * sizeof(buffer[0])); auto len = stream->Read(buffer.Buffer(), buffer.Count()); buffer.SetSize((int)len); ptr = 0; } char StreamReader::ReadBufferChar() { if (ptrIsEnd()) ReadBuffer(); if (ptr