diff options
Diffstat (limited to 'source/core/slang-text-io.cpp')
| -rw-r--r-- | source/core/slang-text-io.cpp | 438 |
1 files changed, 129 insertions, 309 deletions
diff --git a/source/core/slang-text-io.cpp b/source/core/slang-text-io.cpp index ca3b9447c..3f5d739d9 100644 --- a/source/core/slang-text-io.cpp +++ b/source/core/slang-text-io.cpp @@ -1,343 +1,163 @@ #include "slang-text-io.h" -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#define NOMINMAX -#include <Windows.h> -#undef WIN32_LEAN_AND_MEAN -#undef NOMINMAX -#define CONVERT_END_OF_LINE -#endif + +#include "../../slang-com-helper.h" namespace Slang { - class Utf8Encoding : public Encoding - { - public: - virtual void GetBytes(List<char> & result, const String & str) override - { - result.addRange(str.getBuffer(), str.getLength()); - } - virtual String ToString(const char * bytes, int /*length*/) override - { - return String(bytes); - } - }; - class Utf32Encoding : public Encoding - { - public: - virtual void GetBytes(List<char> & result, const String & str) override - { - Index ptr = 0; - while (ptr < str.getLength()) - { - int codePoint = GetUnicodePointFromUTF8([&](int) - { - if (ptr < str.getLength()) - return str[ptr++]; - else - return '\0'; - }); - result.addRange((char*)&codePoint, 4); - } - } - virtual String ToString(const char * bytes, int length) override - { - StringBuilder sb; - int * content = (int*)bytes; - for (int i = 0; i < (length >> 2); i++) - { - char buf[5]; - int count = EncodeUnicodePointToUTF8(buf, content[i]); - for (int j = 0; j < count; j++) - sb.Append(buf[j]); - } - return sb.ProduceString(); - } - }; +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! StreamWriter !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - class Utf16Encoding : public Encoding //UTF16 - { - private: - bool reverseOrder = false; - public: - Utf16Encoding(bool pReverseOrder) - : reverseOrder(pReverseOrder) - {} - virtual void GetBytes(List<char> & result, const String & str) override - { - Index ptr = 0; - while (ptr < str.getLength()) - { - int codePoint = GetUnicodePointFromUTF8([&](int) - { - if (ptr < str.getLength()) - return str[ptr++]; - else - return '\0'; - }); - unsigned short buffer[2]; - int count; - if (!reverseOrder) - count = EncodeUnicodePointToUTF16(buffer, codePoint); - else - count = EncodeUnicodePointToUTF16Reversed(buffer, codePoint); - result.addRange((char*)buffer, count * 2); - } - } - virtual String ToString(const char * bytes, int length) override - { - int ptr = 0; - StringBuilder sb; - while (ptr < length) - { - int codePoint = GetUnicodePointFromUTF16([&](int) - { - if (ptr < length) - return bytes[ptr++]; - else - return '\0'; - }); - char buf[5]; - int count = EncodeUnicodePointToUTF8(buf, codePoint); - for (int i = 0; i < count; i++) - sb.Append(buf[i]); - } - return sb.ProduceString(); - } - }; - - Utf8Encoding __utf8Encoding; - Utf16Encoding __utf16Encoding(false); - Utf16Encoding __utf16EncodingReversed(true); - Utf32Encoding __utf32Encoding; - - Encoding * Encoding::UTF8 = &__utf8Encoding; - Encoding * Encoding::UTF16 = &__utf16Encoding; - Encoding * Encoding::UTF16Reversed = &__utf16EncodingReversed; - Encoding * Encoding::UTF32 = &__utf32Encoding; - - const unsigned short Utf16Header = 0xFEFF; - const unsigned short Utf16ReversedHeader = 0xFFFE; +SlangResult StreamWriter::init(const String& path, CharEncoding* encoding) +{ + RefPtr<FileStream> fileStream = new FileStream; + SLANG_RETURN_ON_FAIL(fileStream->init(path, FileMode::Create)); + return init(fileStream, encoding); +} - StreamWriter::StreamWriter(const String & path, Encoding * encoding) +SlangResult StreamWriter::init(RefPtr<Stream> stream, CharEncoding* encoding) +{ + m_stream = stream; + m_encoding = encoding; + if (encoding == CharEncoding::UTF16) { - this->stream = new FileStream(path, FileMode::Create); - this->encoding = encoding; - if (encoding == Encoding::UTF16) - { - this->stream->write(&Utf16Header, 2); - } - else if (encoding == Encoding::UTF16Reversed) - { - this->stream->write(&Utf16ReversedHeader, 2); - } + SLANG_RETURN_ON_FAIL(m_stream->write(&kUTF16Header, 2)); } - StreamWriter::StreamWriter(RefPtr<Stream> stream, Encoding * encoding) + else if (encoding == CharEncoding::UTF16Reversed) { - this->stream = stream; - this->encoding = encoding; - if (encoding == Encoding::UTF16) - { - this->stream->write(&Utf16Header, 2); - } - else if (encoding == Encoding::UTF16Reversed) - { - this->stream->write(&Utf16ReversedHeader, 2); - } + SLANG_RETURN_ON_FAIL(m_stream->write(&kUTF16ReversedHeader, 2)); } - void StreamWriter::Write(const String & str) - { - encodingBuffer.clear(); - StringBuilder sb; - String newLine; + + return SLANG_OK; +} + +SlangResult StreamWriter::writeSlice(const UnownedStringSlice& slice) +{ + // TODO(JS): + // We can do better here. On Linux, this is a no-op and can just write directly (assuming slice only contains \n) + + m_encodingBuffer.clear(); + + StringBuilder sb; #ifdef _WIN32 - newLine = "\r\n"; + const char newLine[] = "\r\n"; #else - newLine = "\n"; + const char newLine[] = "\n"; #endif - for (Index i = 0; i < str.getLength(); i++) + const Index length = slice.getLength(); + + for (Index i = 0; i < length; i++) + { + if (slice[i] == '\r') + sb << newLine; + else if (slice[i] == '\n') { - if (str[i] == '\r') + if (i > 0 && slice[i - 1] != '\r') sb << newLine; - else if (str[i] == '\n') - { - if (i > 0 && str[i - 1] != '\r') - sb << newLine; - } - else - sb << str[i]; } - encoding->GetBytes(encodingBuffer, sb.ProduceString()); - stream->write(encodingBuffer.getBuffer(), encodingBuffer.getCount()); - } - void StreamWriter::Write(const char * str) - { - Write(String(str)); + else + sb << slice[i]; } - StreamReader::StreamReader(const String & path) - { - stream = new FileStream(path, FileMode::Open); - ReadBuffer(); - encoding = DetermineEncoding(); - if (encoding == 0) - encoding = Encoding::UTF8; - } - StreamReader::StreamReader(RefPtr<Stream> stream, Encoding * encoding) - { - this->stream = stream; - this->encoding = encoding; - ReadBuffer(); - auto determinedEncoding = DetermineEncoding(); - if (this->encoding == nullptr) - this->encoding = determinedEncoding; - } + // NOTE! This assumes that sb contains *complete* utf8 code points, which it might not, as encoder is only able to handle complete code points. + m_encodingBuffer.clear(); + m_encoding->encode(sb.getUnownedSlice(), m_encodingBuffer); + return m_stream->write(m_encodingBuffer.getBuffer(), m_encodingBuffer.getCount()); +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! StreamReader !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +StreamReader::StreamReader() +{ +} + +SlangResult StreamReader::init(const String& path) +{ + RefPtr<FileStream> fileStream = new FileStream; + SLANG_RETURN_ON_FAIL(fileStream->init(path, FileMode::Open)); + return init(fileStream); +} - bool HasNullBytes(char * str, int len) +SlangResult StreamReader::init(RefPtr<Stream> stream, CharEncoding* encoding) +{ + m_stream = stream; + m_encoding = encoding; + SLANG_RETURN_ON_FAIL(readBuffer()); + + if (encoding == nullptr) { - bool hasSeenNull = false; - for (int i = 0; i < len - 1; i++) - if (str[i] == 0) - hasSeenNull = true; - else if (hasSeenNull) - return true; - return false; + size_t offset; + m_encodingType = CharEncoding::determineEncoding((const Byte*)m_buffer.getBuffer(), m_buffer.getCount(), offset); + m_encoding = CharEncoding::getEncoding(m_encodingType); + m_index = Index(offset); + } + else + { + m_encodingType = encoding->getEncodingType(); + m_encoding = encoding; } - Encoding * StreamReader::DetermineEncoding() - { - if (buffer.getCount() >= 3 && (unsigned char)(buffer[0]) == 0xEF && (unsigned char)(buffer[1]) == 0xBB && (unsigned char)(buffer[2]) == 0xBF) - { - ptr += 3; - return Encoding::UTF8; - } - else if (*((unsigned short*)(buffer.getBuffer())) == 0xFEFF) - { - ptr += 2; - return Encoding::UTF16; - } - else if (*((unsigned short*)(buffer.getBuffer())) == 0xFFFE) - { - ptr += 2; - return Encoding::UTF16Reversed; - } - else - { - // find null bytes - if (HasNullBytes(buffer.getBuffer(), (int)buffer.getCount())) - { - return Encoding::UTF16; - } - return Encoding::UTF8; - } - } + return SLANG_OK; +} - void StreamReader::ReadBuffer() - { - buffer.setCount(4096); - memset(buffer.getBuffer(), 0, buffer.getCount() * sizeof(buffer[0])); - auto len = stream->read(buffer.getBuffer(), buffer.getCount()); - buffer.setCount((int)len); - ptr = 0; - } +SlangResult StreamReader::readBuffer() +{ + m_buffer.setCount(0); + m_index = 0; - char StreamReader::ReadBufferChar() - { - if (ptr<buffer.getCount()) - { - return buffer[ptr++]; - } - if (!stream->isEnd()) - ReadBuffer(); - if (ptr<buffer.getCount()) - { - return buffer[ptr++]; - } - return 0; - } - int TextReader::Read(char * destBuffer, int length) + if (m_stream->isEnd()) + { + return SLANG_OK; + } + + m_buffer.setCount(4096); + + // TODO(JS): Not clear this is necessary + memset(m_buffer.getBuffer(), 0, m_buffer.getCount() * sizeof(m_buffer[0])); + + size_t readBytes; + SLANG_RETURN_ON_FAIL(m_stream->read(m_buffer.getBuffer(), m_buffer.getCount(), readBytes)); + + m_buffer.setCount(Index(readBytes)); + m_index = 0; + return SLANG_OK; +} + +char StreamReader::readBufferChar() +{ + if (m_index < m_buffer.getCount()) { - int i = 0; - for (i = 0; i<length; i++) - { - try - { - auto ch = Read(); - if (IsEnd()) - break; - if (ch == '\r') - { - if (Peak() == '\n') - Read(); - break; - } - else if (ch == '\n') - { - break; - } - destBuffer[i] = ch; - } - catch (const EndOfStreamException& ) - { - break; - } - } - return i; + return m_buffer[m_index++]; } - String StreamReader::ReadLine() + + readBuffer(); + + if (m_index < m_buffer.getCount()) { - StringBuilder sb(256); - while (!IsEnd()) - { - try - { - auto ch = Read(); - if (IsEnd()) - break; - if (ch == '\r') - { - if (Peak() == '\n') - Read(); - break; - } - else if (ch == '\n') - { - break; - } - sb.Append(ch); - } - catch (const EndOfStreamException&) - { - break; - } - } - return sb.ProduceString(); + return m_buffer[m_index++]; } - String StreamReader::ReadToEnd() + return 0; +} + +SlangResult StreamReader::readToEnd(String& outString) +{ + StringBuilder sb(16384); + while (!isEnd()) { - StringBuilder sb(16384); - while (!IsEnd()) + auto ch = read(); + if (isEnd()) + break; + if (ch == '\r') { - try - { - auto ch = Read(); - if (IsEnd()) - break; - if (ch == '\r') - { - sb.Append('\n'); - if (Peak() == '\n') - Read(); - } - else - sb.Append(ch); - } - catch (const EndOfStreamException&) - { - break; - } + sb.Append('\n'); + if (peek() == '\n') + read(); } - return sb.ProduceString(); + else + sb.Append(ch); } + + outString = sb.ProduceString(); + return SLANG_OK; } + +} // namespace Slang |