From 2d3392f22c894957d17dd13486e0565c4ecea89c Mon Sep 17 00:00:00 2001 From: jsmall-nvidia Date: Fri, 27 May 2022 17:28:05 -0400 Subject: Added NativeStringType (#2252) * #include an absolute path didn't work - because paths were taken to always be relative. * Use TerminatedUnownedStringSlice for literals in output C++. * Remove Escape/Unescape functions used in slang-token-reader.cpp Add target type of 'host-cpp' etc to map to the target types. * Fix some corner cases around string encoding. * Added unit test for string escaping. Fixed some assorted escaping bugs. * Updated test output. * Added decode test. * Stop using hex output, to get around 'greedy' aspect. Use octal instead. --- tools/slang-unit-test/unit-offset-container.cpp | 117 --------------------- .../slang-unit-test/unit-test-offset-container.cpp | 117 +++++++++++++++++++++ tools/slang-unit-test/unit-test-string-escape.cpp | 79 ++++++++++++++ 3 files changed, 196 insertions(+), 117 deletions(-) delete mode 100644 tools/slang-unit-test/unit-offset-container.cpp create mode 100644 tools/slang-unit-test/unit-test-offset-container.cpp create mode 100644 tools/slang-unit-test/unit-test-string-escape.cpp (limited to 'tools') diff --git a/tools/slang-unit-test/unit-offset-container.cpp b/tools/slang-unit-test/unit-offset-container.cpp deleted file mode 100644 index 6a179c319..000000000 --- a/tools/slang-unit-test/unit-offset-container.cpp +++ /dev/null @@ -1,117 +0,0 @@ -// unit-test-path.cpp - -#include "../../source/core/slang-offset-container.h" - -#include "tools/unit-test/slang-unit-test.h" - -using namespace Slang; - -static void _checkEncodeDecode(uint32_t size) -{ - uint8_t encode[OffsetString::kMaxSizeEncodeSize]; - - size_t encodeSize = OffsetString::calcEncodedSize(size, encode); - - size_t decodedSize; - const char* chars = OffsetString::decodeSize((const char*)encode, decodedSize); - - SLANG_CHECK(decodedSize == size); - SLANG_CHECK(chars - (const char*)encode == encodeSize); -} - -namespace { // anonymous - -struct Root -{ - Offset32Array > dirs; - Offset32Ptr name; - float value; -}; - -} // anonymous - -SLANG_UNIT_TEST(offsetContainer) -{ - _checkEncodeDecode(253); - - for (int64_t i = 0; i < 0x100000000; i += (i / 2) + 1) - { - _checkEncodeDecode(uint32_t(i)); - } - - { - OffsetContainer container; - - const char* strings[] = - { - "Hello", - "World", - nullptr, - }; - - { - auto& base = container.asBase(); - - Offset32Ptr root = container.newObject(); - - auto array = container.newArray>(SLANG_COUNT_OF(strings)); - for (Int i = 0; i < SLANG_COUNT_OF(strings); ++i) - { - base[array[i]] = container.newString(strings[i]); - } - base[root]->dirs = array; - } - - { - List copy; - copy.addRange(container.getData(), container.getDataCount()); - - MemoryOffsetBase base; - base.set(copy.getBuffer(), copy.getCount()); - - Root* root = (Root*)(copy.getBuffer() + kStartOffset); - - SLANG_CHECK(root->dirs.getCount() == SLANG_COUNT_OF(strings)); - - Int count = root->dirs.getCount(); - for (Int i = 0; i < count; ++i) - { - OffsetString* str = base.asRaw(base.asRaw(root->dirs[i])); - - const char* check = strings[i]; - - if (check) - { - SLANG_CHECK(str != nullptr); - const char* strCstr = str->getCstr(); - SLANG_CHECK(strcmp(strCstr, check) == 0); - } - else - { - SLANG_CHECK(str == nullptr); - } - } - - { - Index index = 0; - for (const auto v : root->dirs) - { - OffsetString* str = base.asRaw(base.asRaw(v)); - const char* check = strings[index]; - if (check) - { - SLANG_CHECK(str != nullptr); - const char* strCstr = str->getCstr(); - SLANG_CHECK(strcmp(strCstr, check) == 0); - } - else - { - SLANG_CHECK(str == nullptr); - } - - index ++; - } - } - } - } -} diff --git a/tools/slang-unit-test/unit-test-offset-container.cpp b/tools/slang-unit-test/unit-test-offset-container.cpp new file mode 100644 index 000000000..9d8e3a9ff --- /dev/null +++ b/tools/slang-unit-test/unit-test-offset-container.cpp @@ -0,0 +1,117 @@ +// unit-test-offset-container.cpp + +#include "../../source/core/slang-offset-container.h" + +#include "tools/unit-test/slang-unit-test.h" + +using namespace Slang; + +static void _checkEncodeDecode(uint32_t size) +{ + uint8_t encode[OffsetString::kMaxSizeEncodeSize]; + + size_t encodeSize = OffsetString::calcEncodedSize(size, encode); + + size_t decodedSize; + const char* chars = OffsetString::decodeSize((const char*)encode, decodedSize); + + SLANG_CHECK(decodedSize == size); + SLANG_CHECK(chars - (const char*)encode == encodeSize); +} + +namespace { // anonymous + +struct Root +{ + Offset32Array > dirs; + Offset32Ptr name; + float value; +}; + +} // anonymous + +SLANG_UNIT_TEST(offsetContainer) +{ + _checkEncodeDecode(253); + + for (int64_t i = 0; i < 0x100000000; i += (i / 2) + 1) + { + _checkEncodeDecode(uint32_t(i)); + } + + { + OffsetContainer container; + + const char* strings[] = + { + "Hello", + "World", + nullptr, + }; + + { + auto& base = container.asBase(); + + Offset32Ptr root = container.newObject(); + + auto array = container.newArray>(SLANG_COUNT_OF(strings)); + for (Int i = 0; i < SLANG_COUNT_OF(strings); ++i) + { + base[array[i]] = container.newString(strings[i]); + } + base[root]->dirs = array; + } + + { + List copy; + copy.addRange(container.getData(), container.getDataCount()); + + MemoryOffsetBase base; + base.set(copy.getBuffer(), copy.getCount()); + + Root* root = (Root*)(copy.getBuffer() + kStartOffset); + + SLANG_CHECK(root->dirs.getCount() == SLANG_COUNT_OF(strings)); + + Int count = root->dirs.getCount(); + for (Int i = 0; i < count; ++i) + { + OffsetString* str = base.asRaw(base.asRaw(root->dirs[i])); + + const char* check = strings[i]; + + if (check) + { + SLANG_CHECK(str != nullptr); + const char* strCstr = str->getCstr(); + SLANG_CHECK(strcmp(strCstr, check) == 0); + } + else + { + SLANG_CHECK(str == nullptr); + } + } + + { + Index index = 0; + for (const auto v : root->dirs) + { + OffsetString* str = base.asRaw(base.asRaw(v)); + const char* check = strings[index]; + if (check) + { + SLANG_CHECK(str != nullptr); + const char* strCstr = str->getCstr(); + SLANG_CHECK(strcmp(strCstr, check) == 0); + } + else + { + SLANG_CHECK(str == nullptr); + } + + index ++; + } + } + } + } +} diff --git a/tools/slang-unit-test/unit-test-string-escape.cpp b/tools/slang-unit-test/unit-test-string-escape.cpp new file mode 100644 index 000000000..337573081 --- /dev/null +++ b/tools/slang-unit-test/unit-test-string-escape.cpp @@ -0,0 +1,79 @@ +// unit-test-string-escape.cpp + +#include "../../source/core/slang-string-escape-util.h" + +#include "tools/unit-test/slang-unit-test.h" + +using namespace Slang; + +static bool _checkConversion(StringEscapeHandler* handler, const UnownedStringSlice& check) +{ + StringBuilder buf; + handler->appendEscaped(check, buf); + + StringBuilder decode; + handler->appendUnescaped(buf.getUnownedSlice(), decode); + + return decode == check; +} + +static bool _checkDecode(const UnownedStringSlice& encoded, const UnownedStringSlice& decoded) +{ + auto handler = StringEscapeUtil::getHandler(StringEscapeUtil::Style::Cpp); + + StringBuilder buf; + StringEscapeUtil::appendUnquoted(handler, encoded, buf); + return buf == decoded; +} + +#define SLANG_ENCODED_DECODED(x) \ + const auto encoded = toSlice(#x); \ + const auto decoded = toSlice(x); + +SLANG_UNIT_TEST(StringEscape) +{ + // Check greedy hex digits + { + // \x can have any number of hex digits + const char text[] = "\x000001"; + SLANG_ASSERT(SLANG_COUNT_OF(text) == 2 && text[0] == 1); + } + + // Check octal greedy + { + //\ + up to 3 octal digits + const char text[] = "\0011"; + SLANG_ASSERT(SLANG_COUNT_OF(text) == 3 && text[0] == 1 && text[1] == '1'); + + const char text2[] = "\78"; + SLANG_ASSERT(SLANG_COUNT_OF(text2) == 3 && text2[0] == 7 && text2[1] == '8'); + } + + { + auto handler = StringEscapeUtil::getHandler(StringEscapeUtil::Style::Cpp); + + SLANG_CHECK(_checkConversion(handler, toSlice("\0\1\2""2"))); + } + + { + auto handler = StringEscapeUtil::getHandler(StringEscapeUtil::Style::Cpp); + + // We can't just use '\uxxxx', because it has to be translatable into an output character in MSVC (not into utf8) + // Can make work perhaps with something like + // #pragma execution_character_set("utf-8") + // But for now we don't worry + // + // Visual Studio does not appear to support '\U' by default, presumably because wchar_t is 16 bits + + { + SLANG_ENCODED_DECODED("\a\b\0hey~\u0023\n\0"); + SLANG_CHECK(_checkDecode(encoded, decoded)); + } + + { + SLANG_ENCODED_DECODED("\n\v\b\t\1\02\003\x5z\x00007f\0"); + SLANG_CHECK(_checkDecode(encoded, decoded)); + } + } +} + -- cgit v1.2.3