From 87eb789ae03cbd7a79d5433cefb37f10bec86753 Mon Sep 17 00:00:00 2001 From: jsmall-nvidia Date: Fri, 19 Nov 2021 10:22:56 -0500 Subject: Improvements to JSON/RTTI (#2022) * #include an absolute path didn't work - because paths were taken to always be relative. * Use 'Process' to communicate with an command line tool. * Remove slang-win-stream * Tidy up windows ProcessUtil. * First version of BufferedReadStream. * Windows working IPC for steams. * Test proxy count option. * Split Process/ProcessUtil. Process is platform dependant. ProcessUtil are functions that are platform independent. * First implementation of Unix Process interface. * Unix process compiles on cygwin. * Fix typo in unix process. * Separate unix pipe stream error of invalid access, from pipe availability. * Fix in standard line extraction. * Make fd non blocking. * Fix issues with Windows Process streams. * Added UnixPipe. * Some fixes around UnixPipeStream. * Make a unix stream closed explicit. * Hack to debug linux process/stream. * Revert to old linux pipe handling. * Pass executable path for unit tests. Split out CommandLine into own source. * Small improvements in process/command line. * Check process behavior with crash. * Make stderr and stdout unbuffered for crash testing. * Only turn disable buffering in crash test. * Disable crash test on CI. * Fix crash on clang/linux. * Enable crash test. Remove _appendBuffer as can use StreamUtil functionality. * Added inital processing for http headers. * Small improvements to HttpHeader. * First pass HTTPPacketConnection working on windows. * Enable other Process communication tests. * Update comments. * WIP JSON RPC. * Add terminate to Process. Made JSONRPC a Util. * Small tidy up around HTTPPacketConnection. * Improve process termination options. * WIP for test-server. * Add diagnostics error handling to test-server. * Improved JSON support. Parsing/creating JSON-RPC messages. * WIP JSONRPC parsing. * First pass RttiInfo support. * WIP converting between JSON/native types. * Project files. * Split out RttiUtil. Made RttiInfo constuction thread safe. * WIP RTTI<->JSON. * Add diagnostics to JSON<->native conversions. * Make RttiInfo for structs globals. Avoids problem around derived types (like pointers), being able to cause an abort. * Add pointer support to RTTI. Fixed some compilation issues on linux. * Add fixed array support. * Added Rtti unit test. * Add rtti unit test. * Split out quoted/unquoted key handling. Fix bugs in JSON value/container. Added JSON native test. * Make default array allocator use malloc/free. Remove the new[] handler (doesn't work on visuals studio). * Fix for linux warning. * Remove some test code. * Fix issues on x86 win. * Fix warning on aarch64. * Fix some bugs in JSON parsing/handling. Make Rtti work copy/dtor/ctor struct types. * Testing JSON<->native with fixed array. Make makeArrayView explicit if it's just a single value. Added array type. * Fix getting arrayView. * Improve JSON diagnostic name. --- source/compiler-core/slang-json-diagnostic-defs.h | 1 + source/compiler-core/slang-json-native.cpp | 46 +++++++++++++++++++++++ source/compiler-core/slang-json-parser.cpp | 4 ++ source/compiler-core/slang-json-value.cpp | 29 ++++++++------ source/compiler-core/slang-json-value.h | 8 +++- 5 files changed, 75 insertions(+), 13 deletions(-) (limited to 'source/compiler-core') diff --git a/source/compiler-core/slang-json-diagnostic-defs.h b/source/compiler-core/slang-json-diagnostic-defs.h index 23bea9b79..4733a3291 100644 --- a/source/compiler-core/slang-json-diagnostic-defs.h +++ b/source/compiler-core/slang-json-diagnostic-defs.h @@ -40,5 +40,6 @@ DIAGNOSTIC(20009, Error, unableToConvertField, "unable to convert field '$0' in DIAGNOSTIC(20010, Error, fieldNotFound, "field '$0' not found in type '$1'") DIAGNOSTIC(20011, Error, fieldNotDefinedOnType, "field '$0' not defined on type '$1'") DIAGNOSTIC(20011, Error, fieldRequiredOnType, "field '$0' required on '$1'") +DIAGNOSTIC(20012, Error, tooManyElementsForArray, "too many elements ($0) for array array. Max allowed is $1") #undef DIAGNOSTIC diff --git a/source/compiler-core/slang-json-native.cpp b/source/compiler-core/slang-json-native.cpp index 5b2fb5db4..9c972eba3 100644 --- a/source/compiler-core/slang-json-native.cpp +++ b/source/compiler-core/slang-json-native.cpp @@ -203,6 +203,33 @@ SlangResult JSONToNativeConverter::convert(const JSONValue& in, const RttiInfo* return SLANG_OK; } + case RttiInfo::Kind::FixedArray: + { + if (in.getKind() != JSONValue::Kind::Array) + { + return SLANG_FAIL; + } + const FixedArrayRttiInfo* fixedArrayRttiInfo = static_cast(rttiInfo); + const auto elementType = fixedArrayRttiInfo->m_elementType; + const Index elementCount = Index(fixedArrayRttiInfo->m_elementCount); + const auto elementSize = elementType->m_size; + + auto srcArray = m_container->getArray(in); + + if (srcArray.getCount() > elementCount) + { + m_sink->diagnose(in.loc, JSONDiagnostics::tooManyElementsForArray, srcArray.getCount(), elementCount); + return SLANG_FAIL; + } + + Byte* dstEles = (Byte*)out; + for (Index i = 0; i < elementCount; ++i, dstEles += elementSize) + { + SLANG_RETURN_ON_FAIL(convert(srcArray[i], elementType, dstEles)); + } + + return SLANG_OK; + } case RttiInfo::Kind::Dictionary: { // We can *only* serialize this into a straight JSON object iff the key is a string-like type @@ -341,6 +368,25 @@ SlangResult NativeToJSONConverter::convert(const RttiInfo* rttiInfo, const void* out = m_container->createArray(dstValues.getBuffer(), count); return SLANG_OK; } + case RttiInfo::Kind::FixedArray: + { + const FixedArrayRttiInfo* fixedArrayRttiInfo = static_cast(rttiInfo); + const auto elementType = fixedArrayRttiInfo->m_elementType; + const auto elementCount = Index(fixedArrayRttiInfo->m_elementCount); + const auto elementSize = elementType->m_size; + + List dstValues; + dstValues.setCount(elementCount); + + const Byte* src = (const Byte*)in; + for (Index i = 0; i < elementCount; ++i, src += elementSize) + { + SLANG_RETURN_ON_FAIL(convert(elementType, src, dstValues[i])); + } + + out = m_container->createArray(dstValues.getBuffer(), elementCount); + return SLANG_OK; + } case RttiInfo::Kind::Dictionary: { const DictionaryRttiInfo* listRttiInfo = static_cast(rttiInfo); diff --git a/source/compiler-core/slang-json-parser.cpp b/source/compiler-core/slang-json-parser.cpp index fe9a0f580..52e46258a 100644 --- a/source/compiler-core/slang-json-parser.cpp +++ b/source/compiler-core/slang-json-parser.cpp @@ -308,6 +308,8 @@ void JSONWriter::endObject(SourceLoc loc) m_state = m_stack.getLast(); m_stack.removeLast(); + + _postValue(); } void JSONWriter::startArray(SourceLoc loc) @@ -343,6 +345,8 @@ void JSONWriter::endArray(SourceLoc loc) m_state = m_stack.getLast(); m_stack.removeLast(); + + _postValue(); } void JSONWriter::addUnquotedKey(const UnownedStringSlice& key, SourceLoc loc) diff --git a/source/compiler-core/slang-json-value.cpp b/source/compiler-core/slang-json-value.cpp index 1e18bee39..4fad3764f 100644 --- a/source/compiler-core/slang-json-value.cpp +++ b/source/compiler-core/slang-json-value.cpp @@ -957,8 +957,7 @@ JSONBuilder::JSONBuilder(JSONContainer* container, Flags flags): { m_state.m_kind = State::Kind::Root; m_state.m_startIndex = 0; - - m_keyValue.reset(); + m_state.resetKey(); m_rootValue.reset(); } @@ -968,9 +967,9 @@ void JSONBuilder::reset() // Reset the state m_state.m_kind = State::Kind::Root; m_state.m_startIndex = 0; + m_state.resetKey(); // Clear the work values - m_keyValue.reset(); m_rootValue.reset(); // Clear the lists @@ -1037,17 +1036,24 @@ void JSONBuilder::_add(const JSONValue& value) } case State::Kind::Object: { - m_keyValue.value = value; - const Index index = _findKeyIndex(m_keyValue.key); + SLANG_ASSERT(m_state.hasKey()); + + JSONKeyValue keyValue; + keyValue.key = m_state.m_key; + keyValue.keyLoc = m_state.m_keyLoc; + keyValue.value = value; + + const Index index = _findKeyIndex(keyValue.key); if (index >= 0) { - m_keyValues[index] = m_keyValue; + m_keyValues[index] = keyValue; } else { - m_keyValues.add(m_keyValue); + m_keyValues.add(keyValue); } - m_keyValue.reset(); + + m_state.resetKey(); break; } } @@ -1059,6 +1065,7 @@ void JSONBuilder::startObject(SourceLoc loc) m_state.m_kind = State::Kind::Object; m_state.m_startIndex = m_keyValues.getCount(); m_state.m_loc = loc; + m_state.resetKey(); } void JSONBuilder::endObject(SourceLoc loc) @@ -1082,6 +1089,7 @@ void JSONBuilder::startArray(SourceLoc loc) m_state.m_kind = State::Kind::Array; m_state.m_startIndex = m_values.getCount(); m_state.m_loc = loc; + m_state.resetKey(); } void JSONBuilder::endArray(SourceLoc loc) @@ -1110,9 +1118,8 @@ void JSONBuilder::addQuotedKey(const UnownedStringSlice& key, SourceLoc loc) void JSONBuilder::addUnquotedKey(const UnownedStringSlice& key, SourceLoc loc) { - SLANG_ASSERT(m_keyValue.key == JSONKey(0)); - m_keyValue.key = m_container->getKey(key); - m_keyValue.keyLoc = loc; + SLANG_ASSERT(!m_state.hasKey()); + m_state.setKey(m_container->getKey(key), loc); } void JSONBuilder::addLexemeValue(JSONTokenType type, const UnownedStringSlice& value, SourceLoc loc) diff --git a/source/compiler-core/slang-json-value.h b/source/compiler-core/slang-json-value.h index 2eae4ae08..119479285 100644 --- a/source/compiler-core/slang-json-value.h +++ b/source/compiler-core/slang-json-value.h @@ -334,9 +334,15 @@ protected: Object, Array, }; + void setKey(JSONKey key, SourceLoc loc) { m_key = key; m_keyLoc = loc; } + void resetKey() { m_key = JSONKey(0); m_keyLoc = SourceLoc(); } + bool hasKey() const { return m_key != JSONKey(0); } + Kind m_kind; Index m_startIndex; SourceLoc m_loc; + JSONKey m_key; + SourceLoc m_keyLoc; }; void _popState(); @@ -353,8 +359,6 @@ protected: State m_state; JSONContainer* m_container; - - JSONKeyValue m_keyValue; JSONValue m_rootValue; StringBuilder m_work; -- cgit v1.2.3