RTTI/JSON (#2021)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Use 'Process' to communicate with an command line tool.

* Remove slang-win-stream

* Tidy up windows ProcessUtil.

* First version of BufferedReadStream.

* Windows working IPC for steams.

* Test proxy count option.

* Split Process/ProcessUtil. Process is platform dependant. ProcessUtil are functions that are platform independent.

* First implementation of Unix Process interface.

* Unix process compiles on cygwin.

* Fix typo in unix process.

* Separate unix pipe stream error of invalid access, from pipe availability.

* Fix in standard line extraction.

* Make fd non blocking.

* Fix issues with Windows Process streams.

* Added UnixPipe.

* Some fixes around UnixPipeStream.

* Make a unix stream closed explicit.

* Hack to debug linux process/stream.

* Revert to old linux pipe handling.

* Pass executable path for unit tests.
Split out CommandLine into own source.

* Small improvements in process/command line.

* Check process behavior with crash.

* Make stderr and stdout unbuffered for crash testing.

* Only turn disable buffering in crash test.

* Disable crash test on CI.

* Fix crash on clang/linux.

* Enable crash test.
Remove _appendBuffer as can use StreamUtil functionality.

* Added inital processing for http headers.

* Small improvements to HttpHeader.

* First pass HTTPPacketConnection working on windows.

* Enable other Process communication tests.

* Update comments.

* WIP JSON RPC.

* Add terminate to Process.
Made JSONRPC a Util.

* Small tidy up around HTTPPacketConnection.

* Improve process termination options.

* WIP for test-server.

* Add diagnostics error handling to test-server.

* Improved JSON support.
Parsing/creating JSON-RPC messages.

* WIP JSONRPC parsing.

* First pass RttiInfo support.

* WIP converting between JSON/native types.

* Project files.

* Split out RttiUtil.
Made RttiInfo constuction thread safe.

* WIP RTTI<->JSON.

* Add diagnostics to JSON<->native conversions.

* Make RttiInfo for structs globals. Avoids problem around derived types (like pointers), being able to cause an abort.

* Add pointer support to RTTI.
Fixed some compilation issues on linux.

* Add fixed array support.

* Added Rtti unit test.

* Add rtti unit test.

* Split out quoted/unquoted key handling.
Fix bugs in JSON value/container.
Added JSON native test.

* Make default array allocator use malloc/free.
Remove the new[] handler (doesn't work on visuals studio).

* Fix for linux warning.

* Remove some test code.

* Fix issues on x86 win.

* Fix warning on aarch64.

1 files changed, 377 insertions, 0 deletions

diff --git a/source/compiler-core/slang-json-native.cpp b/source/compiler-core/slang-json-native.cpp
new file mode 100644
index 000000000..5b2fb5db4
--- /dev/null
+++ b/source/compiler-core/slang-json-native.cpp
@@ -0,0 +1,377 @@
+#include "slang-json-native.h"
+
+#include "../../slang-com-helper.h"
+
+#include "../core/slang-rtti-util.h"
+
+#include "slang-json-diagnostics.h"
+
+namespace Slang {
+
+/* !!!!!!!!!!!!!!!!!!!!!!!!!!!! JSONToNativeConverter !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */
+
+/* static */Index JSONToNativeConverter::_getFieldCount(const StructRttiInfo* structRttiInfo)
+{
+    if (structRttiInfo->m_super)
+    {
+        return _getFieldCount(structRttiInfo->m_super) + structRttiInfo->m_fieldCount;
+    }
+    else
+    {
+        return structRttiInfo->m_fieldCount;
+    }
+}
+
+/* static */Index JSONToNativeConverter::_findFieldIndex(const StructRttiInfo* structRttiInfo, const UnownedStringSlice& fieldName)
+{
+    if (structRttiInfo->m_super)
+    {
+        const Index index = _findFieldIndex(structRttiInfo->m_super, fieldName);
+        if (index >= 0)
+        {
+            return index + _getFieldCount(structRttiInfo->m_super);
+        }
+    }
+
+    ConstArrayView<StructRttiInfo::Field> fields(structRttiInfo->m_fields, structRttiInfo->m_fieldCount);
+
+    Index index = fields.findFirstIndex([fieldName](const StructRttiInfo::Field& field) ->bool { return fieldName == field.m_name; });
+    if (index >= 0 && structRttiInfo->m_super)
+    {
+        index += _getFieldCount(structRttiInfo->m_super);
+    }
+
+    return index;
+}
+
+SlangResult JSONToNativeConverter::_structToNative(const ConstArrayView<JSONKeyValue>& pairs, const StructRttiInfo* structRttiInfo, void* out, Index& outFieldCount)
+{
+    Index fieldCount = 0;
+
+    if (structRttiInfo->m_super)
+    {
+        SLANG_RETURN_ON_FAIL(_structToNative(pairs, structRttiInfo->m_super, out, fieldCount));
+    }
+
+    Byte* dst = (Byte*)out;
+
+    const Index count = structRttiInfo->m_fieldCount;
+
+    for (Index i = 0; i < count; ++i)
+    {
+        const auto& field = structRttiInfo->m_fields[i];
+
+        auto key = m_container->findKey(UnownedStringSlice(field.m_name));
+
+        if (key == 0)
+        {
+            if (field.m_flags & StructRttiInfo::Flag::Optional)
+            {
+                continue;
+            }
+
+            m_sink->diagnose(SourceLoc(), JSONDiagnostics::fieldRequiredOnType, field.m_name, structRttiInfo->m_name);
+
+            // Unable to find this key
+            return SLANG_FAIL;
+        }
+
+        // If there are any of the pairs, that are not in the type.. it's an error
+        const Index index = pairs.findFirstIndex([key](const JSONKeyValue& pair) -> bool { return pair.key == key; });
+        if (index < 0)
+        {
+            if (field.m_flags & StructRttiInfo::Flag::Optional)
+            {
+                continue;
+            }
+
+            m_sink->diagnose(SourceLoc(), JSONDiagnostics::fieldRequiredOnType, field.m_name, structRttiInfo->m_name);
+
+            // Unable to find this key
+            return SLANG_FAIL;
+        }
+
+        auto& pair = pairs[index];
+
+        // Copy the field over
+        SLANG_RETURN_ON_FAIL(convert(pair.value, field.m_type, dst + field.m_offset));
+
+        // Field was handled
+        ++fieldCount;
+    }
+
+    // Write off the amount of fields converted/handled. 
+    outFieldCount = fieldCount;
+    return SLANG_OK;
+}
+
+SlangResult JSONToNativeConverter::convert(const JSONValue& in, const RttiInfo* rttiInfo, void* out)
+{
+    if (rttiInfo->isIntegral())
+    {
+        return RttiUtil::setInt(m_container->asInteger(in), rttiInfo, out);
+    }
+    else if (rttiInfo->isFloat())
+    {
+        return RttiUtil::setFromDouble(m_container->asFloat(in), rttiInfo, out);
+    }
+
+    switch (rttiInfo->m_kind)
+    {
+        case RttiInfo::Kind::Bool:
+        {
+            *(bool*)out = m_container->asBool(in);
+            return SLANG_OK;
+        }
+        case RttiInfo::Kind::Struct:
+        {
+            if (in.getKind() != JSONValue::Kind::Object)
+            {
+                return SLANG_FAIL;
+            }
+
+            auto pairs = m_container->getObject(in);
+            const StructRttiInfo* structRttiInfo = static_cast<const StructRttiInfo*>(rttiInfo);
+
+            Index fieldCount = 0;
+            SLANG_RETURN_ON_FAIL(_structToNative(pairs, structRttiInfo, out, fieldCount));
+
+            if (fieldCount != pairs.getCount())
+            {
+                // We want to find the fields not found in the type
+
+                for (auto& pair : pairs)
+                {
+                    UnownedStringSlice fieldName = m_container->getStringFromKey(pair.key);
+                    const Index index = _findFieldIndex(structRttiInfo, UnownedStringSlice(fieldName));
+
+                    if (index < 0)
+                    {
+                        m_sink->diagnose(pair.keyLoc, JSONDiagnostics::fieldNotDefinedOnType, fieldName, structRttiInfo->m_name);
+                    }
+                }
+
+                // If these are different then there are fields defined in the object that are *not* defined in class definition
+                return SLANG_FAIL;
+            }
+
+            return SLANG_OK;
+        }
+        case RttiInfo::Kind::Enum:
+        {
+            return SLANG_E_NOT_IMPLEMENTED;
+        }
+        case RttiInfo::Kind::String:
+        {
+            *(String*)out = m_container->getTransientString(in);
+            return SLANG_OK;
+        }
+        case RttiInfo::Kind::UnownedStringSlice:
+        {
+            // Problem -> if the slice is a lexeme, then when we decode with getString, it will lose scope.
+            // So we do something a bit odd and place the decoding string
+
+            *(UnownedStringSlice*)out = m_container->getString(in);
+            return SLANG_OK;
+        }
+        case RttiInfo::Kind::List:
+        {
+            if (in.getKind() != JSONValue::Kind::Array)
+            {
+                return SLANG_FAIL;
+            }
+
+            typedef List<Byte> Type;
+            Type& list = *(Type*)out;
+
+            auto arr = m_container->getArray(in);
+
+            const Index count = arr.getCount();
+
+            const ListRttiInfo* listRttiInfo = static_cast<const ListRttiInfo*>(rttiInfo);
+            auto elementType = listRttiInfo->m_elementType;
+
+            SLANG_RETURN_ON_FAIL(RttiUtil::setListCount(elementType, out, arr.getCount()));
+
+            // Okay, we need to copy over one by one
+
+            Byte* dstEles = list.getBuffer();
+            for (Index i = 0; i < count; ++i, dstEles += elementType->m_size)
+            {
+                SLANG_RETURN_ON_FAIL(convert(arr[i], elementType, dstEles));
+            }
+
+            return SLANG_OK;
+        }
+        case RttiInfo::Kind::Dictionary:
+        {
+            // We can *only* serialize this into a straight JSON object iff the key is a string-like type
+            // We could turn into (say) an array of keys and values
+            break;
+        }
+        case RttiInfo::Kind::Other:
+        {
+            if (rttiInfo == GetRttiInfo<JSONValue>::get())
+            {
+                // Do we need to copy into the container?
+                // As it stands we have to assume src is stored in container.
+                *(JSONValue*)out = in;
+                return SLANG_OK;
+            }
+            return SLANG_FAIL;
+        }
+        default: break;
+    }
+    return SLANG_FAIL;
+}
+
+
+/* !!!!!!!!!!!!!!!!!!!!!!!!!!!! NativeToJSONConverter !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */
+
+SlangResult NativeToJSONConverter::_structToJSON(const StructRttiInfo* structRttiInfo, const void* src, List<JSONKeyValue>& outPairs)
+{
+    // Do the super class first
+    if (structRttiInfo->m_super)
+    {
+        SLANG_RETURN_ON_FAIL(_structToJSON(structRttiInfo, src, outPairs));
+    }
+
+    const Byte* base = (const Byte*)src;
+    const Index count = structRttiInfo->m_fieldCount;
+
+    for (Index i = 0; i < count; ++i)
+    {
+        const auto& field = structRttiInfo->m_fields[i];
+
+        if (field.m_flags & StructRttiInfo::Flag::Optional)
+        {
+            const RttiDefaultValue defaultValue = RttiDefaultValue(field.m_flags & uint8_t(RttiDefaultValue::Mask));
+            if (RttiUtil::isDefault(defaultValue, field.m_type, base + field.m_offset))
+            {
+                // If it's a default, we don't bother writing it
+                continue;
+            }
+        }
+
+        JSONKeyValue pair;
+        pair.key = m_container->getKey(UnownedStringSlice(field.m_name));
+        auto res = convert(field.m_type, base + field.m_offset, pair.value);
+
+        if (SLANG_FAILED(res))
+        {
+            m_sink->diagnose(SourceLoc(), JSONDiagnostics::unableToConvertField, field.m_name, structRttiInfo->m_name);
+            return res;
+        }
+
+        outPairs.add(pair);
+    }
+
+    return SLANG_OK;
+}
+
+
+SlangResult NativeToJSONConverter::convert(const RttiInfo* rttiInfo, const void* in, JSONValue& out)
+{
+    if (rttiInfo->isIntegral())
+    {
+        out = JSONValue::makeInt(RttiUtil::getInt64(rttiInfo, in));
+        return SLANG_OK;
+    }
+    else if (rttiInfo->isFloat())
+    {
+        out = JSONValue::makeFloat(RttiUtil::asDouble(rttiInfo, in));
+        return SLANG_OK;
+    }
+
+    switch (rttiInfo->m_kind)
+    {
+        case RttiInfo::Kind::Invalid:   return SLANG_FAIL;
+        case RttiInfo::Kind::Bool:
+        {
+            out = JSONValue::makeBool(RttiUtil::asBool(rttiInfo, in));
+            return SLANG_OK;
+        }
+        case RttiInfo::Kind::String:
+        {
+            const String& str = *(const String*)in;
+            out = m_container->createString(str.getUnownedSlice());
+            return SLANG_OK;
+        }
+        case RttiInfo::Kind::UnownedStringSlice:
+        {
+            const UnownedStringSlice& slice = *(const UnownedStringSlice*)in;
+            out = m_container->createString(slice);
+            return SLANG_OK;
+        }
+        case RttiInfo::Kind::Struct:
+        {
+            const StructRttiInfo* structRttiInfo = static_cast<const StructRttiInfo*>(rttiInfo);
+
+            List<JSONKeyValue> pairs;
+            SLANG_RETURN_ON_FAIL(_structToJSON(structRttiInfo, in, pairs));
+            out = m_container->createObject(pairs.getBuffer(), pairs.getCount());
+            return SLANG_OK;
+        }
+        case RttiInfo::Kind::Enum:
+        {   
+            return SLANG_E_NOT_IMPLEMENTED;
+        }
+        case RttiInfo::Kind::List:
+        {
+            const ListRttiInfo* listRttiInfo = static_cast<const ListRttiInfo*>(rttiInfo);
+            const auto elementRttiInfo = listRttiInfo->m_elementType;
+
+            // The src probably *doesn't* contain bytes, but can cast like this because
+            // we only need the count (which doesn't depend on <T>), and the backing buffer
+            const List<Byte>& srcValuesList = *(const List<Byte>*)in;
+
+            const Index count = srcValuesList.getCount();
+            const Byte* srcValues = srcValuesList.getBuffer();
+
+            List<JSONValue> dstValues;
+            dstValues.setCount(count);
+
+            const size_t elementStride = elementRttiInfo->m_size;
+
+            for (Index i = 0; i < count; ++i, srcValues += elementStride)
+            {
+                SLANG_RETURN_ON_FAIL(convert(elementRttiInfo, srcValues, dstValues[i]));
+            }
+
+            out = m_container->createArray(dstValues.getBuffer(), count);
+            return SLANG_OK;
+        }
+        case RttiInfo::Kind::Dictionary:
+        {
+            const DictionaryRttiInfo* listRttiInfo = static_cast<const DictionaryRttiInfo*>(rttiInfo);
+            const auto keyRttiInfo = listRttiInfo->m_keyType;
+            const auto valueRttiInfo = listRttiInfo->m_valueType;
+
+            SLANG_UNUSED(keyRttiInfo);
+            SLANG_UNUSED(valueRttiInfo);
+
+            // We can *only* serialize this into a straight JSON object iff the key is a string-like type
+            // We could turn into (say) an array of keys and values
+
+            break;
+        }
+        case RttiInfo::Kind::Other:
+        {
+            if (rttiInfo == GetRttiInfo<JSONValue>::get())
+            {
+                // Do we need to copy into the container?
+                // As it stands we have to assume src is stored in container.
+                const JSONValue& src = *(const JSONValue*)in;
+
+                out = src;
+                return SLANG_OK;
+            }
+            break;
+        }
+        default: break;
+    }
+
+    return SLANG_E_NOT_IMPLEMENTED;
+}
+
+} // namespace Slang