Support array args for JSON-RPC (#2046)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Add ability to send/receive JSON-RPC params optionally as an array.

* More array conversions to json-native.

* Simplified setting up of 'CallStyle' on JSONRPCConnection.

* Small simplification in JSONRPCConnection.

* Small improvements around JSON-RPC connection.

* Improve some comments.
Kick CI build.

6 files changed, 201 insertions, 24 deletions

diff --git a/source/compiler-core/slang-json-native.cpp b/source/compiler-core/slang-json-native.cpp
index 9c972eba3..6e54457d0 100644
--- a/source/compiler-core/slang-json-native.cpp
+++ b/source/compiler-core/slang-json-native.cpp
@@ -252,6 +252,52 @@ SlangResult JSONToNativeConverter::convert(const JSONValue& in, const RttiInfo*
     return SLANG_FAIL;
 }
 
+SlangResult JSONToNativeConverter::convertArrayToStruct(const JSONValue& value, const RttiInfo* rttiInfo, void* out)
+{
+    // Check converting JSON array into a struct, as that's what this method supports
+    if (!(rttiInfo->m_kind == RttiInfo::Kind::Struct &&
+        value.getKind() == JSONValue::Kind::Array))
+    {
+        // If they are the wrong types then just fail
+        return SLANG_FAIL;
+    }
+
+    // Find the total amount of fields, and all the classes involved
+    Index totalFieldCount = 0;
+
+    ShortList<const StructRttiInfo*, 8> infos;
+    for (const StructRttiInfo* cur = static_cast<const StructRttiInfo*>(rttiInfo); cur; cur = cur->m_super)
+    {
+        totalFieldCount += cur->m_fieldCount;
+        infos.add(cur);
+    }
+
+    // Must have the same amount of fields
+    auto array = m_container->getArray(value);
+    if (array.getCount() != totalFieldCount)
+    {
+        return SLANG_FAIL;
+    }
+
+    Byte* dstBase = (Byte*)out;
+
+    // We work in the order from the base class to the final type
+    Index argIndex = 0;
+    for (Index i = infos.getCount() - 1; i >= 0; --i)
+    {
+        auto info = infos[i];
+
+        const Index fieldCount = info->m_fieldCount;
+        for (Index j = 0; j < fieldCount; ++j)
+        {
+            // Convert the field
+            const auto& field = info->m_fields[j];
+            SLANG_RETURN_ON_FAIL(convert(array[argIndex++], field.m_type, dstBase + field.m_offset));
+        }
+    }
+
+    return SLANG_OK;
+}
 
 /* !!!!!!!!!!!!!!!!!!!!!!!!!!!! NativeToJSONConverter !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */
 
@@ -420,4 +466,52 @@ SlangResult NativeToJSONConverter::convert(const RttiInfo* rttiInfo, const void*
     return SLANG_E_NOT_IMPLEMENTED;
 }
 
+SlangResult NativeToJSONConverter::convertStructToArray(const RttiInfo* rttiInfo, const void* in, JSONValue& out)
+{
+    if (rttiInfo->m_kind != RttiInfo::Kind::Struct)
+    {
+        // Must be a struct
+        return SLANG_FAIL;
+    }
+
+    // Work out the total amount of fields, and all invloved struct types
+    Index totalFieldsCount = 0;
+    ShortList<const StructRttiInfo*, 8> infos;
+    for (const StructRttiInfo* cur = static_cast<const StructRttiInfo*>(rttiInfo); cur; cur = cur->m_super)
+    {
+        totalFieldsCount += Index(cur->m_fieldCount);
+        infos.add(cur);
+    }
+
+    // Convert the args/params
+    List<JSONValue> argsArray;
+    argsArray.setCount(totalFieldsCount);
+
+    // NOTE! We do no special handling here around optional parameters.
+    // All fields of the input args are output
+    {
+        Index argsArrayIndex = 0;
+        const Byte* argsBase = (const Byte*)in;
+
+        // Work in the order from the base class to the actual type
+        for (Index i = infos.getCount() - 1; i >= 0; --i)
+        {
+            auto structRttiInfo = infos[i];
+            const Index fieldCount = Index(structRttiInfo->m_fieldCount);
+
+            for (Index j = 0; j < fieldCount; ++j)
+            {
+                const auto& field = structRttiInfo->m_fields[j];
+                // Convert the field
+                SLANG_RETURN_ON_FAIL(convert(field.m_type, argsBase + field.m_offset, argsArray[argsArrayIndex++]));
+            }
+        }
+    }
+
+    // Okay now we convert the List to the output, which will just be a JSON array.
+    SLANG_RETURN_ON_FAIL(convert(&argsArray, out));
+
+    return SLANG_OK;
+}
+
 } // namespace Slang
diff --git a/source/compiler-core/slang-json-native.h b/source/compiler-core/slang-json-native.h
index f9ec93119..4235a99a8 100644
--- a/source/compiler-core/slang-json-native.h
+++ b/source/compiler-core/slang-json-native.h
@@ -15,6 +15,10 @@ struct JSONToNativeConverter
     template <typename T>
     SlangResult convert(const JSONValue& value, T* in) { return convert(value, GetRttiInfo<T>::get(), (void*)in); }
 
+    template <typename T>
+    SlangResult convertArrayToStruct(const JSONValue& value, T* in) { return convertArrayToStruct(value, GetRttiInfo<T>::get(), (void*)in); }
+    SlangResult convertArrayToStruct(const JSONValue& value, const RttiInfo* rttiInfo, void* out);
+
     JSONToNativeConverter(JSONContainer* container, DiagnosticSink* sink):
         m_container(container),
         m_sink(sink)
@@ -37,6 +41,11 @@ struct NativeToJSONConverter
     template <typename T>
     SlangResult convert(T* in, JSONValue& out) { return convert(GetRttiInfo<T>::get(), (const void*)in, out); }
 
+    SlangResult convertStructToArray(const RttiInfo* rttiInfo, const void* in, JSONValue& out);
+    template <typename T>
+    SlangResult convertStructToArray(T* in, JSONValue& out) { return convertStructToArray(GetRttiInfo<T>::get(), (const void*)in, out); }
+
+
     NativeToJSONConverter(JSONContainer* container, DiagnosticSink* sink) :
         m_container(container),
         m_sink(sink)
diff --git a/source/compiler-core/slang-json-rpc-connection.cpp b/source/compiler-core/slang-json-rpc-connection.cpp
index 0e37ccd77..cad2f8e19 100644
--- a/source/compiler-core/slang-json-rpc-connection.cpp
+++ b/source/compiler-core/slang-json-rpc-connection.cpp
@@ -3,17 +3,26 @@
 
 #include "../core/slang-string-util.h"
 #include "../core/slang-process-util.h"
+#include "../core/slang-short-list.h"
 
 #include "slang-json-rpc.h"
 #include "slang-json-native.h"
 
 namespace Slang {
 
-SlangResult JSONRPCConnection::init(HTTPPacketConnection* connection, Process* process)
+SlangResult JSONRPCConnection::init(HTTPPacketConnection* connection, CallStyle defaultCallStyle, Process* process)
 {
     m_connection = connection;
     m_process = process;
 
+    {
+        // If a call style isn't set, use the prefered style
+        const CallStyle preferedCallStyle = CallStyle::Array;
+        defaultCallStyle = (defaultCallStyle == CallStyle::Default) ? preferedCallStyle : defaultCallStyle;
+        m_defaultCallStyle = defaultCallStyle;
+    }
+        
+    
     m_sourceManager.initialize(nullptr, nullptr);
     m_diagnosticSink.init(&m_sourceManager, &JSONLexer::calcLexemeLocation);
     m_container.setSourceManager(&m_sourceManager);
@@ -21,7 +30,7 @@ SlangResult JSONRPCConnection::init(HTTPPacketConnection* connection, Process* p
     return SLANG_OK;
 }
 
-SlangResult JSONRPCConnection::initWithStdStreams(Process* process)
+SlangResult JSONRPCConnection::initWithStdStreams(CallStyle defaultCallStyle, Process* process)
 {
     RefPtr<Stream> stdinStream, stdoutStream;
 
@@ -31,7 +40,7 @@ SlangResult JSONRPCConnection::initWithStdStreams(Process* process)
     RefPtr<BufferedReadStream> readStream(new BufferedReadStream(stdinStream));
 
     RefPtr<HTTPPacketConnection> connection = new HTTPPacketConnection(readStream, stdoutStream);
-    return init(connection, process);
+    return init(connection, defaultCallStyle, process);
 }
 
 void JSONRPCConnection::clearBuffers()
@@ -115,19 +124,40 @@ SlangResult JSONRPCConnection::sendError(JSONRPC::ErrorCode errorCode, const Uno
     errorResponse.error.message = msg;
     errorResponse.id = id;
 
-    
     return sendRPC(&errorResponse);
 }
 
+SlangResult JSONRPCConnection::toNativeArgsOrSendError(const JSONValue& srcArgs, const RttiInfo* dstArgsRttiInfo, void* dstArgs, const JSONValue& id)
+{
+    if (dstArgsRttiInfo->m_kind == RttiInfo::Kind::Struct &&
+        srcArgs.getKind() == JSONValue::Kind::Array)
+    {
+        JSONToNativeConverter converter(&m_container, &m_diagnosticSink);
+        if (SLANG_FAILED(converter.convertArrayToStruct(srcArgs, dstArgsRttiInfo, dstArgs)))
+        {
+            return sendError(JSONRPC::ErrorCode::InvalidRequest, id);
+        }
+        return SLANG_OK;
+    }
+    else
+    {
+        return toNativeOrSendError(srcArgs, dstArgsRttiInfo, dstArgs, id);
+    }
+}
+
 SlangResult JSONRPCConnection::toNativeOrSendError(const JSONValue& value, const RttiInfo* info, void* dst, const JSONValue& id)
 {
     m_diagnosticSink.outputBuffer.Clear();
-    if (SLANG_FAILED(JSONRPCUtil::convertToNative(&m_container, value, &m_diagnosticSink, info, dst)))
+
+    JSONToNativeConverter converter(&m_container, &m_diagnosticSink);
+    
+    if (SLANG_FAILED(converter.convert(value, info, dst)))
     {
         return sendError(JSONRPC::ErrorCode::InvalidRequest, id);
     }
+
     return SLANG_OK;
-}
+} 
 
 SlangResult JSONRPCConnection::sendCall(const UnownedStringSlice& method, const JSONValue& id)
 {
@@ -154,14 +184,31 @@ SlangResult JSONRPCConnection::sendResult(const RttiInfo* rttiInfo, const void*
 
 SlangResult JSONRPCConnection::sendCall(const UnownedStringSlice& method, const RttiInfo* argsRttiInfo, const void* args, const JSONValue& id)
 {
+    return sendCall(m_defaultCallStyle, method, argsRttiInfo, args, id);
+}
+
+SlangResult JSONRPCConnection::sendCall(CallStyle callStyle, const UnownedStringSlice& method, const RttiInfo* argsRttiInfo, const void* args, const JSONValue& id)
+{
     JSONRPCCall call;
     call.id = id;
     call.method = method;
 
-    // Convert the args/params
+    // Set up the converter to now convert the args.
     NativeToJSONConverter converter(&m_container, &m_diagnosticSink);
-    SLANG_RETURN_ON_FAIL(converter.convert(argsRttiInfo, args, call.params));
-
+    
+    // If we have a struct *and* call style is 'array', do special handling
+    if (argsRttiInfo->m_kind == RttiInfo::Kind::Struct &&
+        _getCallStyle(callStyle) == CallStyle::Array)
+    {
+        // Convert the args/params in the 'array' style
+        SLANG_RETURN_ON_FAIL(converter.convertStructToArray(argsRttiInfo, args, call.params));
+    }
+    else
+    {
+        // Convert the args/params in the 'object' sytle
+        SLANG_RETURN_ON_FAIL(converter.convert(argsRttiInfo, args, call.params));
+    }
+    
     // Send the RPC
     SLANG_RETURN_ON_FAIL(sendRPC(&call));
     return SLANG_OK;
diff --git a/source/compiler-core/slang-json-rpc-connection.h b/source/compiler-core/slang-json-rpc-connection.h
index 263baf799..1436e9601 100644
--- a/source/compiler-core/slang-json-rpc-connection.h
+++ b/source/compiler-core/slang-json-rpc-connection.h
@@ -31,12 +31,19 @@ class JSONRPCConnection : public RefObject
 {
 public:
 
+    enum class CallStyle
+    {
+        Default,        ///< The default
+        Object,         ///< Params are passed as an object
+        Array,          ///< Params are passed as an array
+    };
+
         /// An init function must be called before use
         /// If a process is implementing the server it should be passed in if the process needs to shut down if the connection does
-    SlangResult init(HTTPPacketConnection* connection, Process* process = nullptr);
+    SlangResult init(HTTPPacketConnection* connection, CallStyle callStyle = CallStyle::Default, Process* process = nullptr);
 
         /// Initialize using stdin/out streams for input/output. 
-    SlangResult initWithStdStreams(Process* process = nullptr);
+    SlangResult initWithStdStreams(CallStyle callStyle = CallStyle::Default, Process* process = nullptr);
 
         /// Disconnect. May block while server shuts down
     void disconnect();
@@ -46,6 +53,15 @@ public:
     template <typename T>
     SlangResult toNativeOrSendError(const JSONValue& value, T* data, const JSONValue& id) { return toNativeOrSendError(value, GetRttiInfo<T>::get(), data, id); }
 
+        /// Convert value to dst.
+        /// The 'Args' aspect here is to handle Args/Params in JSON-RPC which can be specified as an array or object style.
+        /// This call will automatically handle either case.
+        /// toNativeOrSendError does not assume the thing being converted is args, and so doesn't allow such a transformation.
+        /// Will write error response on failure.
+    SlangResult toNativeArgsOrSendError(const JSONValue& srcArgs, const RttiInfo* dstArgsRttiInfo, void* dstArgs, const JSONValue& id);
+    template <typename T>
+    SlangResult toNativeArgsOrSendError(const JSONValue& srcArgs, T* dstArgs, const JSONValue& id) { return toNativeArgsOrSendError(srcArgs, GetRttiInfo<T>::get(), dstArgs, id); }
+
     template <typename T>
     SlangResult toValidNativeOrSendError(const JSONValue& value, T* data, const JSONValue& id);
 
@@ -58,12 +74,19 @@ public:
     SlangResult sendError(JSONRPC::ErrorCode code, const JSONValue& id);
     SlangResult sendError(JSONRPC::ErrorCode errorCode, const UnownedStringSlice& msg, const JSONValue& id);
 
-        /// Send a call
-        /// If no id is needed, id can just be invalid 
+        /// Send a 'call'
+        /// Uses the default CallStyle as set when init
     SlangResult sendCall(const UnownedStringSlice& method, const RttiInfo* argsRttiInfo, const void* args, const JSONValue& id = JSONValue());
     template <typename T>
     SlangResult sendCall(const UnownedStringSlice& method, const T* args, const JSONValue& id = JSONValue()) { return sendCall(method, GetRttiInfo<T>::get(), (const void*)args, id); }
-        ///
+
+        /// Send a 'call'
+        /// Uses the call mechanism specified in callStyle. It is valid to pass as Default. 
+    SlangResult sendCall(CallStyle callStyle, const UnownedStringSlice& method, const RttiInfo* argsRttiInfo, const void* args, const JSONValue& id = JSONValue());
+    template <typename T>
+    SlangResult sendCall(CallStyle callStyle, const UnownedStringSlice& method, const T* args, const JSONValue& id = JSONValue()) { return sendCall(callStyle, method, GetRttiInfo<T>::get(), (const void*)args, id); }
+
+        /// Send a call, wheret there are no arguments
     SlangResult sendCall(const UnownedStringSlice& method, const JSONValue& id = JSONValue());
 
     template <typename T>
@@ -93,12 +116,12 @@ public:
     SlangResult getRPCOrSendError(T* out) { return getRPCOrSendError(GetRttiInfo<T>::get(), (void*)out); }
     SlangResult getRPCOrSendError(const RttiInfo* rttiInfo, void* out);
 
-    /// Get message (has to be part of JSONRPCResultResponse)
+        /// Get message (has to be part of JSONRPCResultResponse)
     template <typename T>
     SlangResult getMessage(T* out) { return getMessage(GetRttiInfo<T>::get(), (void*)out); }
     SlangResult getMessage(const RttiInfo* rttiInfo, void* out);
 
-    /// If there is a message and there is a failure, will send an error response
+        /// If there is a message and there is a failure, will send an error response
     template <typename T>
     SlangResult getMessageOrSendError(T* out) { return getMessageOrSendError(GetRttiInfo<T>::get(), (void*)out); }
     SlangResult getMessageOrSendError(const RttiInfo* rttiInfo, void* out);
@@ -130,17 +153,21 @@ public:
     JSONRPCConnection():m_container(nullptr) {}
 
 protected:
+    CallStyle _getCallStyle(CallStyle callStyle) const { return (callStyle == CallStyle::Default) ? m_defaultCallStyle : callStyle; }
+
     RefPtr<Process> m_process;                       ///< Backing process (optional)
     RefPtr<HTTPPacketConnection> m_connection;       ///< The underlying 'transport' connection, whilst HTTP currently doesn't have to be 
 
-    DiagnosticSink m_diagnosticSink;                        ///< Holds any diagnostics typically generated by parsing JSON, producing JSON from native types
+    DiagnosticSink m_diagnosticSink;                ///< Holds any diagnostics typically generated by parsing JSON, producing JSON from native types
 
-    SourceManager m_sourceManager;                          ///< Holds the JSON text for current message/output. Is cleared regularly.
-    JSONContainer m_container;                              ///< Holds the backing memory for jsonMemory, and used when converting input into output JSON
+    SourceManager m_sourceManager;                  ///< Holds the JSON text for current message/output. Is cleared regularly.
+    JSONContainer m_container;                      ///< Holds the backing memory for jsonMemory, and used when converting input into output JSON
 
-    JSONValue m_jsonRoot;                                   ///< The root JSON value for the currently read message. 
+    JSONValue m_jsonRoot;                           ///< The root JSON value for the currently read message. 
 
-    Int m_terminationTimeOutInMs = 1 * 1000;                ///< Time to wait for termination response. Default is 1 second
+    CallStyle m_defaultCallStyle = CallStyle::Array;    ///< The default calling style
+    
+    Int m_terminationTimeOutInMs = 1 * 1000;        ///< Time to wait for termination response. Default is 1 second
 };
 
 // ---------------------------------------------------------------------------
diff --git a/tools/slang-test/test-context.cpp b/tools/slang-test/test-context.cpp
index 2f0e23815..d5af41a78 100644
--- a/tools/slang-test/test-context.cpp
+++ b/tools/slang-test/test-context.cpp
@@ -128,7 +128,7 @@ SlangResult TestContext::_createJSONRPCConnection(RefPtr<JSONRPCConnection>& out
     RefPtr<HTTPPacketConnection> connection = new HTTPPacketConnection(readStream, writeStream);
     RefPtr<JSONRPCConnection> rpcConnection = new JSONRPCConnection;
 
-    SLANG_RETURN_ON_FAIL(rpcConnection->init(connection, process));
+    SLANG_RETURN_ON_FAIL(rpcConnection->init(connection, JSONRPCConnection::CallStyle::Default, process));
 
     out = rpcConnection;
 
diff --git a/tools/test-server/test-server-main.cpp b/tools/test-server/test-server-main.cpp
index a32d8c542..b4eead664 100644
--- a/tools/test-server/test-server-main.cpp
+++ b/tools/test-server/test-server-main.cpp
@@ -371,7 +371,7 @@ SlangResult TestServer::_executeUnitTest(const JSONRPCCall& call)
     auto id = m_connection->getPersistentValue(call.id);
 
     TestServerProtocol::ExecuteUnitTestArgs args;
-    SLANG_RETURN_ON_FAIL(m_connection->toNativeOrSendError(call.params, &args, call.id));
+    SLANG_RETURN_ON_FAIL(m_connection->toNativeArgsOrSendError(call.params, &args, call.id));
 
     auto sink = m_connection->getSink();
 
@@ -443,7 +443,7 @@ SlangResult TestServer::_executeTool(const JSONRPCCall& call)
 
     TestServerProtocol::ExecuteToolTestArgs args;
     
-    SLANG_RETURN_ON_FAIL(m_connection->toNativeOrSendError(call.params, &args, id));
+    SLANG_RETURN_ON_FAIL(m_connection->toNativeArgsOrSendError(call.params, &args, id));
 
     auto sink = m_connection->getSink();