From 9e084ffab37c276d40931a58633041a2e10de623 Mon Sep 17 00:00:00 2001 From: jsmall-nvidia Date: Tue, 23 Nov 2021 16:23:15 -0500 Subject: JSON-RPC test server (#2026) * #include an absolute path didn't work - because paths were taken to always be relative. * Use 'Process' to communicate with an command line tool. * Remove slang-win-stream * Tidy up windows ProcessUtil. * First version of BufferedReadStream. * Windows working IPC for steams. * Test proxy count option. * Split Process/ProcessUtil. Process is platform dependant. ProcessUtil are functions that are platform independent. * First implementation of Unix Process interface. * Unix process compiles on cygwin. * Fix typo in unix process. * Separate unix pipe stream error of invalid access, from pipe availability. * Fix in standard line extraction. * Make fd non blocking. * Fix issues with Windows Process streams. * Added UnixPipe. * Some fixes around UnixPipeStream. * Make a unix stream closed explicit. * Hack to debug linux process/stream. * Revert to old linux pipe handling. * Pass executable path for unit tests. Split out CommandLine into own source. * Small improvements in process/command line. * Check process behavior with crash. * Make stderr and stdout unbuffered for crash testing. * Only turn disable buffering in crash test. * Disable crash test on CI. * Fix crash on clang/linux. * Enable crash test. Remove _appendBuffer as can use StreamUtil functionality. * Added inital processing for http headers. * Small improvements to HttpHeader. * First pass HTTPPacketConnection working on windows. * Enable other Process communication tests. * Update comments. * WIP JSON RPC. * Add terminate to Process. Made JSONRPC a Util. * Small tidy up around HTTPPacketConnection. * Improve process termination options. * WIP for test-server. * Add diagnostics error handling to test-server. * Improved JSON support. Parsing/creating JSON-RPC messages. * WIP JSONRPC parsing. * First pass RttiInfo support. * WIP converting between JSON/native types. * Project files. * Split out RttiUtil. Made RttiInfo constuction thread safe. * WIP RTTI<->JSON. * Add diagnostics to JSON<->native conversions. * Make RttiInfo for structs globals. Avoids problem around derived types (like pointers), being able to cause an abort. * Add pointer support to RTTI. Fixed some compilation issues on linux. * Add fixed array support. * Added Rtti unit test. * Add rtti unit test. * Split out quoted/unquoted key handling. Fix bugs in JSON value/container. Added JSON native test. * Make default array allocator use malloc/free. Remove the new[] handler (doesn't work on visuals studio). * Fix for linux warning. * Remove some test code. * Fix issues on x86 win. * Fix warning on aarch64. * Fix some bugs in JSON parsing/handling. Make Rtti work copy/dtor/ctor struct types. * Testing JSON<->native with fixed array. Make makeArrayView explicit if it's just a single value. Added array type. * Fix getting arrayView. * Improve JSON diagnostic name. * First pass refactor using Rtti for JSON RPC. * First pass of test server using RTTI/JSON-RPC. * Added JSONRPCConnection. * Fix some naming issues. * First pass of test-server working. * Added unit test support for JSON-RPC test server. * Fix compilation issues on linux around template handling. * Typo fix. * Fix a bug around SourceLoc lookup with JSONContainer. * Set the console type to console for ISlangWriters. * Small improvements to test-server. * Small improvements in test-server. * Small fix. --- tools/slang-test/options.cpp | 5 ++ tools/slang-test/options.h | 1 + tools/slang-test/slang-test-main.cpp | 98 +++++++++++++++++++++++++++++++++--- tools/slang-test/test-context.cpp | 47 +++++++++++++++++ tools/slang-test/test-context.h | 13 ++++- 5 files changed, 157 insertions(+), 7 deletions(-) (limited to 'tools/slang-test') diff --git a/tools/slang-test/options.cpp b/tools/slang-test/options.cpp index d24ea83e2..4fe7ea6bd 100644 --- a/tools/slang-test/options.cpp +++ b/tools/slang-test/options.cpp @@ -81,6 +81,7 @@ static bool _isSubCommand(const char* arg) while (argCursor != argEnd) { char const* arg = *argCursor++; + if (arg[0] != '-') { // We need to determine if this is a command, the confusion is that @@ -129,6 +130,10 @@ static bool _isSubCommand(const char* arg) { optionsOut->defaultSpawnType = SpawnType::UseProxy; } + else if (strcmp(arg, "-use-test-server") == 0) + { + optionsOut->defaultSpawnType = SpawnType::UseTestServer; + } else if (strcmp(arg, "-v") == 0) { optionsOut->shouldBeVerbose = true; diff --git a/tools/slang-test/options.h b/tools/slang-test/options.h index 4a40cba9a..5dc3ca1fb 100644 --- a/tools/slang-test/options.h +++ b/tools/slang-test/options.h @@ -41,6 +41,7 @@ enum class SpawnType UseExe, UseSharedLibrary, UseProxy, + UseTestServer, }; struct Options diff --git a/tools/slang-test/slang-test-main.cpp b/tools/slang-test/slang-test-main.cpp index e007d1978..000ee462c 100644 --- a/tools/slang-test/slang-test-main.cpp +++ b/tools/slang-test/slang-test-main.cpp @@ -655,6 +655,62 @@ Result spawnAndWaitProxy(TestContext* context, const String& testPath, const Com return res; } +static Result _executeRPC(TestContext* context, const UnownedStringSlice& method, const RttiInfo* rttiInfo, const void* args, ExecuteResult& outRes) +{ + JSONRPCConnection* rpcConnection = context->getOrCreateJSONRPCConnection(); + if (!rpcConnection) + { + return SLANG_FAIL; + } + + // Execute + SLANG_RETURN_ON_FAIL(rpcConnection->sendCall(method, rttiInfo, args)); + + // Wait for the result + rpcConnection->waitForResult(context->timeOutInMs); + + if (!rpcConnection->hasMessage()) + { + // We can assume somethings gone wrong. So lets kill the connection and fail. + context->destroyRPCConnection(); + return SLANG_FAIL; + } + + if (rpcConnection->getMessageType() != JSONRPCMessageType::Result) + { + return SLANG_FAIL; + } + + // Get the result + TestServerProtocol::ExecutionResult exeRes; + SLANG_RETURN_ON_FAIL(rpcConnection->getMessage(&exeRes)); + + outRes.resultCode = exeRes.returnCode; + outRes.standardError = exeRes.stdError; + outRes.standardOutput = exeRes.stdOut; + + return SLANG_OK; +} + +template +static Result _executeRPC(TestContext* context, const UnownedStringSlice& method, const T* msg, ExecuteResult& outRes) +{ + return _executeRPC(context, method, GetRttiInfo::get(), (const void*)msg, outRes); +} + +Result spawnAndWaitTestServer(TestContext* context, const String& testPath, const CommandLine& inCmdLine, ExecuteResult& outRes) +{ + String exeName = Path::getFileNameWithoutExt(inCmdLine.m_executable); + + // This is a test tool execution + TestServerProtocol::ExecuteToolTestArgs args; + + args.toolName = exeName; + args.args = inCmdLine.m_args; + + return _executeRPC(context, TestServerProtocol::ExecuteToolTestArgs::g_methodName, &args, outRes); +} + static SlangResult _extractArg(const CommandLine& cmdLine, const String& argName, String& outValue) { SLANG_ASSERT(argName.getLength() > 0 && argName[0] == '-'); @@ -1012,6 +1068,11 @@ ToolReturnCode spawnAndWait(TestContext* context, const String& testPath, SpawnT spawnResult = spawnAndWaitProxy(context, testPath, cmdLine, outExeRes); break; } + case SpawnType::UseTestServer: + { + spawnResult = spawnAndWaitTestServer(context, testPath, cmdLine, outExeRes); + break; + } default: break; } @@ -1574,7 +1635,7 @@ TestResult runReflectionTest(TestContext* context, TestInput& input) TestResult result = TestResult::Pass; // Otherwise we compare to the expected output - if (actualOutput != expectedOutput) + if (!StringUtil::areLinesEqual(actualOutput.getUnownedSlice(), expectedOutput.getUnownedSlice())) { result = TestResult::Fail; } @@ -1981,7 +2042,7 @@ TestResult runCrossCompilerTest(TestContext* context, TestInput& input) TestResult result = TestResult::Pass; // Otherwise we compare to the expected output - if (actualOutput != expectedOutput) + if (!StringUtil::areLinesEqual(actualOutput.getUnownedSlice(), expectedOutput.getUnownedSlice())) { result = TestResult::Fail; } @@ -2135,7 +2196,7 @@ static TestResult _runHLSLComparisonTest( if (standardOutput.getLength() != 0) result = TestResult::Fail; } // Otherwise we compare to the expected output - else if (actualOutput != expectedOutput) + else if (!StringUtil::areLinesEqual(actualOutput.getUnownedSlice(), expectedOutput.getUnownedSlice())) { result = TestResult::Fail; } @@ -2272,7 +2333,7 @@ TestResult runGLSLComparisonTest(TestContext* context, TestInput& input) if( hlslResult == TestResult::Fail ) return TestResult::Fail; if( slangResult == TestResult::Fail ) return TestResult::Fail; - if (actualOutput != expectedOutput) + if (!StringUtil::areLinesEqual(actualOutput.getUnownedSlice(), expectedOutput.getUnownedSlice())) { context->reporter->dumpOutputDifference(expectedOutput, actualOutput); @@ -2511,7 +2572,8 @@ TestResult runComputeComparisonImpl(TestContext* context, TestInput& input, cons auto actualOutput = getOutput(exeRes); auto expectedOutput = getExpectedOutput(outputStem); - if (actualOutput != expectedOutput) + + if (!StringUtil::areLinesEqual(actualOutput.getUnownedSlice(), expectedOutput.getUnownedSlice())) { context->reporter->dumpOutputDifference(expectedOutput, actualOutput); @@ -2817,7 +2879,7 @@ TestResult runHLSLRenderComparisonTestImpl( if( hlslResult == TestResult::Fail ) return TestResult::Fail; if( slangResult == TestResult::Fail ) return TestResult::Fail; - if (actualOutput != expectedOutput) + if (!StringUtil::areLinesEqual(actualOutput.getUnownedSlice(), expectedOutput.getUnownedSlice())) { context->reporter->dumpOutputDifference(expectedOutput, actualOutput); @@ -3478,6 +3540,30 @@ static SlangResult runUnitTestModule(TestContext* context, TestOptions& testOpti reporter->addResult(testResult); } } + else if (spawnType == SpawnType::UseTestServer) + { + TestServerProtocol::ExecuteUnitTestArgs args; + args.enabledApis = context->options.enabledApis; + args.moduleName = moduleName; + args.testName = testName; + + { + TestReporter::TestScope scopeTest(reporter, testOptions.command); + ExecuteResult exeRes; + + SlangResult rpcRes = _executeRPC(context, TestServerProtocol::ExecuteUnitTestArgs::g_methodName, &args, exeRes); + const auto testResult = _asTestResult(ToolReturnCode(exeRes.resultCode)); + + // If the test fails, output any output - which might give information about individual tests that have failed. + if (SLANG_FAILED(rpcRes) || testResult == TestResult::Fail) + { + String output = getOutput(exeRes); + reporter->message(TestMessageType::TestFailure, output.getBuffer()); + } + + reporter->addResult(testResult); + } + } else { TestReporter::TestScope scopeTest(reporter, testOptions.command); diff --git a/tools/slang-test/test-context.cpp b/tools/slang-test/test-context.cpp index 2872ddb47..eeb23b695 100644 --- a/tools/slang-test/test-context.cpp +++ b/tools/slang-test/test-context.cpp @@ -106,6 +106,53 @@ DownstreamCompilerSet* TestContext::getCompilerSet() return compilerSet; } +SlangResult TestContext::_createJSONRPCConnection(RefPtr& out) +{ + RefPtr process; + + { + CommandLine cmdLine; + cmdLine.setExecutable(exeDirectoryPath.getBuffer(), "test-server"); + SLANG_RETURN_ON_FAIL(Process::create(cmdLine, Process::Flag::AttachDebugger, process)); + } + + Stream* writeStream = process->getStream(Process::StreamType::StdIn); + RefPtr readStream(new BufferedReadStream(process->getStream(Process::StreamType::StdOut))); + + RefPtr connection = new HTTPPacketConnection(readStream, writeStream); + RefPtr rpcConnection = new JSONRPCConnection; + + SLANG_RETURN_ON_FAIL(rpcConnection->init(connection, process)); + + out = rpcConnection; + + return SLANG_OK; +} + + +void TestContext::destroyRPCConnection() +{ + if (m_jsonRpcConnection) + { + m_jsonRpcConnection->disconnect(); + m_jsonRpcConnection.setNull(); + } +} + +Slang::JSONRPCConnection* TestContext::getOrCreateJSONRPCConnection() +{ + if (!m_jsonRpcConnection) + { + if (SLANG_FAILED(_createJSONRPCConnection(m_jsonRpcConnection))) + { + return nullptr; + } + } + + return m_jsonRpcConnection; +} + + Slang::DownstreamCompiler* TestContext::getDefaultCompiler(SlangSourceLanguage sourceLanguage) { DownstreamCompilerSet* set = getCompilerSet(); diff --git a/tools/slang-test/test-context.h b/tools/slang-test/test-context.h index 67707028d..203005943 100644 --- a/tools/slang-test/test-context.h +++ b/tools/slang-test/test-context.h @@ -11,6 +11,7 @@ #include "../../source/core/slang-render-api-util.h" #include "../../source/compiler-core/slang-downstream-compiler.h" +#include "../../source/compiler-core/slang-json-rpc-connection.h" #include "../../slang-com-ptr.h" @@ -110,6 +111,9 @@ class TestContext Slang::DownstreamCompilerSet* getCompilerSet(); Slang::DownstreamCompiler* getDefaultCompiler(SlangSourceLanguage sourceLanguage); + Slang::JSONRPCConnection* getOrCreateJSONRPCConnection(); + void destroyRPCConnection(); + /// Ctor TestContext(); /// Dtor @@ -129,14 +133,21 @@ class TestContext Slang::RefPtr compilerSet; Slang::String exeDirectoryPath; - + + Slang::Int timeOutInMs = 100 * 1000; + + protected: + SlangResult _createJSONRPCConnection(Slang::RefPtr& out); + struct SharedLibraryTool { Slang::ComPtr m_sharedLibrary; InnerMainFunc m_func; }; + Slang::RefPtr m_jsonRpcConnection; + SlangSession* m_session; Slang::Dictionary m_sharedLibTools; -- cgit v1.2.3