Use test-server on CI (#2034)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Vary what SpawnType is used, if one isn't explicitly set.

* Terminate on linux if exec fails.

* Use a more sophisticated sleeping mechanism.

* Attempt to make CI tests to work on aarch64 debug.
Small fixes.

8 files changed, 128 insertions, 26 deletions

diff --git a/source/compiler-core/slang-json-rpc-connection.h b/source/compiler-core/slang-json-rpc-connection.h
index b566140ef..263baf799 100644
--- a/source/compiler-core/slang-json-rpc-connection.h
+++ b/source/compiler-core/slang-json-rpc-connection.h
@@ -130,8 +130,8 @@ public:
     JSONRPCConnection():m_container(nullptr) {}
 
 protected:
-    RefPtr<Slang::Process> m_process;                       ///< Backing process (optional)
-    RefPtr<Slang::HTTPPacketConnection> m_connection;       ///< The underlying 'transport' connection, whilst HTTP currently doesn't have to be 
+    RefPtr<Process> m_process;                       ///< Backing process (optional)
+    RefPtr<HTTPPacketConnection> m_connection;       ///< The underlying 'transport' connection, whilst HTTP currently doesn't have to be 
 
     DiagnosticSink m_diagnosticSink;                        ///< Holds any diagnostics typically generated by parsing JSON, producing JSON from native types
 
@@ -140,10 +140,7 @@ protected:
 
     JSONValue m_jsonRoot;                                   ///< The root JSON value for the currently read message. 
 
-        /// Default timeout is 10 seconds
-    Int m_timeOutInMs = 10 * 1000;
-        /// Termination timeout
-    Int m_terminationTimeOutInMs = 1 * 1000;
+    Int m_terminationTimeOutInMs = 1 * 1000;                ///< Time to wait for termination response. Default is 1 second
 };
 
 // ---------------------------------------------------------------------------
diff --git a/source/core/slang-http.cpp b/source/core/slang-http.cpp
index 6a4bb4f92..bb5994635 100644
--- a/source/core/slang-http.cpp
+++ b/source/core/slang-http.cpp
@@ -287,6 +287,46 @@ SlangResult HTTPPacketConnection::update()
     return m_readResult;
 }
 
+
+namespace { // anonymous
+
+// Handles binary backoff like sleeping mechanism.
+struct SleepState
+{
+    void sleep()
+    {
+        Process::sleepCurrentThread(m_intervalInMs);
+        _update();
+    }
+    void reset()
+    {
+        m_intervalInMs = 0;
+        m_count = 0;
+    }
+    void _update()
+    {
+        const Int maxIntervalInMs = 32;
+        const Int initialCountThreshold = 4;
+
+        ++m_count;
+
+        const Int countThreshold = (m_intervalInMs == 0) ? initialCountThreshold : 1;
+
+        // If we hit the count change the interval
+        if (m_count >= countThreshold)
+        {
+            m_intervalInMs = (m_intervalInMs == 0) ? 1 : Math::Min(m_intervalInMs * 2, maxIntervalInMs);
+            // Reset the count
+            m_count = 0;
+        }
+    }
+
+    Int m_intervalInMs = 0;
+    Int m_count = 0;
+};
+
+} // anonymous
+
 SlangResult HTTPPacketConnection::waitForResult(Int timeOutInMs)
 {
     m_readResult = SLANG_OK;
@@ -300,6 +340,8 @@ SlangResult HTTPPacketConnection::waitForResult(Int timeOutInMs)
         startTick = Process::getClockTick();
     }
 
+    SleepState sleepState;
+
     while (m_readState == ReadState::Header ||
         m_readState == ReadState::Content)
     {
@@ -320,8 +362,11 @@ SlangResult HTTPPacketConnection::waitForResult(Int timeOutInMs)
 
         if (prevCount == m_readStream->getCount())
         {
-            // Yield if it appears nothing was read.
-            Process::sleepCurrentThread(0);
+            sleepState.sleep();
+        }
+        else
+        {
+            sleepState.reset();
         }
     }
 
diff --git a/source/core/unix/slang-unix-process.cpp b/source/core/unix/slang-unix-process.cpp
index 5131e37ec..4e8344486 100644
--- a/source/core/unix/slang-unix-process.cpp
+++ b/source/core/unix/slang-unix-process.cpp
@@ -349,6 +349,8 @@ SlangResult UnixPipeStream::write(const void* buffer, size_t length)
     return StringEscapeUtil::getHandler(StringEscapeUtil::Style::Space);
 }
 
+static const int kCannotExecute = 126;
+
 /* static */SlangResult Process::create(const CommandLine& commandLine, Process::Flags flags, RefPtr<Process>& outProcess)
 {
     List<char const*> argPtrs;
@@ -376,6 +378,12 @@ SlangResult UnixPipeStream::write(const void* buffer, size_t length)
         return SLANG_FAIL;
     }
 
+    // TODO(JS): 
+    // Ideally we'd have a mechanism to test if execvp can be successfully executed (at least in principal) before
+    // doing fork. Once we have forked we then have the problem of communicating to the parent process somehow.
+    //
+    // We could do a search down PATH and test files etc, but this seems to repeat a lot of the functionality of exec.
+    //    
     pid_t childPid = fork();
     if (childPid == -1)
     {
@@ -402,6 +410,11 @@ SlangResult UnixPipeStream::write(const void* buffer, size_t length)
         ::close(stdinPipe[0]);
         ::close(stdinPipe[1]);
 
+        // TODO(JS): Strictly speaking if m_executableType is 'Path' then we shouldn't be searching. Ie which
+        // exec we use here should be dependent on the executable type.
+
+        // Path = execvp
+        // Filename = execv
         ::execvp(argPtrs[0], (char* const*)&argPtrs[0]);
 
         // If we get here, then `exec` failed
@@ -410,6 +423,9 @@ SlangResult UnixPipeStream::write(const void* buffer, size_t length)
         // the terminal but in the stderrPipe. 
         fprintf(stderr, "error: `exec` failed\n");
 
+        // Terminate with failure. 
+        exit(kCannotExecute);
+
         return SLANG_FAIL;
     }
     else
diff --git a/tools/slang-test/options.cpp b/tools/slang-test/options.cpp
index d4dedcf81..48a43d621 100644
--- a/tools/slang-test/options.cpp
+++ b/tools/slang-test/options.cpp
@@ -134,10 +134,6 @@ static bool _isSubCommand(const char* arg)
         {
             optionsOut->defaultSpawnType = SpawnType::UseFullyIsolatedTestServer;
         }
-        else if (strcmp(arg, "-use-test-server") == 0)
-        {
-            optionsOut->defaultSpawnType = SpawnType::UseTestServer;
-        }
         else if (strcmp(arg, "-v") == 0)
         {
             optionsOut->shouldBeVerbose = true;
diff --git a/tools/slang-test/options.h b/tools/slang-test/options.h
index 3b62c903b..cf86bc901 100644
--- a/tools/slang-test/options.h
+++ b/tools/slang-test/options.h
@@ -38,7 +38,8 @@ protected:
 
 enum class SpawnType
 {
-    UseExe,
+    Default,                                    ///< Default - typically uses shared library, on CI may use TestServer    
+    UseExe,                                     ///< Tests using executable (for example slangc)
     UseSharedLibrary,                           ///< Runs testing in process (a crash tan take down the 
     UseTestServer,                              ///< Use the test server to isolate testing
     UseFullyIsolatedTestServer,                 ///< Uses a test server for each test (slow!)
@@ -78,11 +79,9 @@ struct Options
     bool dumpOutputOnFailure = false;
 
     // Set the default spawn type to use
-    // Set to SpawnType::UseProxy, if isolation of test execution is desired.
     // Having tests isolated, slows down testing considerably, so using UseSharedLibrary is the most
     // desirable default usually.
-    SpawnType defaultSpawnType = SpawnType::UseSharedLibrary;
-    //SpawnType defaultSpawnType = SpawnType::UseProxy;
+    SpawnType defaultSpawnType = SpawnType::Default;
     
     // kind of output to generate
     TestOutputMode outputMode = TestOutputMode::Default;
diff --git a/tools/slang-test/slang-test-main.cpp b/tools/slang-test/slang-test-main.cpp
index ac8ec0e10..e734e1c4c 100644
--- a/tools/slang-test/slang-test-main.cpp
+++ b/tools/slang-test/slang-test-main.cpp
@@ -674,7 +674,7 @@ static Result _executeRPC(TestContext* context, SpawnType spawnType, const Unown
     SLANG_RETURN_ON_FAIL(rpcConnection->sendCall(method, rttiInfo, args));
 
     // Wait for the result
-    rpcConnection->waitForResult(context->timeOutInMs);
+    rpcConnection->waitForResult(context->connectionTimeOutInMs);
 
     if (!rpcConnection->hasMessage())
     {
@@ -1057,14 +1057,17 @@ ToolReturnCode spawnAndWait(TestContext* context, const String& testPath, SpawnT
 
     const auto& options = context->options;
 
+    const auto finalSpawnType = context->getFinalSpawnType(spawnType);
+
     SlangResult spawnResult = SLANG_FAIL;
-    switch (spawnType)
+    switch (finalSpawnType)
     {
         case SpawnType::UseExe:
         {
             spawnResult = spawnAndWaitExe(context, testPath, cmdLine, outExeRes);
             break;
         }
+        case SpawnType::Default:
         case SpawnType::UseSharedLibrary:
         {
             spawnResult = spawnAndWaitSharedLibrary(context, testPath, cmdLine, outExeRes);
@@ -1073,7 +1076,7 @@ ToolReturnCode spawnAndWait(TestContext* context, const String& testPath, SpawnT
         case SpawnType::UseFullyIsolatedTestServer:
         case SpawnType::UseTestServer:
         {
-            spawnResult = spawnAndWaitTestServer(context, spawnType, testPath, cmdLine, outExeRes);
+            spawnResult = spawnAndWaitTestServer(context, finalSpawnType, testPath, cmdLine, outExeRes);
             break;
         }
         default: break;
@@ -3601,7 +3604,8 @@ SlangResult innerMain(int argc, char** argv)
     {
         SlangSession* session = context.getSession();
 
-        StdWriters::getOut().print("Supported backends:");
+        auto out = StdWriters::getOut();
+        out.print("Supported backends:");
 
         for (int i = 0; i < SLANG_PASS_THROUGH_COUNT_OF; ++i)
         {
@@ -3624,11 +3628,11 @@ SlangResult innerMain(int argc, char** argv)
                 SLANG_ASSERT(passThroughCategories[i] == nullptr);
                 passThroughCategories[i] = categorySet.add(buf.getBuffer() + 1, fullTestCategory);
 
-                StdWriters::getOut().write(buf.getBuffer(), buf.getLength());
+                out.write(buf.getBuffer(), buf.getLength());
             }
         }
 
-        StdWriters::getOut().print("\n");
+        out.print("\n");
     }
 
     // Working out what renderApis is worked on on demand through
@@ -3727,18 +3731,20 @@ SlangResult innerMain(int argc, char** argv)
             TestReporter::SuiteScope suiteScope(&reporter, "unit tests");
             TestReporter::set(&reporter);
 
+            const auto spawnType = context.getFinalSpawnType();
+
             // Run the unit tests
             {
                 TestOptions testOptions;
                 testOptions.categories.add(unitTestCategory);
                 testOptions.categories.add(smokeTestCategory);
-                runUnitTestModule(&context, testOptions, context.options.defaultSpawnType, "slang-unit-test-tool");
+                runUnitTestModule(&context, testOptions, spawnType, "slang-unit-test-tool");
             }
 
             {
                 TestOptions testOptions;
                 testOptions.categories.add(unitTestCategory);
-                runUnitTestModule(&context, testOptions, context.options.defaultSpawnType, "gfx-unit-test-tool");
+                runUnitTestModule(&context, testOptions, spawnType, "gfx-unit-test-tool");
             }
              
             TestReporter::set(nullptr);
diff --git a/tools/slang-test/test-context.cpp b/tools/slang-test/test-context.cpp
index eeb23b695..39dd52313 100644
--- a/tools/slang-test/test-context.cpp
+++ b/tools/slang-test/test-context.cpp
@@ -15,6 +15,12 @@ using namespace Slang;
 TestContext::TestContext() 
 {
     m_session = nullptr;
+
+    /// if we are testing on arm, debug, we may want to increase the connection timeout
+#if (SLANG_PROCESSOR_ARM || SLANG_PROCESSOR_ARM_64) && defined(_DEBUG)
+    // 5 mins(!). This seems to be the order of time needed for timeout on a CI ARM test system on debug
+    connectionTimeOutInMs = 1000 * 60 * 5;
+#endif
 }
 
 Result TestContext::init(const char* exePath)
@@ -170,4 +176,25 @@ bool TestContext::canRunTestWithRenderApiFlags(Slang::RenderApiFlags requiredFla
     return (requiredFlags & options.enabledApis) == requiredFlags;
 }
 
+SpawnType TestContext::getFinalSpawnType(SpawnType spawnType)
+{
+    if (spawnType == SpawnType::Default)
+    {
+        if (options.outputMode == TestOutputMode::Default)
+        {
+            return SpawnType::UseSharedLibrary;
+        }
+        else
+        {
+            return SpawnType::UseTestServer;
+        }
+    }
+
+    // Just return whatever spawnType was passed in
+    return spawnType;
+}
 
+SpawnType TestContext::getFinalSpawnType()
+{
+    return getFinalSpawnType(options.defaultSpawnType);
+}
diff --git a/tools/slang-test/test-context.h b/tools/slang-test/test-context.h
index 203005943..03b74217f 100644
--- a/tools/slang-test/test-context.h
+++ b/tools/slang-test/test-context.h
@@ -107,6 +107,12 @@ class TestContext
         /// True if can run unit tests
     bool canRunUnitTests() const { return options.apiOnly == false; }
 
+        /// Given a spawn type, return the final spawn type.
+        /// In particular we want 'Default' spawn type to vary by the environment (for example running on test server on CI)
+    SpawnType getFinalSpawnType(SpawnType spawnType);
+
+    SpawnType getFinalSpawnType();
+
         /// Get compiler set
     Slang::DownstreamCompilerSet* getCompilerSet();
     Slang::DownstreamCompiler* getDefaultCompiler(SlangSourceLanguage sourceLanguage);
@@ -134,8 +140,18 @@ class TestContext
 
     Slang::String exeDirectoryPath;
 
-    Slang::Int timeOutInMs = 100 * 1000;
-
+        /// Timeout time for communication over connection.
+        /// NOTE! If the timeout is hit, the connection will be destroyed, and then recreated.
+        /// For tests that compile the stdlib, if that takes this time, the stdlib will be
+        /// repeatedly compiled and each time fail.
+        /// NOTE! This timeout may be altered in the ctor for a specific target, the initializatoin
+        /// value is just the default.
+        ///
+        /// TODO(JS): We could split the stdlib compilation from other actions, and have timeout specific for
+        /// that. To do this we could have a 'compileStdLib' RPC method.
+        ///
+        /// Current default is 2 mins.
+    Slang::Int connectionTimeOutInMs = 2 * 60 * 1000;
 
 protected:
     SlangResult _createJSONRPCConnection(Slang::RefPtr<Slang::JSONRPCConnection>& out);