Use test-server on CI (#2034)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Vary what SpawnType is used, if one isn't explicitly set.

* Terminate on linux if exec fails.

* Use a more sophisticated sleeping mechanism.

* Attempt to make CI tests to work on aarch64 debug.
Small fixes.

5 files changed, 62 insertions, 18 deletions

diff --git a/tools/slang-test/options.cpp b/tools/slang-test/options.cpp
index d4dedcf81..48a43d621 100644
--- a/tools/slang-test/options.cpp
+++ b/tools/slang-test/options.cpp
@@ -134,10 +134,6 @@ static bool _isSubCommand(const char* arg)
         {
             optionsOut->defaultSpawnType = SpawnType::UseFullyIsolatedTestServer;
         }
-        else if (strcmp(arg, "-use-test-server") == 0)
-        {
-            optionsOut->defaultSpawnType = SpawnType::UseTestServer;
-        }
         else if (strcmp(arg, "-v") == 0)
         {
             optionsOut->shouldBeVerbose = true;
diff --git a/tools/slang-test/options.h b/tools/slang-test/options.h
index 3b62c903b..cf86bc901 100644
--- a/tools/slang-test/options.h
+++ b/tools/slang-test/options.h
@@ -38,7 +38,8 @@ protected:
 
 enum class SpawnType
 {
-    UseExe,
+    Default,                                    ///< Default - typically uses shared library, on CI may use TestServer    
+    UseExe,                                     ///< Tests using executable (for example slangc)
     UseSharedLibrary,                           ///< Runs testing in process (a crash tan take down the 
     UseTestServer,                              ///< Use the test server to isolate testing
     UseFullyIsolatedTestServer,                 ///< Uses a test server for each test (slow!)
@@ -78,11 +79,9 @@ struct Options
     bool dumpOutputOnFailure = false;
 
     // Set the default spawn type to use
-    // Set to SpawnType::UseProxy, if isolation of test execution is desired.
     // Having tests isolated, slows down testing considerably, so using UseSharedLibrary is the most
     // desirable default usually.
-    SpawnType defaultSpawnType = SpawnType::UseSharedLibrary;
-    //SpawnType defaultSpawnType = SpawnType::UseProxy;
+    SpawnType defaultSpawnType = SpawnType::Default;
     
     // kind of output to generate
     TestOutputMode outputMode = TestOutputMode::Default;
diff --git a/tools/slang-test/slang-test-main.cpp b/tools/slang-test/slang-test-main.cpp
index ac8ec0e10..e734e1c4c 100644
--- a/tools/slang-test/slang-test-main.cpp
+++ b/tools/slang-test/slang-test-main.cpp
@@ -674,7 +674,7 @@ static Result _executeRPC(TestContext* context, SpawnType spawnType, const Unown
     SLANG_RETURN_ON_FAIL(rpcConnection->sendCall(method, rttiInfo, args));
 
     // Wait for the result
-    rpcConnection->waitForResult(context->timeOutInMs);
+    rpcConnection->waitForResult(context->connectionTimeOutInMs);
 
     if (!rpcConnection->hasMessage())
     {
@@ -1057,14 +1057,17 @@ ToolReturnCode spawnAndWait(TestContext* context, const String& testPath, SpawnT
 
     const auto& options = context->options;
 
+    const auto finalSpawnType = context->getFinalSpawnType(spawnType);
+
     SlangResult spawnResult = SLANG_FAIL;
-    switch (spawnType)
+    switch (finalSpawnType)
     {
         case SpawnType::UseExe:
         {
             spawnResult = spawnAndWaitExe(context, testPath, cmdLine, outExeRes);
             break;
         }
+        case SpawnType::Default:
         case SpawnType::UseSharedLibrary:
         {
             spawnResult = spawnAndWaitSharedLibrary(context, testPath, cmdLine, outExeRes);
@@ -1073,7 +1076,7 @@ ToolReturnCode spawnAndWait(TestContext* context, const String& testPath, SpawnT
         case SpawnType::UseFullyIsolatedTestServer:
         case SpawnType::UseTestServer:
         {
-            spawnResult = spawnAndWaitTestServer(context, spawnType, testPath, cmdLine, outExeRes);
+            spawnResult = spawnAndWaitTestServer(context, finalSpawnType, testPath, cmdLine, outExeRes);
             break;
         }
         default: break;
@@ -3601,7 +3604,8 @@ SlangResult innerMain(int argc, char** argv)
     {
         SlangSession* session = context.getSession();
 
-        StdWriters::getOut().print("Supported backends:");
+        auto out = StdWriters::getOut();
+        out.print("Supported backends:");
 
         for (int i = 0; i < SLANG_PASS_THROUGH_COUNT_OF; ++i)
         {
@@ -3624,11 +3628,11 @@ SlangResult innerMain(int argc, char** argv)
                 SLANG_ASSERT(passThroughCategories[i] == nullptr);
                 passThroughCategories[i] = categorySet.add(buf.getBuffer() + 1, fullTestCategory);
 
-                StdWriters::getOut().write(buf.getBuffer(), buf.getLength());
+                out.write(buf.getBuffer(), buf.getLength());
             }
         }
 
-        StdWriters::getOut().print("\n");
+        out.print("\n");
     }
 
     // Working out what renderApis is worked on on demand through
@@ -3727,18 +3731,20 @@ SlangResult innerMain(int argc, char** argv)
             TestReporter::SuiteScope suiteScope(&reporter, "unit tests");
             TestReporter::set(&reporter);
 
+            const auto spawnType = context.getFinalSpawnType();
+
             // Run the unit tests
             {
                 TestOptions testOptions;
                 testOptions.categories.add(unitTestCategory);
                 testOptions.categories.add(smokeTestCategory);
-                runUnitTestModule(&context, testOptions, context.options.defaultSpawnType, "slang-unit-test-tool");
+                runUnitTestModule(&context, testOptions, spawnType, "slang-unit-test-tool");
             }
 
             {
                 TestOptions testOptions;
                 testOptions.categories.add(unitTestCategory);
-                runUnitTestModule(&context, testOptions, context.options.defaultSpawnType, "gfx-unit-test-tool");
+                runUnitTestModule(&context, testOptions, spawnType, "gfx-unit-test-tool");
             }
              
             TestReporter::set(nullptr);
diff --git a/tools/slang-test/test-context.cpp b/tools/slang-test/test-context.cpp
index eeb23b695..39dd52313 100644
--- a/tools/slang-test/test-context.cpp
+++ b/tools/slang-test/test-context.cpp
@@ -15,6 +15,12 @@ using namespace Slang;
 TestContext::TestContext() 
 {
     m_session = nullptr;
+
+    /// if we are testing on arm, debug, we may want to increase the connection timeout
+#if (SLANG_PROCESSOR_ARM || SLANG_PROCESSOR_ARM_64) && defined(_DEBUG)
+    // 5 mins(!). This seems to be the order of time needed for timeout on a CI ARM test system on debug
+    connectionTimeOutInMs = 1000 * 60 * 5;
+#endif
 }
 
 Result TestContext::init(const char* exePath)
@@ -170,4 +176,25 @@ bool TestContext::canRunTestWithRenderApiFlags(Slang::RenderApiFlags requiredFla
     return (requiredFlags & options.enabledApis) == requiredFlags;
 }
 
+SpawnType TestContext::getFinalSpawnType(SpawnType spawnType)
+{
+    if (spawnType == SpawnType::Default)
+    {
+        if (options.outputMode == TestOutputMode::Default)
+        {
+            return SpawnType::UseSharedLibrary;
+        }
+        else
+        {
+            return SpawnType::UseTestServer;
+        }
+    }
+
+    // Just return whatever spawnType was passed in
+    return spawnType;
+}
 
+SpawnType TestContext::getFinalSpawnType()
+{
+    return getFinalSpawnType(options.defaultSpawnType);
+}
diff --git a/tools/slang-test/test-context.h b/tools/slang-test/test-context.h
index 203005943..03b74217f 100644
--- a/tools/slang-test/test-context.h
+++ b/tools/slang-test/test-context.h
@@ -107,6 +107,12 @@ class TestContext
         /// True if can run unit tests
     bool canRunUnitTests() const { return options.apiOnly == false; }
 
+        /// Given a spawn type, return the final spawn type.
+        /// In particular we want 'Default' spawn type to vary by the environment (for example running on test server on CI)
+    SpawnType getFinalSpawnType(SpawnType spawnType);
+
+    SpawnType getFinalSpawnType();
+
         /// Get compiler set
     Slang::DownstreamCompilerSet* getCompilerSet();
     Slang::DownstreamCompiler* getDefaultCompiler(SlangSourceLanguage sourceLanguage);
@@ -134,8 +140,18 @@ class TestContext
 
     Slang::String exeDirectoryPath;
 
-    Slang::Int timeOutInMs = 100 * 1000;
-
+        /// Timeout time for communication over connection.
+        /// NOTE! If the timeout is hit, the connection will be destroyed, and then recreated.
+        /// For tests that compile the stdlib, if that takes this time, the stdlib will be
+        /// repeatedly compiled and each time fail.
+        /// NOTE! This timeout may be altered in the ctor for a specific target, the initializatoin
+        /// value is just the default.
+        ///
+        /// TODO(JS): We could split the stdlib compilation from other actions, and have timeout specific for
+        /// that. To do this we could have a 'compileStdLib' RPC method.
+        ///
+        /// Current default is 2 mins.
+    Slang::Int connectionTimeOutInMs = 2 * 60 * 1000;
 
 protected:
     SlangResult _createJSONRPCConnection(Slang::RefPtr<Slang::JSONRPCConnection>& out);