Retry when a few unit tests failed. (#6912)

This PR allows the failed unit-tests to be retried at the end as in a single threaded manner.

The purpose of the retry is to increase the stability of CI.

4 files changed, 54 insertions, 18 deletions

diff --git a/tools/slang-test/slang-test-main.cpp b/tools/slang-test/slang-test-main.cpp
index 1fb70ed03..42432e328 100644
--- a/tools/slang-test/slang-test-main.cpp
+++ b/tools/slang-test/slang-test-main.cpp
@@ -4452,7 +4452,7 @@ static SlangResult _runTestsOnFile(TestContext* context, String filePath)
                     fileTestInfo->outputStem = outputStem;
                     fileTestInfo->options = testDetails.options;
 
-                    std::lock_guard lock(context->mutexFailedFileTests);
+                    std::lock_guard lock(context->mutexFailedTests);
                     context->failedFileTests.add(fileTestInfo);
                 }
                 else
@@ -4747,17 +4747,28 @@ static SlangResult runUnitTestModule(
                     TestServerProtocol::ExecuteUnitTestArgs::g_methodName,
                     &args,
                     exeRes);
-                const auto testResult = _asTestResult(ToolReturnCode(exeRes.resultCode));
+                auto testResult = _asTestResult(ToolReturnCode(exeRes.resultCode));
+
+                bool isFailed = (SLANG_FAILED(rpcRes) || testResult == TestResult::Fail);
 
                 // If the test fails, output any output - which might give information about
                 // individual tests that have failed.
-                if (SLANG_FAILED(rpcRes) || testResult == TestResult::Fail)
+                if (isFailed)
                 {
                     String output = getOutput(exeRes);
                     reporter->message(TestMessageType::TestFailure, output.getBuffer());
                 }
 
-                reporter->addResult(testResult);
+                if (isFailed && !context->isRetry &&
+                    !context->getTestReporter()->m_expectedFailureList.contains(test.testName))
+                {
+                    std::lock_guard lock(context->mutexFailedTests);
+                    context->failedUnitTests.add(test.command);
+                }
+                else
+                {
+                    reporter->addResult(testResult);
+                }
             }
         }
         else
@@ -5031,20 +5042,36 @@ SlangResult innerMain(int argc, char** argv)
             TestReporter::SuiteScope suiteScope(&reporter, "unit tests");
             TestReporter::set(&reporter);
 
-            const auto spawnType = context.getFinalSpawnType();
-
-            // Run the unit tests
+            for (bool isRetry : {false, true})
             {
-                TestOptions testOptions;
-                testOptions.categories.add(unitTestCategory);
-                testOptions.categories.add(smokeTestCategory);
-                runUnitTestModule(&context, testOptions, spawnType, "slang-unit-test-tool");
-            }
+                auto spawnType = context.getFinalSpawnType();
 
-            {
-                TestOptions testOptions;
-                testOptions.categories.add(unitTestCategory);
-                runUnitTestModule(&context, testOptions, spawnType, "gfx-unit-test-tool");
+                context.isRetry = false;
+                if (isRetry)
+                {
+                    if (context.failedUnitTests.getCount() == 0)
+                        break;
+
+                    printf("Retrying unit tests...\n");
+                    context.isRetry = true;
+                    context.options.testPrefixes = context.failedUnitTests;
+                    context.failedUnitTests.clear();
+                    spawnType = SpawnType::Default;
+                }
+
+                // Run the unit tests
+                {
+                    TestOptions testOptions;
+                    testOptions.categories.add(unitTestCategory);
+                    testOptions.categories.add(smokeTestCategory);
+                    runUnitTestModule(&context, testOptions, spawnType, "slang-unit-test-tool");
+                }
+
+                {
+                    TestOptions testOptions;
+                    testOptions.categories.add(unitTestCategory);
+                    runUnitTestModule(&context, testOptions, spawnType, "gfx-unit-test-tool");
+                }
             }
 
             TestReporter::set(nullptr);
diff --git a/tools/slang-test/test-context.h b/tools/slang-test/test-context.h
index 5a30e19f7..97562da99 100644
--- a/tools/slang-test/test-context.h
+++ b/tools/slang-test/test-context.h
@@ -180,9 +180,10 @@ public:
     std::mutex mutex;
     Slang::RefPtr<Slang::JSONRPCConnection> m_languageServerConnection;
 
-
-    std::mutex mutexFailedFileTests;
+    bool isRetry;
+    std::mutex mutexFailedTests;
     Slang::List<Slang::RefPtr<FileTestInfo>> failedFileTests;
+    Slang::List<Slang::String> failedUnitTests;
 
     Slang::IFileCheck* getFileCheck() { return m_fileCheck; };
 
diff --git a/tools/slang-test/test-reporter.cpp b/tools/slang-test/test-reporter.cpp
index 88d24d212..1f6c00637 100644
--- a/tools/slang-test/test-reporter.cpp
+++ b/tools/slang-test/test-reporter.cpp
@@ -160,6 +160,11 @@ void TestReporter::addResult(TestResult result)
     m_numCurrentResults++;
 }
 
+TestResult TestReporter::getResult() const
+{
+    return m_currentInfo.testResult;
+}
+
 void TestReporter::addExecutionTime(double time)
 {
     std::lock_guard<std::recursive_mutex> lock(m_mutex);
diff --git a/tools/slang-test/test-reporter.h b/tools/slang-test/test-reporter.h
index bf95803b5..321e65ae2 100644
--- a/tools/slang-test/test-reporter.h
+++ b/tools/slang-test/test-reporter.h
@@ -105,6 +105,9 @@ public:
     /// Returns true if all run tests succeeded
     bool didAllSucceed() const;
 
+    /// Returns a result from the current test
+    TestResult getResult() const;
+
     void outputSummary();
 
     SlangResult init(