slang-test: retry failed test at the end. (#5255)

3 files changed, 87 insertions, 29 deletions

diff --git a/tools/slang-test/slang-test-main.cpp b/tools/slang-test/slang-test-main.cpp
index d387c815a..b493b40b0 100644
--- a/tools/slang-test/slang-test-main.cpp
+++ b/tools/slang-test/slang-test-main.cpp
@@ -103,6 +103,14 @@ struct TestOptions
     bool isSynthesized = false;
 };
 
+struct FileTestInfoImpl : public FileTestInfo
+{
+    String testName;
+    String filePath;
+    String outputStem;
+    TestOptions options;
+};
+
 struct TestDetails
 {
     TestDetails() {}
@@ -4042,13 +4050,29 @@ static SlangResult _runTestsOnFile(
             if (_canIgnore(context, testDetails))
             {
                 testResult = TestResult::Ignored;
+                context->getTestReporter()->addResult(testResult);
             }
             else
             {
                 testResult = runTest(context, filePath, outputStem, testName, testDetails.options);
+                if (testResult == TestResult::Fail
+                    && !context->getTestReporter()->m_expectedFailureList.contains(testName))
+                {
+                    RefPtr<FileTestInfoImpl> fileTestInfo = new FileTestInfoImpl();
+                    fileTestInfo->filePath = filePath;
+                    fileTestInfo->testName = testName;
+                    fileTestInfo->outputStem = outputStem;
+                    fileTestInfo->options = testDetails.options;
+
+                    std::lock_guard lock(context->mutexFailedFileTests);
+                    context->failedFileTests.add(fileTestInfo);
+                }
+                else
+                {
+                    context->getTestReporter()->addResult(testResult);
+                }
             }
 
-            context->getTestReporter()->addResult(testResult);
 
             // Could determine if to continue or not here... based on result
         }        
@@ -4615,6 +4639,32 @@ SlangResult innerMain(int argc, char** argv)
             TestReporter::set(nullptr);
         }
 
+        // If we have a couple failed tests, they maybe intermittent failures due to parallel
+        // excution or driver instability. We can try running them again.
+        static constexpr int kFailedTestLimitForRetry = 16;
+        if (context.failedFileTests.getCount() <= kFailedTestLimitForRetry)
+        {
+            printf("Retrying %d failed tests...\n", (int)context.failedFileTests.getCount());
+            for (auto& test : context.failedFileTests)
+            {
+                FileTestInfoImpl* fileTestInfo = static_cast<FileTestInfoImpl*>(test.Ptr());
+                TestReporter::SuiteScope suiteScope(&reporter, "tests");
+                TestReporter::TestScope scope(&reporter, fileTestInfo->testName);
+                auto newResult = runTest(&context, fileTestInfo->filePath, fileTestInfo->outputStem, fileTestInfo->testName, fileTestInfo->options);
+                reporter.addResult(newResult);
+            }
+        }
+        else
+        {
+            // If there are too many failed tests, don't bother retrying.
+            for (auto& test : context.failedFileTests)
+            {
+                FileTestInfoImpl* fileTestInfo = static_cast<FileTestInfoImpl*>(test.Ptr());
+                TestReporter::TestScope scope(&reporter, fileTestInfo->testName);
+                reporter.addResult(TestResult::Fail);
+            }
+        }
+
         reporter.outputSummary();
         return reporter.didAllSucceed() ? SLANG_OK : SLANG_FAIL;
     }
diff --git a/tools/slang-test/test-context.h b/tools/slang-test/test-context.h
index c1bd82c8c..a472248eb 100644
--- a/tools/slang-test/test-context.h
+++ b/tools/slang-test/test-context.h
@@ -85,62 +85,66 @@ struct TestRequirements
     Slang::RenderApiFlags usedRenderApiFlags = 0;                               ///< Used render api flags (some might be implied)
 };
 
+struct FileTestInfo : public Slang::RefObject
+{
+};
+
 class TestContext
 {
-    public:
+public:
 
     typedef Slang::TestToolUtil::InnerMainFunc InnerMainFunc;
 
-        /// Get the slang session
-    SlangSession* getSession() const { return m_session;  }
+    /// Get the slang session
+    SlangSession* getSession() const { return m_session; }
 
     SlangResult init(const char* exePath);
 
-        /// Get the inner main function (from shared library)
+    /// Get the inner main function (from shared library)
     InnerMainFunc getInnerMainFunc(const Slang::String& dirPath, const Slang::String& name);
-        /// Set the function for the shared library
+    /// Set the function for the shared library
     void setInnerMainFunc(const Slang::String& name, InnerMainFunc func);
 
     void setTestRequirements(TestRequirements* req);
 
     TestRequirements* getTestRequirements() const;
 
-        /// If true tests aren't being run just the information on testing is being accumulated
+    /// If true tests aren't being run just the information on testing is being accumulated
     bool isCollectingRequirements() const { return getTestRequirements() != nullptr; }
-        /// If set, then tests are executed
+    /// If set, then tests are executed
     bool isExecuting() const { return getTestRequirements() == nullptr; }
 
-        /// True if a render API filter is enabled
+    /// True if a render API filter is enabled
     bool isRenderApiFilterEnabled() const { return options.enabledApis != Slang::RenderApiFlag::AllOf && options.enabledApis != 0; }
 
-        /// True if a test with the requiredFlags can in principal run (it may not be possible if the API is not available though)
+    /// True if a test with the requiredFlags can in principal run (it may not be possible if the API is not available though)
     bool canRunTestWithRenderApiFlags(Slang::RenderApiFlags requiredFlags);
 
-        /// True if can run unit tests
+    /// True if can run unit tests
     bool canRunUnitTests() const { return options.apiOnly == false; }
 
-        /// Given a spawn type, return the final spawn type.
-        /// In particular we want 'Default' spawn type to vary by the environment (for example running on test server on CI)
+    /// Given a spawn type, return the final spawn type.
+    /// In particular we want 'Default' spawn type to vary by the environment (for example running on test server on CI)
     SpawnType getFinalSpawnType(SpawnType spawnType);
 
     SpawnType getFinalSpawnType();
 
-        /// Get compiler set
+    /// Get compiler set
     Slang::DownstreamCompilerSet* getCompilerSet();
     Slang::IDownstreamCompiler* getDefaultCompiler(SlangSourceLanguage sourceLanguage);
 
     Slang::JSONRPCConnection* getOrCreateJSONRPCConnection();
     void destroyRPCConnection();
 
-        /// Ctor
+    /// Ctor
     TestContext();
-        /// Dtor
+    /// Dtor
     ~TestContext();
 
     Options options;
     TestCategorySet categorySet;
 
-        /// If set then tests are not run, but their requirements are set 
+    /// If set then tests are not run, but their requirements are set 
 
     PassThroughFlags availableBackendFlags = 0;
     Slang::RenderApiFlags availableRenderApiFlags = 0;
@@ -152,17 +156,17 @@ class TestContext
     Slang::String dllDirectoryPath;
     Slang::String exePath;
 
-        /// Timeout time for communication over connection.
-        /// NOTE! If the timeout is hit, the connection will be destroyed, and then recreated.
-        /// For tests that compile the stdlib, if that takes this time, the stdlib will be
-        /// repeatedly compiled and each time fail.
-        /// NOTE! This timeout may be altered in the ctor for a specific target, the initializatoin
-        /// value is just the default.
-        ///
-        /// TODO(JS): We could split the stdlib compilation from other actions, and have timeout specific for
-        /// that. To do this we could have a 'compileStdLib' RPC method.
-        ///
-        /// Current default is 60 seconds.
+    /// Timeout time for communication over connection.
+    /// NOTE! If the timeout is hit, the connection will be destroyed, and then recreated.
+    /// For tests that compile the stdlib, if that takes this time, the stdlib will be
+    /// repeatedly compiled and each time fail.
+    /// NOTE! This timeout may be altered in the ctor for a specific target, the initializatoin
+    /// value is just the default.
+    ///
+    /// TODO(JS): We could split the stdlib compilation from other actions, and have timeout specific for
+    /// that. To do this we could have a 'compileStdLib' RPC method.
+    ///
+    /// Current default is 60 seconds.
     Slang::Int connectionTimeOutInMs = 60 * 1000;
 
     void setThreadIndex(int index);
@@ -175,6 +179,10 @@ class TestContext
     std::mutex mutex;
     Slang::RefPtr<Slang::JSONRPCConnection> m_languageServerConnection;
 
+    
+    std::mutex mutexFailedFileTests;
+    Slang::List<Slang::RefPtr<FileTestInfo>> failedFileTests;
+
     Slang::IFileCheck* getFileCheck() { return m_fileCheck; };
 
 protected:
diff --git a/tools/slang-test/test-reporter.h b/tools/slang-test/test-reporter.h
index 1d2857463..775732745 100644
--- a/tools/slang-test/test-reporter.h
+++ b/tools/slang-test/test-reporter.h
@@ -23,7 +23,7 @@ enum class TestOutputMode
 
 class TestReporter : public ITestReporter
 {
-    public:
+public:
 
     struct TestInfo
     {