Add deterministic shuffling of tests in directory (#8622)

Fixes #8621

Add command line options for enable shuffling as well as providing a
custom seed. Use Mersenne-Twister engine for a deterministic shuffle.

3 files changed, 36 insertions, 3 deletions

diff --git a/tools/slang-test/options.cpp b/tools/slang-test/options.cpp
index 1333407d1..051e9bcd2 100644
--- a/tools/slang-test/options.cpp
+++ b/tools/slang-test/options.cpp
@@ -90,6 +90,8 @@ static bool _isSubCommand(const char* arg)
         "  -use-test-server               Run tests using test server\n"
         "  -use-fully-isolated-test-server  Run each test in isolated server\n"
         "  -capability <name>             Compile with the given capability\n"
+        "  -shuffle-tests                 Shuffle tests in directories\n"
+        "  -shuffle-seed <seed>           Set shuffle seed (default: 1)\n"
 
         // Recent Windows runtime versions started opening a dialog popup window when
         // `abort()` is called, which breaks the CI workflow and some scripts that
@@ -272,6 +274,24 @@ static bool _isSubCommand(const char* arg)
         {
             optionsOut->generateHLSLBaselines = true;
         }
+        else if (strcmp(arg, "-shuffle-tests") == 0)
+        {
+            optionsOut->shuffleTests = true;
+        }
+        else if (strcmp(arg, "-shuffle-seed") == 0)
+        {
+            if (argCursor == argEnd)
+            {
+                stdError.print("error: expected operand for '%s'\n", arg);
+                showHelp(stdError);
+                return SLANG_FAIL;
+            }
+            optionsOut->shuffleSeed = stringToInt(*argCursor++);
+            if (optionsOut->shuffleSeed <= 0)
+            {
+                optionsOut->shuffleSeed = 1;
+            }
+        }
         else if (strcmp(arg, "-release") == 0)
         {
             // Assumed to be handle by .bat file that called us
diff --git a/tools/slang-test/options.h b/tools/slang-test/options.h
index 17ff666c3..e7ffe0f97 100644
--- a/tools/slang-test/options.h
+++ b/tools/slang-test/options.h
@@ -156,6 +156,12 @@ struct Options
 
     /// Display help message
     static void showHelp(Slang::WriterHelper stdOut);
+
+    /// Whether to shuffle tests
+    bool shuffleTests = false;
+
+    /// Seed for shuffling deterministically
+    uint32_t shuffleSeed = 1;
 };
 
 #endif // OPTIONS_H_INCLUDED
diff --git a/tools/slang-test/slang-test-main.cpp b/tools/slang-test/slang-test-main.cpp
index 04283e170..db0102304 100644
--- a/tools/slang-test/slang-test-main.cpp
+++ b/tools/slang-test/slang-test-main.cpp
@@ -37,6 +37,7 @@
 #include "stb_image.h"
 
 #include <math.h>
+#include <random>
 #include <stdarg.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -4870,12 +4871,18 @@ void runTestsInDirectory(TestContext* context)
     // NTFS on Windows stores files in sorted order but not on Linux/Macos.
     // Because of that, the testing on Linux/Macos were randomly failing, which
     // is a good thing because it reveals problems. But it is useless
-    // if we cannot reproduce the failures deterministrically.
+    // if we cannot reproduce the failures deterministically.
     // https://github.com/shader-slang/slang/issues/7388
-    //
-    // TODO: We need a way to shuffle the list in a deterministic manner.
+
     files.sort();
 
+    // If asked, shuffle the list using seed for deterministic behavior.
+    if (context->options.shuffleTests)
+    {
+        std::mt19937 mt(context->options.shuffleSeed);
+        std::shuffle(files.begin(), files.end(), mt);
+    }
+
     auto processFile = [&](String file)
     {
         if (shouldRunTest(context, file))