Break down record replay to individual tests to avoid timeout (#7340)

* Break down RecordReply to individual tests to avoid timeout

In Debug build, RecordReplay unit-test was timing out.
It was running six tests all in one unit-test, but this commit breaks it
down to individual test so that each unit test can be done within the
timeout limit.

This issue has seen only in Debug build but it has been unnoticed
because even when the test failed with test-server, it was still passing
on its retry because the time-out applies only when using test-server.

* Reduce the retry from 2 times to 1 time

* Remove RecordReplay from expected failure

5 files changed, 94 insertions, 32 deletions

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 142c8db51..0d9d0af87 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -197,7 +197,6 @@ jobs:
               -category ${{ matrix.test-category }} \
               -api all-cpu \
               -expected-failure-list tests/expected-failure-github.txt \
-              -expected-failure-list tests/expected-failure-record-replay-tests.txt \
               -skip-reference-image-generation \
               -show-adapter-info \
               -enable-debug-layers false
@@ -207,7 +206,6 @@ jobs:
               -category ${{ matrix.test-category }} \
               -api all-dx12 \
               -expected-failure-list tests/expected-failure-github.txt \
-              -expected-failure-list tests/expected-failure-record-replay-tests.txt \
               -expected-failure-list tests/expected-failure-github-runner.txt \
               -skip-reference-image-generation \
               -show-adapter-info  \
@@ -218,7 +216,6 @@ jobs:
               -category ${{ matrix.test-category }} \
               -api all-dx12 \
               -expected-failure-list tests/expected-failure-github.txt \
-              -expected-failure-list tests/expected-failure-record-replay-tests.txt \
               -expected-failure-list tests/expected-failure-github-runner.txt \
               -skip-reference-image-generation \
               -show-adapter-info \
diff --git a/source/slang-record-replay/record/record-manager.cpp b/source/slang-record-replay/record/record-manager.cpp
index 3be120c43..07f1d4e72 100644
--- a/source/slang-record-replay/record/record-manager.cpp
+++ b/source/slang-record-replay/record/record-manager.cpp
@@ -1,8 +1,10 @@
 #include "record-manager.h"
 
 #include "../../core/slang-io.h"
+#include "../../core/slang-platform.h"
 #include "../util/record-utility.h"
 
+#include <cstdlib>
 #include <sstream>
 #include <thread>
 
@@ -14,7 +16,19 @@ RecordManager::RecordManager(uint64_t globalSessionHandle)
     std::stringstream ss;
     ss << "gs-" << globalSessionHandle << "-t-" << std::this_thread::get_id() << ".cap";
 
-    m_recordFileDirectory = Slang::Path::combine(m_recordFileDirectory, "slang-record");
+    // Check for custom record directory from environment variable
+    Slang::StringBuilder customRecordDirBuilder;
+    if (SLANG_SUCCEEDED(Slang::PlatformUtil::getEnvironmentVariable(
+            Slang::UnownedStringSlice::fromLiteral("SLANG_RECORD_DIRECTORY"),
+            customRecordDirBuilder)))
+    {
+        m_recordFileDirectory = customRecordDirBuilder.toString();
+    }
+    else
+    {
+        m_recordFileDirectory = Slang::Path::combine(m_recordFileDirectory, "slang-record");
+    }
+
     if (!Slang::File::exists(m_recordFileDirectory))
     {
         if (!Slang::Path::createDirectoryRecursive(m_recordFileDirectory))
diff --git a/tests/expected-failure-record-replay-tests.txt b/tests/expected-failure-record-replay-tests.txt
deleted file mode 100644
index 9b2f264f5..000000000
--- a/tests/expected-failure-record-replay-tests.txt
+++ /dev/null
@@ -1 +0,0 @@
-slang-unit-test-tool/RecordReplay.internal
diff --git a/tools/slang-test/slang-test-main.cpp b/tools/slang-test/slang-test-main.cpp
index f0a140549..7e2956c3f 100644
--- a/tools/slang-test/slang-test-main.cpp
+++ b/tools/slang-test/slang-test-main.cpp
@@ -5063,8 +5063,7 @@ SlangResult innerMain(int argc, char** argv)
             TestReporter::SuiteScope suiteScope(&reporter, "unit tests");
             TestReporter::set(&reporter);
 
-            // Try the unit tests up to 3 times
-            for (bool isRetry : {false, true, true})
+            for (bool isRetry : {false, true})
             {
                 auto spawnType = context.getFinalSpawnType();
                 context.isRetry = isRetry;
diff --git a/tools/slang-unit-test/unit-test-record-replay.cpp b/tools/slang-unit-test/unit-test-record-replay.cpp
index efa01a67c..cab4bb5b8 100644
--- a/tools/slang-unit-test/unit-test-record-replay.cpp
+++ b/tools/slang-unit-test/unit-test-record-replay.cpp
@@ -146,7 +146,7 @@ static bool disableLogInReplayer()
     return retCode == 0;
 }
 
-static void findRecordFileName(List<String>* fileNames)
+static void findRecordFileName(List<String>* fileNames, const String& recordDir)
 {
     struct Visitor : Path::Visitor
     {
@@ -165,7 +165,7 @@ static void findRecordFileName(List<String>* fileNames)
     };
 
     Visitor visitor(fileNames);
-    Path::find("slang-record", "*.cap", &visitor);
+    Path::find(recordDir.getBuffer(), "*.cap", &visitor);
 }
 
 static SlangResult launchProcessAndReadStdout(
@@ -216,6 +216,7 @@ static SlangResult launchProcessAndReadStdout(
 static SlangResult runExample(
     UnitTestContext* context,
     const char* exampleName,
+    const String& recordDir,
     List<entryHashInfo>& outHashes)
 {
     SlangResult finalRes = SLANG_OK;
@@ -228,6 +229,8 @@ static SlangResult runExample(
     StringBuilder msgBuilder;
     SlangResult res = SLANG_OK;
 
+    // Set unique record directory for this test
+    writeEnvironmentVariable("SLANG_RECORD_DIRECTORY", recordDir.getBuffer());
     enableRecordLayer();
     res = launchProcessAndReadStdout(context, optArgs, exampleName, process, exeRes);
     disableRecordLayer();
@@ -264,10 +267,13 @@ static SlangResult runExample(
     return SLANG_OK;
 }
 
-static SlangResult replayExample(UnitTestContext* context, List<entryHashInfo>& outHashes)
+static SlangResult replayExample(
+    UnitTestContext* context,
+    const String& recordDir,
+    List<entryHashInfo>& outHashes)
 {
     List<String> fileNames;
-    findRecordFileName(&fileNames);
+    findRecordFileName(&fileNames, recordDir);
     if (fileNames.getCount() == 0)
     {
         getTestReporter()->message(TestMessageType::TestFailure, "No record files found\n");
@@ -275,7 +281,7 @@ static SlangResult replayExample(UnitTestContext* context, List<entryHashInfo>&
     }
 
     List<String> optArgs;
-    String recordFileName = Path::combine("slang-record", fileNames[0]);
+    String recordFileName = Path::combine(recordDir, fileNames[0]);
     optArgs.add(recordFileName.getBuffer());
 
     RefPtr<Process> process;
@@ -377,14 +383,14 @@ static SlangResult resultCompare(
     return SLANG_OK;
 }
 
-static SlangResult cleanupRecordFiles()
+static SlangResult cleanupRecordFiles(const String& recordDir)
 {
-    SlangResult res = Path::removeNonEmpty("slang-record");
+    SlangResult res = Path::removeNonEmpty(recordDir.getBuffer());
     if (SLANG_FAILED(res))
     {
-        getTestReporter()->message(
-            TestMessageType::TestFailure,
-            "Failed to remove 'slang-record' directory\n");
+        StringBuilder msgBuilder;
+        msgBuilder << "Failed to remove '" << recordDir << "' directory\n";
+        getTestReporter()->message(TestMessageType::TestFailure, msgBuilder.toString().getBuffer());
     }
 
     return res;
@@ -392,29 +398,34 @@ static SlangResult cleanupRecordFiles()
 
 static SlangResult runTest(UnitTestContext* context, const char* testName)
 {
+    // Create unique directory for this test to avoid conflicts
+    StringBuilder recordDirBuilder;
+    recordDirBuilder << "slang-record-" << testName;
+    String recordDir = recordDirBuilder.toString();
+
     List<entryHashInfo> expectHashes;
     List<entryHashInfo> resultHashes;
     SlangResult res = SLANG_OK;
-    if ((res = runExample(context, testName, expectHashes)) != SLANG_OK)
-    {
-        goto error;
-    }
-
-    if ((res = replayExample(context, resultHashes)) != SLANG_OK)
-    {
-        goto error;
-    }
 
-    if ((res = resultCompare(expectHashes, resultHashes)) != SLANG_OK)
+    // Run the example to generate recording
+    res = runExample(context, testName, recordDir, expectHashes);
+    if (SLANG_SUCCEEDED(res))
     {
-        goto error;
+        // Replay the recording
+        res = replayExample(context, recordDir, resultHashes);
+        if (SLANG_SUCCEEDED(res))
+        {
+            // Compare results
+            res = resultCompare(expectHashes, resultHashes);
+        }
     }
 
-error:
-    cleanupRecordFiles();
+    // Always cleanup, regardless of success or failure
+    cleanupRecordFiles(recordDir);
     return res;
 }
 
+#if 0
 static SlangResult runTests(UnitTestContext* context)
 {
     const char* testBinaryNames[] = {
@@ -445,14 +456,56 @@ static SlangResult runTests(UnitTestContext* context)
 
     return finalRes;
 }
+#endif
 
 // Those examples all depend on the Vulkan, so we only run them on non-Apple platforms.
 // In the future, we may be able to modify the examples further to remove all the render APIs
 // such that it can be ran on Apple platforms.
 #if !(SLANG_APPLE_FAMILY)
-SLANG_UNIT_TEST(RecordReplay)
+
+SLANG_UNIT_TEST(RecordReplay_cpu_hello_world)
+{
+    SLANG_CHECK(SLANG_SUCCEEDED(runTest(unitTestContext, "cpu-hello-world")));
+}
+
+SLANG_UNIT_TEST(RecordReplay_triangle)
+{
+    SLANG_CHECK(SLANG_SUCCEEDED(runTest(unitTestContext, "triangle")));
+}
+
+SLANG_UNIT_TEST(RecordReplay_ray_tracing)
+{
+    SLANG_CHECK(SLANG_SUCCEEDED(runTest(unitTestContext, "ray-tracing")));
+}
+
+SLANG_UNIT_TEST(RecordReplay_ray_tracing_pipeline)
+{
+    SLANG_CHECK(SLANG_SUCCEEDED(runTest(unitTestContext, "ray-tracing-pipeline")));
+}
+
+SLANG_UNIT_TEST(RecordReplay_autodiff_texture)
+{
+    SLANG_CHECK(SLANG_SUCCEEDED(runTest(unitTestContext, "autodiff-texture")));
+}
+
+SLANG_UNIT_TEST(RecordReplay_gpu_printing)
+{
+    SLANG_CHECK(SLANG_SUCCEEDED(runTest(unitTestContext, "gpu-printing")));
+}
+
+#if 0
+// These examples requires reflection API to replay, we have to disable
+// it for now. "model-viewer",
+
+SLANG_UNIT_TEST(RecordReplay_shader_object)
 {
-    SLANG_CHECK(SLANG_SUCCEEDED(runTests(unitTestContext)));
+    SLANG_CHECK(SLANG_SUCCEEDED(runTest(unitTestContext, "shader-object")));
 }
 
+SLANG_UNIT_TEST(RecordReplay_model_viewer)
+{
+    SLANG_CHECK(SLANG_SUCCEEDED(runTest(unitTestContext, "model-viewer")));
+}
+#endif
+
 #endif