Fix retry logic for unit test (#7471)

* Fix the ignored unit-tests on retry

* Retrigger CI

* Add more error messages

* Don't use test-server for retry of unit-test to see error messages

* Clean up cl.yml

Remove 'has-gpu' because it is unused after debug became full-gpu-test.
Renamed files to make the meaning more clear:
- Renamed expected-failure.txt to expected-failure-via-glsl.txt
- Renamed expected-failure-github-runner.txt to expected-failure-no-gpu.txt

* Rename cpu-hello-world.slang to avoid name conflict to example

We have an example whose executable name is cpu-hello-world.exe.
It gets built when you run `cmake --build`, but it gets overwritten by
slang-test when it tests `tests/cpu-program/cpu-hello-world.slang`.

This PR renames to avoid the name conflict.

* Remove debug code

---------

Co-authored-by: Yong He <yonghe@outlook.com>

8 files changed, 47 insertions, 19 deletions

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5ac310510..52008e28f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -79,7 +79,7 @@ jobs:
             platform: x86_64
             test-category: full
             full-gpu-tests: true
-            runs-on: [Windows, self-hosted, "GCP-T4"]
+            runs-on: [Windows, self-hosted, GCP-T4]
             has-gpu: true
             server-count: 8
       fail-fast: false
@@ -231,23 +231,12 @@ jobs:
               -skip-reference-image-generation \
               -show-adapter-info \
               -enable-debug-layers false
-          elif [[ "${{matrix.has-gpu}}" == "true" ]]; then
-            "$bin_dir/slang-test" \
-              -use-test-server \
-              -category ${{ matrix.test-category }} \
-              -api all-dx12 \
-              -expected-failure-list tests/expected-failure-github.txt \
-              -expected-failure-list tests/expected-failure-github-runner.txt \
-              -skip-reference-image-generation \
-              -show-adapter-info  \
-              -enable-debug-layers false
           else
             "$bin_dir/slang-test" \
               -use-test-server \
               -category ${{ matrix.test-category }} \
-              -api all-dx12 \
               -expected-failure-list tests/expected-failure-github.txt \
-              -expected-failure-list tests/expected-failure-github-runner.txt \
+              -expected-failure-list tests/expected-failure-no-gpu.txt \
               -skip-reference-image-generation \
               -show-adapter-info \
               -enable-debug-layers false
@@ -276,7 +265,7 @@ jobs:
             -category ${{ matrix.test-category }} \
             -emit-spirv-via-glsl \
             -api vk \
-            -expected-failure-list tests/expected-failure.txt \
+            -expected-failure-list tests/expected-failure-via-glsl.txt \
             -skip-reference-image-generation \
             -show-adapter-info
       - name: Run slang-rhi tests
diff --git a/tests/cpu-program/cpu-hello-world.slang b/tests/cpu-program/cpu-hello-world-test.slang
index a44761e8f..a44761e8f 100644
--- a/tests/cpu-program/cpu-hello-world.slang
+++ b/tests/cpu-program/cpu-hello-world-test.slang
diff --git a/tests/cpu-program/cpu-hello-world.slang.expected b/tests/cpu-program/cpu-hello-world-test.slang.expected
index 81531657f..81531657f 100644
--- a/tests/cpu-program/cpu-hello-world.slang.expected
+++ b/tests/cpu-program/cpu-hello-world-test.slang.expected
diff --git a/tests/expected-failure-github-runner.txt b/tests/expected-failure-github-runner.txt
deleted file mode 100644
index 1da3a9669..000000000
--- a/tests/expected-failure-github-runner.txt
+++ /dev/null
@@ -1 +0,0 @@
-slang-unit-test-tool/cudaCodeGenBug.internal
diff --git a/tests/expected-failure-no-gpu.txt b/tests/expected-failure-no-gpu.txt
new file mode 100644
index 000000000..93c5394f7
--- /dev/null
+++ b/tests/expected-failure-no-gpu.txt
@@ -0,0 +1,6 @@
+slang-unit-test-tool/RecordReplay_triangle.internal
+slang-unit-test-tool/RecordReplay_ray_tracing.internal
+slang-unit-test-tool/RecordReplay_ray_tracing_pipeline.internal
+slang-unit-test-tool/RecordReplay_autodiff_texture.internal
+slang-unit-test-tool/RecordReplay_gpu_printing.internal
+slang-unit-test-tool/cudaCodeGenBug.internal
diff --git a/tests/expected-failure.txt b/tests/expected-failure-via-glsl.txt
index 7283c8d97..7283c8d97 100644
--- a/tests/expected-failure.txt
+++ b/tests/expected-failure-via-glsl.txt
diff --git a/tools/slang-test/slang-test-main.cpp b/tools/slang-test/slang-test-main.cpp
index f03ae8292..a5f5ee316 100644
--- a/tools/slang-test/slang-test-main.cpp
+++ b/tools/slang-test/slang-test-main.cpp
@@ -949,21 +949,37 @@ static Result _executeRPC(
     JSONRPCConnection* rpcConnection = context->getOrCreateJSONRPCConnection();
     if (!rpcConnection)
     {
+        context->getTestReporter()->messageFormat(
+            TestMessageType::RunError,
+            "JSON RPC failure: getOrCreateJSONRPCConnection()");
         return SLANG_FAIL;
     }
 
     // Execute
     if (SLANG_FAILED(rpcConnection->sendCall(method, rttiInfo, args)))
     {
+        context->getTestReporter()->messageFormat(
+            TestMessageType::RunError,
+            "JSON RPC failure: sendCall()");
+
         context->destroyRPCConnection();
         return SLANG_FAIL;
     }
 
     // Wait for the result
-    rpcConnection->waitForResult(context->connectionTimeOutInMs);
+    if (SLANG_FAILED(rpcConnection->waitForResult(context->connectionTimeOutInMs)))
+    {
+        context->getTestReporter()->messageFormat(
+            TestMessageType::RunError,
+            "JSON RPC failure: waitForResult()");
+    }
 
     if (!rpcConnection->hasMessage())
     {
+        context->getTestReporter()->messageFormat(
+            TestMessageType::RunError,
+            "JSON RPC failure: hasMessage()");
+
         // We can assume somethings gone wrong. So lets kill the connection and fail.
         context->destroyRPCConnection();
         return SLANG_FAIL;
@@ -971,6 +987,10 @@ static Result _executeRPC(
 
     if (rpcConnection->getMessageType() != JSONRPCMessageType::Result)
     {
+        context->getTestReporter()->messageFormat(
+            TestMessageType::RunError,
+            "JSON RPC failure: getMessageType() != JSONRPCMessageType::Result");
+
         context->destroyRPCConnection();
         return SLANG_FAIL;
     }
@@ -979,6 +999,10 @@ static Result _executeRPC(
     TestServerProtocol::ExecutionResult exeRes;
     if (SLANG_FAILED(rpcConnection->getMessage(&exeRes)))
     {
+        context->getTestReporter()->messageFormat(
+            TestMessageType::RunError,
+            "JSON RPC failure: getMessage()");
+
         context->destroyRPCConnection();
         return SLANG_FAIL;
     }
@@ -4780,8 +4804,8 @@ static SlangResult runUnitTestModule(
                 }
 
                 // If the test failed and it is not an expected failure, add it to the list of
-                // failed unit tests.
-                if (isFailed &&
+                // failed unit tests so that we can retry.
+                if (isFailed && !context->isRetry &&
                     !context->getTestReporter()->m_expectedFailureList.contains(test.testName))
                 {
                     std::lock_guard lock(context->mutexFailedTests);
@@ -5142,7 +5166,7 @@ SlangResult innerMain(int argc, char** argv)
 
 int main(int argc, char** argv)
 {
-    const SlangResult res = innerMain(argc, argv);
+    SlangResult res = innerMain(argc, argv);
     slang::shutdown();
     Slang::RttiInfo::deallocateAll();
 
diff --git a/tools/slang-unit-test/unit-test-record-replay.cpp b/tools/slang-unit-test/unit-test-record-replay.cpp
index 1dc1a36d2..bc5baeafd 100644
--- a/tools/slang-unit-test/unit-test-record-replay.cpp
+++ b/tools/slang-unit-test/unit-test-record-replay.cpp
@@ -256,6 +256,16 @@ static SlangResult runExample(
         hashLines.add(line);
     }
 
+    if (hashLines.getCount() == 0)
+    {
+        msgBuilder << "Hash value is not found for '" << exampleName << "'\n";
+        msgBuilder << "Process ret code: " << exeRes.resultCode << "\n";
+        msgBuilder << "Standard output:\n" << exeRes.standardOutput << "\n";
+        msgBuilder << "Standard error:\n" << exeRes.standardError << "\n";
+        getTestReporter()->message(TestMessageType::TestFailure, msgBuilder.toString().getBuffer());
+        return SLANG_FAIL;
+    }
+
     res = parseHashes(hashLines, outHashes);
     if (SLANG_FAILED(res))
     {