From 3cbc500118d27f82c89f401ffb34826264e9cb60 Mon Sep 17 00:00:00 2001 From: Jay Kwak <82421531+jkwak-work@users.noreply.github.com> Date: Wed, 18 Jun 2025 16:38:39 -0700 Subject: Fix retry logic for unit test (#7471) * Fix the ignored unit-tests on retry * Retrigger CI * Add more error messages * Don't use test-server for retry of unit-test to see error messages * Clean up cl.yml Remove 'has-gpu' because it is unused after debug became full-gpu-test. Renamed files to make the meaning more clear: - Renamed expected-failure.txt to expected-failure-via-glsl.txt - Renamed expected-failure-github-runner.txt to expected-failure-no-gpu.txt * Rename cpu-hello-world.slang to avoid name conflict to example We have an example whose executable name is cpu-hello-world.exe. It gets built when you run `cmake --build`, but it gets overwritten by slang-test when it tests `tests/cpu-program/cpu-hello-world.slang`. This PR renames to avoid the name conflict. * Remove debug code --------- Co-authored-by: Yong He --- .github/workflows/ci.yml | 17 ++---------- tests/cpu-program/cpu-hello-world-test.slang | 7 +++++ .../cpu-hello-world-test.slang.expected | 6 ++++ tests/cpu-program/cpu-hello-world.slang | 7 ----- tests/cpu-program/cpu-hello-world.slang.expected | 6 ---- tests/expected-failure-github-runner.txt | 1 - tests/expected-failure-no-gpu.txt | 6 ++++ tests/expected-failure-via-glsl.txt | 7 +++++ tests/expected-failure.txt | 7 ----- tools/slang-test/slang-test-main.cpp | 32 +++++++++++++++++++--- tools/slang-unit-test/unit-test-record-replay.cpp | 10 +++++++ 11 files changed, 67 insertions(+), 39 deletions(-) create mode 100644 tests/cpu-program/cpu-hello-world-test.slang create mode 100644 tests/cpu-program/cpu-hello-world-test.slang.expected delete mode 100644 tests/cpu-program/cpu-hello-world.slang delete mode 100644 tests/cpu-program/cpu-hello-world.slang.expected delete mode 100644 tests/expected-failure-github-runner.txt create mode 100644 tests/expected-failure-no-gpu.txt create mode 100644 tests/expected-failure-via-glsl.txt delete mode 100644 tests/expected-failure.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ac310510..52008e28f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,7 +79,7 @@ jobs: platform: x86_64 test-category: full full-gpu-tests: true - runs-on: [Windows, self-hosted, "GCP-T4"] + runs-on: [Windows, self-hosted, GCP-T4] has-gpu: true server-count: 8 fail-fast: false @@ -231,23 +231,12 @@ jobs: -skip-reference-image-generation \ -show-adapter-info \ -enable-debug-layers false - elif [[ "${{matrix.has-gpu}}" == "true" ]]; then - "$bin_dir/slang-test" \ - -use-test-server \ - -category ${{ matrix.test-category }} \ - -api all-dx12 \ - -expected-failure-list tests/expected-failure-github.txt \ - -expected-failure-list tests/expected-failure-github-runner.txt \ - -skip-reference-image-generation \ - -show-adapter-info \ - -enable-debug-layers false else "$bin_dir/slang-test" \ -use-test-server \ -category ${{ matrix.test-category }} \ - -api all-dx12 \ -expected-failure-list tests/expected-failure-github.txt \ - -expected-failure-list tests/expected-failure-github-runner.txt \ + -expected-failure-list tests/expected-failure-no-gpu.txt \ -skip-reference-image-generation \ -show-adapter-info \ -enable-debug-layers false @@ -276,7 +265,7 @@ jobs: -category ${{ matrix.test-category }} \ -emit-spirv-via-glsl \ -api vk \ - -expected-failure-list tests/expected-failure.txt \ + -expected-failure-list tests/expected-failure-via-glsl.txt \ -skip-reference-image-generation \ -show-adapter-info - name: Run slang-rhi tests diff --git a/tests/cpu-program/cpu-hello-world-test.slang b/tests/cpu-program/cpu-hello-world-test.slang new file mode 100644 index 000000000..a44761e8f --- /dev/null +++ b/tests/cpu-program/cpu-hello-world-test.slang @@ -0,0 +1,7 @@ +//TEST:EXECUTABLE: + +export __extern_cpp int main() +{ + printf("Hello World.\n"); + return 0; +} diff --git a/tests/cpu-program/cpu-hello-world-test.slang.expected b/tests/cpu-program/cpu-hello-world-test.slang.expected new file mode 100644 index 000000000..81531657f --- /dev/null +++ b/tests/cpu-program/cpu-hello-world-test.slang.expected @@ -0,0 +1,6 @@ +result code = 0 +standard error = { +} +standard output = { +Hello World. +} diff --git a/tests/cpu-program/cpu-hello-world.slang b/tests/cpu-program/cpu-hello-world.slang deleted file mode 100644 index a44761e8f..000000000 --- a/tests/cpu-program/cpu-hello-world.slang +++ /dev/null @@ -1,7 +0,0 @@ -//TEST:EXECUTABLE: - -export __extern_cpp int main() -{ - printf("Hello World.\n"); - return 0; -} diff --git a/tests/cpu-program/cpu-hello-world.slang.expected b/tests/cpu-program/cpu-hello-world.slang.expected deleted file mode 100644 index 81531657f..000000000 --- a/tests/cpu-program/cpu-hello-world.slang.expected +++ /dev/null @@ -1,6 +0,0 @@ -result code = 0 -standard error = { -} -standard output = { -Hello World. -} diff --git a/tests/expected-failure-github-runner.txt b/tests/expected-failure-github-runner.txt deleted file mode 100644 index 1da3a9669..000000000 --- a/tests/expected-failure-github-runner.txt +++ /dev/null @@ -1 +0,0 @@ -slang-unit-test-tool/cudaCodeGenBug.internal diff --git a/tests/expected-failure-no-gpu.txt b/tests/expected-failure-no-gpu.txt new file mode 100644 index 000000000..93c5394f7 --- /dev/null +++ b/tests/expected-failure-no-gpu.txt @@ -0,0 +1,6 @@ +slang-unit-test-tool/RecordReplay_triangle.internal +slang-unit-test-tool/RecordReplay_ray_tracing.internal +slang-unit-test-tool/RecordReplay_ray_tracing_pipeline.internal +slang-unit-test-tool/RecordReplay_autodiff_texture.internal +slang-unit-test-tool/RecordReplay_gpu_printing.internal +slang-unit-test-tool/cudaCodeGenBug.internal diff --git a/tests/expected-failure-via-glsl.txt b/tests/expected-failure-via-glsl.txt new file mode 100644 index 000000000..7283c8d97 --- /dev/null +++ b/tests/expected-failure-via-glsl.txt @@ -0,0 +1,7 @@ +tests/language-feature/saturated-cooperation/simple.slang (vk) +tests/language-feature/saturated-cooperation/fuse3.slang (vk) +tests/language-feature/saturated-cooperation/fuse-product.slang (vk) +tests/language-feature/saturated-cooperation/fuse.slang (vk) +tests/bugs/byte-address-buffer-interlocked-add-f32.slang (vk) +tests/ir/loop-unroll-0.slang.1 (vk) +tests/hlsl-intrinsic/texture/float-atomics.slang (vk) diff --git a/tests/expected-failure.txt b/tests/expected-failure.txt deleted file mode 100644 index 7283c8d97..000000000 --- a/tests/expected-failure.txt +++ /dev/null @@ -1,7 +0,0 @@ -tests/language-feature/saturated-cooperation/simple.slang (vk) -tests/language-feature/saturated-cooperation/fuse3.slang (vk) -tests/language-feature/saturated-cooperation/fuse-product.slang (vk) -tests/language-feature/saturated-cooperation/fuse.slang (vk) -tests/bugs/byte-address-buffer-interlocked-add-f32.slang (vk) -tests/ir/loop-unroll-0.slang.1 (vk) -tests/hlsl-intrinsic/texture/float-atomics.slang (vk) diff --git a/tools/slang-test/slang-test-main.cpp b/tools/slang-test/slang-test-main.cpp index f03ae8292..a5f5ee316 100644 --- a/tools/slang-test/slang-test-main.cpp +++ b/tools/slang-test/slang-test-main.cpp @@ -949,21 +949,37 @@ static Result _executeRPC( JSONRPCConnection* rpcConnection = context->getOrCreateJSONRPCConnection(); if (!rpcConnection) { + context->getTestReporter()->messageFormat( + TestMessageType::RunError, + "JSON RPC failure: getOrCreateJSONRPCConnection()"); return SLANG_FAIL; } // Execute if (SLANG_FAILED(rpcConnection->sendCall(method, rttiInfo, args))) { + context->getTestReporter()->messageFormat( + TestMessageType::RunError, + "JSON RPC failure: sendCall()"); + context->destroyRPCConnection(); return SLANG_FAIL; } // Wait for the result - rpcConnection->waitForResult(context->connectionTimeOutInMs); + if (SLANG_FAILED(rpcConnection->waitForResult(context->connectionTimeOutInMs))) + { + context->getTestReporter()->messageFormat( + TestMessageType::RunError, + "JSON RPC failure: waitForResult()"); + } if (!rpcConnection->hasMessage()) { + context->getTestReporter()->messageFormat( + TestMessageType::RunError, + "JSON RPC failure: hasMessage()"); + // We can assume somethings gone wrong. So lets kill the connection and fail. context->destroyRPCConnection(); return SLANG_FAIL; @@ -971,6 +987,10 @@ static Result _executeRPC( if (rpcConnection->getMessageType() != JSONRPCMessageType::Result) { + context->getTestReporter()->messageFormat( + TestMessageType::RunError, + "JSON RPC failure: getMessageType() != JSONRPCMessageType::Result"); + context->destroyRPCConnection(); return SLANG_FAIL; } @@ -979,6 +999,10 @@ static Result _executeRPC( TestServerProtocol::ExecutionResult exeRes; if (SLANG_FAILED(rpcConnection->getMessage(&exeRes))) { + context->getTestReporter()->messageFormat( + TestMessageType::RunError, + "JSON RPC failure: getMessage()"); + context->destroyRPCConnection(); return SLANG_FAIL; } @@ -4780,8 +4804,8 @@ static SlangResult runUnitTestModule( } // If the test failed and it is not an expected failure, add it to the list of - // failed unit tests. - if (isFailed && + // failed unit tests so that we can retry. + if (isFailed && !context->isRetry && !context->getTestReporter()->m_expectedFailureList.contains(test.testName)) { std::lock_guard lock(context->mutexFailedTests); @@ -5142,7 +5166,7 @@ SlangResult innerMain(int argc, char** argv) int main(int argc, char** argv) { - const SlangResult res = innerMain(argc, argv); + SlangResult res = innerMain(argc, argv); slang::shutdown(); Slang::RttiInfo::deallocateAll(); diff --git a/tools/slang-unit-test/unit-test-record-replay.cpp b/tools/slang-unit-test/unit-test-record-replay.cpp index 1dc1a36d2..bc5baeafd 100644 --- a/tools/slang-unit-test/unit-test-record-replay.cpp +++ b/tools/slang-unit-test/unit-test-record-replay.cpp @@ -256,6 +256,16 @@ static SlangResult runExample( hashLines.add(line); } + if (hashLines.getCount() == 0) + { + msgBuilder << "Hash value is not found for '" << exampleName << "'\n"; + msgBuilder << "Process ret code: " << exeRes.resultCode << "\n"; + msgBuilder << "Standard output:\n" << exeRes.standardOutput << "\n"; + msgBuilder << "Standard error:\n" << exeRes.standardError << "\n"; + getTestReporter()->message(TestMessageType::TestFailure, msgBuilder.toString().getBuffer()); + return SLANG_FAIL; + } + res = parseHashes(hashLines, outHashes); if (SLANG_FAILED(res)) { -- cgit v1.2.3