From b9dc21d362f65f22bc707bede733a9537b80460a Mon Sep 17 00:00:00 2001 From: Jay Kwak <82421531+jkwak-work@users.noreply.github.com> Date: Wed, 4 Jun 2025 10:35:52 -0700 Subject: Break down record replay to individual tests to avoid timeout (#7340) * Break down RecordReply to individual tests to avoid timeout In Debug build, RecordReplay unit-test was timing out. It was running six tests all in one unit-test, but this commit breaks it down to individual test so that each unit test can be done within the timeout limit. This issue has seen only in Debug build but it has been unnoticed because even when the test failed with test-server, it was still passing on its retry because the time-out applies only when using test-server. * Reduce the retry from 2 times to 1 time * Remove RecordReplay from expected failure --- tools/slang-test/slang-test-main.cpp | 3 +- tools/slang-unit-test/unit-test-record-replay.cpp | 103 ++++++++++++++++------ 2 files changed, 79 insertions(+), 27 deletions(-) (limited to 'tools') diff --git a/tools/slang-test/slang-test-main.cpp b/tools/slang-test/slang-test-main.cpp index f0a140549..7e2956c3f 100644 --- a/tools/slang-test/slang-test-main.cpp +++ b/tools/slang-test/slang-test-main.cpp @@ -5063,8 +5063,7 @@ SlangResult innerMain(int argc, char** argv) TestReporter::SuiteScope suiteScope(&reporter, "unit tests"); TestReporter::set(&reporter); - // Try the unit tests up to 3 times - for (bool isRetry : {false, true, true}) + for (bool isRetry : {false, true}) { auto spawnType = context.getFinalSpawnType(); context.isRetry = isRetry; diff --git a/tools/slang-unit-test/unit-test-record-replay.cpp b/tools/slang-unit-test/unit-test-record-replay.cpp index efa01a67c..cab4bb5b8 100644 --- a/tools/slang-unit-test/unit-test-record-replay.cpp +++ b/tools/slang-unit-test/unit-test-record-replay.cpp @@ -146,7 +146,7 @@ static bool disableLogInReplayer() return retCode == 0; } -static void findRecordFileName(List* fileNames) +static void findRecordFileName(List* fileNames, const String& recordDir) { struct Visitor : Path::Visitor { @@ -165,7 +165,7 @@ static void findRecordFileName(List* fileNames) }; Visitor visitor(fileNames); - Path::find("slang-record", "*.cap", &visitor); + Path::find(recordDir.getBuffer(), "*.cap", &visitor); } static SlangResult launchProcessAndReadStdout( @@ -216,6 +216,7 @@ static SlangResult launchProcessAndReadStdout( static SlangResult runExample( UnitTestContext* context, const char* exampleName, + const String& recordDir, List& outHashes) { SlangResult finalRes = SLANG_OK; @@ -228,6 +229,8 @@ static SlangResult runExample( StringBuilder msgBuilder; SlangResult res = SLANG_OK; + // Set unique record directory for this test + writeEnvironmentVariable("SLANG_RECORD_DIRECTORY", recordDir.getBuffer()); enableRecordLayer(); res = launchProcessAndReadStdout(context, optArgs, exampleName, process, exeRes); disableRecordLayer(); @@ -264,10 +267,13 @@ static SlangResult runExample( return SLANG_OK; } -static SlangResult replayExample(UnitTestContext* context, List& outHashes) +static SlangResult replayExample( + UnitTestContext* context, + const String& recordDir, + List& outHashes) { List fileNames; - findRecordFileName(&fileNames); + findRecordFileName(&fileNames, recordDir); if (fileNames.getCount() == 0) { getTestReporter()->message(TestMessageType::TestFailure, "No record files found\n"); @@ -275,7 +281,7 @@ static SlangResult replayExample(UnitTestContext* context, List& } List optArgs; - String recordFileName = Path::combine("slang-record", fileNames[0]); + String recordFileName = Path::combine(recordDir, fileNames[0]); optArgs.add(recordFileName.getBuffer()); RefPtr process; @@ -377,14 +383,14 @@ static SlangResult resultCompare( return SLANG_OK; } -static SlangResult cleanupRecordFiles() +static SlangResult cleanupRecordFiles(const String& recordDir) { - SlangResult res = Path::removeNonEmpty("slang-record"); + SlangResult res = Path::removeNonEmpty(recordDir.getBuffer()); if (SLANG_FAILED(res)) { - getTestReporter()->message( - TestMessageType::TestFailure, - "Failed to remove 'slang-record' directory\n"); + StringBuilder msgBuilder; + msgBuilder << "Failed to remove '" << recordDir << "' directory\n"; + getTestReporter()->message(TestMessageType::TestFailure, msgBuilder.toString().getBuffer()); } return res; @@ -392,29 +398,34 @@ static SlangResult cleanupRecordFiles() static SlangResult runTest(UnitTestContext* context, const char* testName) { + // Create unique directory for this test to avoid conflicts + StringBuilder recordDirBuilder; + recordDirBuilder << "slang-record-" << testName; + String recordDir = recordDirBuilder.toString(); + List expectHashes; List resultHashes; SlangResult res = SLANG_OK; - if ((res = runExample(context, testName, expectHashes)) != SLANG_OK) - { - goto error; - } - - if ((res = replayExample(context, resultHashes)) != SLANG_OK) - { - goto error; - } - if ((res = resultCompare(expectHashes, resultHashes)) != SLANG_OK) + // Run the example to generate recording + res = runExample(context, testName, recordDir, expectHashes); + if (SLANG_SUCCEEDED(res)) { - goto error; + // Replay the recording + res = replayExample(context, recordDir, resultHashes); + if (SLANG_SUCCEEDED(res)) + { + // Compare results + res = resultCompare(expectHashes, resultHashes); + } } -error: - cleanupRecordFiles(); + // Always cleanup, regardless of success or failure + cleanupRecordFiles(recordDir); return res; } +#if 0 static SlangResult runTests(UnitTestContext* context) { const char* testBinaryNames[] = { @@ -445,14 +456,56 @@ static SlangResult runTests(UnitTestContext* context) return finalRes; } +#endif // Those examples all depend on the Vulkan, so we only run them on non-Apple platforms. // In the future, we may be able to modify the examples further to remove all the render APIs // such that it can be ran on Apple platforms. #if !(SLANG_APPLE_FAMILY) -SLANG_UNIT_TEST(RecordReplay) + +SLANG_UNIT_TEST(RecordReplay_cpu_hello_world) +{ + SLANG_CHECK(SLANG_SUCCEEDED(runTest(unitTestContext, "cpu-hello-world"))); +} + +SLANG_UNIT_TEST(RecordReplay_triangle) +{ + SLANG_CHECK(SLANG_SUCCEEDED(runTest(unitTestContext, "triangle"))); +} + +SLANG_UNIT_TEST(RecordReplay_ray_tracing) +{ + SLANG_CHECK(SLANG_SUCCEEDED(runTest(unitTestContext, "ray-tracing"))); +} + +SLANG_UNIT_TEST(RecordReplay_ray_tracing_pipeline) +{ + SLANG_CHECK(SLANG_SUCCEEDED(runTest(unitTestContext, "ray-tracing-pipeline"))); +} + +SLANG_UNIT_TEST(RecordReplay_autodiff_texture) +{ + SLANG_CHECK(SLANG_SUCCEEDED(runTest(unitTestContext, "autodiff-texture"))); +} + +SLANG_UNIT_TEST(RecordReplay_gpu_printing) +{ + SLANG_CHECK(SLANG_SUCCEEDED(runTest(unitTestContext, "gpu-printing"))); +} + +#if 0 +// These examples requires reflection API to replay, we have to disable +// it for now. "model-viewer", + +SLANG_UNIT_TEST(RecordReplay_shader_object) { - SLANG_CHECK(SLANG_SUCCEEDED(runTests(unitTestContext))); + SLANG_CHECK(SLANG_SUCCEEDED(runTest(unitTestContext, "shader-object"))); } +SLANG_UNIT_TEST(RecordReplay_model_viewer) +{ + SLANG_CHECK(SLANG_SUCCEEDED(runTest(unitTestContext, "model-viewer"))); +} +#endif + #endif -- cgit v1.2.3