Enable Windows full debug testsuite in CI (#7085)

* Unify Debug Layer Control Logic and Add Disable Option for Debug Builds

This PR refactors and unifies the debug layer control logic in slang-test.
A new `-disable-debug-layers` option is introduced, allowing debug builds to skip enabling the validation (debug) layer.
This is currently needed to ensure stability in the debug test suite.

Previously, different toggles such as ENABLE_VALIDATION_LAYER, ENABLE_DEBUG_LAYER, and debugLayerEnabled were used inconsistently across different components of slang-test. This PR standardizes the logic by using a single variable, debugLayerEnabled, to control the enabling/disabling of the debug layer internally.

Notes:
By default, the debug/validation layer is enabled in debug builds and is not supported in release builds of slang-test.

Fixes: #7132

* Disable spirv-opt for the DebugFunctionDefinition issue

* Run debug build only in GCP machines

* Fix VUID-vkCmdPipelineBarrier-pBufferMemoryBarriers-02818

dstAcessMask can't include VK_ACCESS_TRANSFER_READ_BIT when stage mask
has  VK_PIPELINE_STAGE_RAY_TRACING_SHADER_BIT_KHR

* Set failed retry limit to 32

---------

Co-authored-by: slangbot <ellieh+slangbot@nvidia.com>
Co-authored-by: slangbot <186143334+slangbot@users.noreply.github.com>

5 files changed, 41 insertions, 12 deletions

diff --git a/tools/slang-test/options.cpp b/tools/slang-test/options.cpp
index 6be590ef2..da4f9cffe 100644
--- a/tools/slang-test/options.cpp
+++ b/tools/slang-test/options.cpp
@@ -1,4 +1,4 @@
-// test-context.cpp
+// options.cpp
 #include "options.h"
 
 #include "../../source/core/slang-io.h"
@@ -88,6 +88,10 @@ static bool _isSubCommand(const char* arg)
         "  -use-test-server               Run tests using test server\n"
         "  -use-fully-isolated-test-server  Run each test in isolated server\n"
         "  -capability <name>             Compile with the given capability\n"
+#if _DEBUG
+        "  -disable-debug-layers          Disable the debug layers (default enabled in debug "
+        "build)\n"
+#endif
         "\n"
         "Output modes:\n"
         "  -appveyor                      Use AppVeyor output format\n"
@@ -406,6 +410,12 @@ static bool _isSubCommand(const char* arg)
         {
             optionsOut->skipReferenceImageGeneration = true;
         }
+#if _DEBUG
+        else if (strcmp(arg, "-disable-debug-layers") == 0)
+        {
+            optionsOut->debugLayerEnabled = false;
+        }
+#endif
         else
         {
             stdError.print("unknown option '%s'\n", arg);
diff --git a/tools/slang-test/options.h b/tools/slang-test/options.h
index 7fda33198..6e408374e 100644
--- a/tools/slang-test/options.h
+++ b/tools/slang-test/options.h
@@ -83,6 +83,15 @@ struct Options
     // integration builds.
     bool dumpOutputOnFailure = false;
 
+    // When true it will run with debug layer (e.g. vulkan validation layer)
+#if _DEBUG
+    // Default is true for debug build
+    bool debugLayerEnabled = true;
+#else
+    // Default is false for release build
+    bool debugLayerEnabled = false;
+#endif
+
     // Set the default spawn type to use
     // Having tests isolated, slows down testing considerably, so using UseSharedLibrary is the most
     // desirable default usually.
diff --git a/tools/slang-test/slang-test-main.cpp b/tools/slang-test/slang-test-main.cpp
index 92e50737e..02d305a46 100644
--- a/tools/slang-test/slang-test-main.cpp
+++ b/tools/slang-test/slang-test-main.cpp
@@ -3607,8 +3607,10 @@ TestResult runComputeComparisonImpl(
     // gets misinterpreted as the result from the test.
     // This is due to the limitation that Slang RPC implementation expects only
     // one time communication.
-    if (input.spawnType != SpawnType::UseTestServer)
+    if (context->options.debugLayerEnabled && input.spawnType != SpawnType::UseTestServer)
+    {
         cmdLine.addArg("-enable-debug-layers");
+    }
 #endif
 
     if (context->isExecuting())
@@ -4344,7 +4346,7 @@ static SlangResult _runTestsOnFile(TestContext* context, String filePath)
             context->setTestRequirements(&requirements);
             runTest(context, filePath, filePath, filePath, testDetails.options);
 
-            //
+
             apiUsedFlags |= requirements.usedRenderApiFlags;
             explictUsedApiFlags |= (requirements.explicitRenderApi != RenderApiType::Unknown)
                                        ? (RenderApiFlags(1) << int(requirements.explicitRenderApi))
@@ -4703,6 +4705,7 @@ static SlangResult runUnitTestModule(
     unitTestContext.slangGlobalSession = context->getSession();
     unitTestContext.workDirectory = "";
     unitTestContext.enabledApis = context->options.enabledApis;
+    unitTestContext.enableDebugLayers = context->options.debugLayerEnabled;
     unitTestContext.executableDirectory = context->exeDirectoryPath.getBuffer();
 
     auto testCount = testModule->getTestCount();
@@ -5054,21 +5057,20 @@ SlangResult innerMain(int argc, char** argv)
             TestReporter::SuiteScope suiteScope(&reporter, "unit tests");
             TestReporter::set(&reporter);
 
-            for (bool isRetry : {false, true})
+            // Try the unit tests up to 3 times
+            for (bool isRetry : {false, true, true})
             {
-                auto spawnType = context.getFinalSpawnType();
-
-                context.isRetry = false;
+                // Use default spawn type for unit tests as the test server one is unstable
+                auto spawnType = SpawnType::Default;
+                context.isRetry = isRetry;
                 if (isRetry)
                 {
                     if (context.failedUnitTests.getCount() == 0)
                         break;
 
-                    printf("Retrying unit tests...\n");
-                    context.isRetry = true;
+                    printf("Retrying unit tests with default spawn type...\n");
                     context.options.testPrefixes = context.failedUnitTests;
                     context.failedUnitTests.clear();
-                    spawnType = SpawnType::Default;
                 }
 
                 // Run the unit tests
@@ -5090,14 +5092,20 @@ SlangResult innerMain(int argc, char** argv)
         }
 
         // If we have a couple failed tests, they maybe intermittent failures due to parallel
-        // excution or driver instability. We can try running them again.
+        // excution or driver instability. We can try running them again. Debug build has more
+        // instability at this moment, so we allow more retries.
+#if _DEBUG
+        static constexpr int kFailedTestLimitForRetry = 32;
+#else
         static constexpr int kFailedTestLimitForRetry = 16;
+#endif
         if (context.failedFileTests.getCount() <= kFailedTestLimitForRetry)
         {
             if (context.failedFileTests.getCount() > 0)
                 printf("Retrying %d failed tests...\n", (int)context.failedFileTests.getCount());
             for (auto& test : context.failedFileTests)
             {
+                context.isRetry = true;
                 FileTestInfoImpl* fileTestInfo = static_cast<FileTestInfoImpl*>(test.Ptr());
                 TestReporter::SuiteScope suiteScope(&reporter, "tests");
                 TestReporter::TestScope scope(&reporter, fileTestInfo->testName);
diff --git a/tools/slang-test/test-context.h b/tools/slang-test/test-context.h
index 97562da99..6720cd648 100644
--- a/tools/slang-test/test-context.h
+++ b/tools/slang-test/test-context.h
@@ -181,6 +181,8 @@ public:
     Slang::RefPtr<Slang::JSONRPCConnection> m_languageServerConnection;
 
     bool isRetry;
+    bool enableDebugLayers;
+
     std::mutex mutexFailedTests;
     Slang::List<Slang::RefPtr<FileTestInfo>> failedFileTests;
     Slang::List<Slang::String> failedUnitTests;
diff --git a/tools/slang-test/test-reporter.cpp b/tools/slang-test/test-reporter.cpp
index 1f6c00637..352245028 100644
--- a/tools/slang-test/test-reporter.cpp
+++ b/tools/slang-test/test-reporter.cpp
@@ -693,7 +693,7 @@ void TestReporter::outputSummary()
             printf("\n===\n\n");
             if (m_failedTestCount)
             {
-                printf("failing tests:\n");
+                printf("%d failing tests:\n", m_failedTestCount);
                 printf("---\n");
                 for (const auto& testInfo : m_testInfos)
                 {