Add a new slang-test option `-enable-debug-layers` (#7300)

* Add a new slang-test option `-enable-debug-layers`

A variable `disableDebugLayer` is renamed to `enableDebugLayers`,
and a corresponding command-line argument is added,
`-enable-debug-layers`.

The previous option `-disable-debug-layer` is still available, but it
prints a deprecation warning message.

The reason why it is added is to make the option available to both Debug
and Release. On Debug build, it will be enabled by default, and it will
be disabled on Release build. We should be able to not only disable it,
but also enable it on Release build.

Ideally this option should be enabled all the time, but currently there
are too many VUID error messages printed and we are enabling only for
Debug build for now.

Note that the CI/CD will run with the option disabled until we resolve
all of VUID errors.

8 files changed, 53 insertions, 35 deletions

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 62da40295..58e5d6019 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -200,7 +200,7 @@ jobs:
               -expected-failure-list tests/expected-failure-record-replay-tests.txt \
               -skip-reference-image-generation \
               -show-adapter-info \
-              ${{ matrix.config == 'debug' && '-disable-debug-layers' || '' }}
+              -enable-debug-layers false
           elif [[ "${{matrix.has-gpu}}" == "true" ]]; then
             "$bin_dir/slang-test" \
               -use-test-server \
@@ -211,7 +211,7 @@ jobs:
               -expected-failure-list tests/expected-failure-github-runner.txt \
               -skip-reference-image-generation \
               -show-adapter-info  \
-              ${{ matrix.config == 'debug' && '-disable-debug-layers' || '' }}
+              -enable-debug-layers false
           else
             "$bin_dir/slang-test" \
               -use-test-server \
@@ -222,7 +222,7 @@ jobs:
               -expected-failure-list tests/expected-failure-github-runner.txt \
               -skip-reference-image-generation \
               -show-adapter-info \
-              ${{ matrix.config == 'debug' && '-disable-debug-layers' || '' }}
+              -enable-debug-layers false
           fi
       - name: Run Slang examples
         if: steps.filter.outputs.should-run == 'true' && matrix.platform != 'wasm' && matrix.full-gpu-tests
diff --git a/source/compiler-core/slang-test-server-protocol.h b/source/compiler-core/slang-test-server-protocol.h
index a15e19701..7d7ad2389 100644
--- a/source/compiler-core/slang-test-server-protocol.h
+++ b/source/compiler-core/slang-test-server-protocol.h
@@ -17,6 +17,7 @@ struct ExecuteUnitTestArgs
     String moduleName;
     String testName;
     uint32_t enabledApis;
+    bool enableDebugLayers;
 
     static const UnownedStringSlice g_methodName;
     static const StructRttiInfo g_rttiInfo;
diff --git a/tools/render-test/render-test-main.cpp b/tools/render-test/render-test-main.cpp
index 81b937a2a..48ebbd332 100644
--- a/tools/render-test/render-test-main.cpp
+++ b/tools/render-test/render-test-main.cpp
@@ -1396,10 +1396,8 @@ static SlangResult _innerMain(
         DeviceDesc desc = {};
         desc.deviceType = options.deviceType;
 
-#if _DEBUG
-        desc.enableValidation = true;
+        desc.enableValidation = options.enableDebugLayers;
         desc.debugCallback = &debugCallback;
-#endif
 
         desc.slang.lineDirectiveMode = SLANG_LINE_DIRECTIVE_MODE_NONE;
         if (options.generateSPIRVDirectly)
diff --git a/tools/slang-test/options.cpp b/tools/slang-test/options.cpp
index da4f9cffe..aa538e75e 100644
--- a/tools/slang-test/options.cpp
+++ b/tools/slang-test/options.cpp
@@ -88,6 +88,8 @@ static bool _isSubCommand(const char* arg)
         "  -use-test-server               Run tests using test server\n"
         "  -use-fully-isolated-test-server  Run each test in isolated server\n"
         "  -capability <name>             Compile with the given capability\n"
+        "  -enable-debug-layers [true|false] Enable or disable Validation Layer for Vulkan\n"
+        "                                 and Debug Device for DX\n"
 #if _DEBUG
         "  -disable-debug-layers          Disable the debug layers (default enabled in debug "
         "build)\n"
@@ -120,6 +122,19 @@ static bool _isSubCommand(const char* arg)
     char const* const* argCursor = argv;
     char const* const* argEnd = argCursor + argCount;
 
+#if _DEBUG
+    // Enabling debug layers by default in debug builds.
+    // For DX12 it will use the debug layer, for Vulkan it will enable validation layers.
+    //
+    // CI/CD will explicitly disable this until we address all of VUID errors.
+    // https://github.com/shader-slang/slang/issues/4798
+    //
+    // When you run the Debug build locally, you may see more errors if not disabled with
+    // '-enable-debug-layers false'.
+    //
+    optionsOut->enableDebugLayers = true;
+#endif
+
     // first argument is the application name
     if (argCursor != argEnd)
     {
@@ -410,10 +425,32 @@ static bool _isSubCommand(const char* arg)
         {
             optionsOut->skipReferenceImageGeneration = true;
         }
+        else if (strcmp(arg, "-enable-debug-layers") == 0)
+        {
+            optionsOut->enableDebugLayers = true;
+
+            if (argCursor == argEnd)
+            {
+                stdError.print("error: expected operand for '%s'\n", arg);
+                showHelp(stdError);
+                return SLANG_FAIL;
+            }
+
+            // Check for false variants
+            const char* value = *argCursor++;
+            if (value[0] == 'f' || value[0] == 'F' || value[0] == 'n' || value[0] == 'N' ||
+                value[0] == '0' ||
+                ((value[0] == 'o' || value[0] == 'O') && (value[1] == 'f' || value[1] == 'F')))
+            {
+                optionsOut->enableDebugLayers = false;
+            }
+        }
 #if _DEBUG
         else if (strcmp(arg, "-disable-debug-layers") == 0)
         {
-            optionsOut->debugLayerEnabled = false;
+            stdError.print("warning: '-disable-debug-layers' is deprecated, use "
+                           "'-enable-debug-layers false'\n");
+            optionsOut->enableDebugLayers = false;
         }
 #endif
         else
diff --git a/tools/slang-test/options.h b/tools/slang-test/options.h
index 6e408374e..b2f71d3e3 100644
--- a/tools/slang-test/options.h
+++ b/tools/slang-test/options.h
@@ -84,13 +84,7 @@ struct Options
     bool dumpOutputOnFailure = false;
 
     // When true it will run with debug layer (e.g. vulkan validation layer)
-#if _DEBUG
-    // Default is true for debug build
-    bool debugLayerEnabled = true;
-#else
-    // Default is false for release build
-    bool debugLayerEnabled = false;
-#endif
+    bool enableDebugLayers = false;
 
     // Set the default spawn type to use
     // Having tests isolated, slows down testing considerably, so using UseSharedLibrary is the most
diff --git a/tools/slang-test/slang-test-main.cpp b/tools/slang-test/slang-test-main.cpp
index 72af73cc1..f0a140549 100644
--- a/tools/slang-test/slang-test-main.cpp
+++ b/tools/slang-test/slang-test-main.cpp
@@ -3394,6 +3394,11 @@ static void _addRenderTestOptions(const Options& options, CommandLine& ioCmdLine
         ioCmdLine.addArg("-capability");
         ioCmdLine.addArg(capability);
     }
+
+    if (options.enableDebugLayers)
+    {
+        ioCmdLine.addArg("-enable-debug-layers");
+    }
 }
 
 static SlangResult _extractProfileTime(const UnownedStringSlice& text, double& timeOut)
@@ -3602,17 +3607,6 @@ TestResult runComputeComparisonImpl(
     auto actualOutputFile = outputStem + ".actual.txt";
     cmdLine.addArg(actualOutputFile);
 
-#if _DEBUG
-    // When using test server, any validation warning printed from the backend
-    // gets misinterpreted as the result from the test.
-    // This is due to the limitation that Slang RPC implementation expects only
-    // one time communication.
-    if (context->options.debugLayerEnabled && input.spawnType != SpawnType::UseTestServer)
-    {
-        cmdLine.addArg("-enable-debug-layers");
-    }
-#endif
-
     if (context->isExecuting())
     {
         // clear the stale actual output file first. This will allow us to detect error if
@@ -4705,7 +4699,7 @@ static SlangResult runUnitTestModule(
     unitTestContext.slangGlobalSession = context->getSession();
     unitTestContext.workDirectory = "";
     unitTestContext.enabledApis = context->options.enabledApis;
-    unitTestContext.enableDebugLayers = context->options.debugLayerEnabled;
+    unitTestContext.enableDebugLayers = context->options.enableDebugLayers;
     unitTestContext.executableDirectory = context->exeDirectoryPath.getBuffer();
 
     auto testCount = testModule->getTestCount();
@@ -4749,6 +4743,7 @@ static SlangResult runUnitTestModule(
         {
             TestServerProtocol::ExecuteUnitTestArgs args;
             args.enabledApis = context->options.enabledApis;
+            args.enableDebugLayers = context->options.enableDebugLayers;
             args.moduleName = moduleName;
             args.testName = test.testName;
 
diff --git a/tools/slang-test/test-context.h b/tools/slang-test/test-context.h
index 6720cd648..e760e8dda 100644
--- a/tools/slang-test/test-context.h
+++ b/tools/slang-test/test-context.h
@@ -180,9 +180,7 @@ public:
     std::mutex mutex;
     Slang::RefPtr<Slang::JSONRPCConnection> m_languageServerConnection;
 
-    bool isRetry;
-    bool enableDebugLayers;
-
+    bool isRetry = false;
     std::mutex mutexFailedTests;
     Slang::List<Slang::RefPtr<FileTestInfo>> failedFileTests;
     Slang::List<Slang::String> failedUnitTests;
diff --git a/tools/test-server/test-server-main.cpp b/tools/test-server/test-server-main.cpp
index c00cab428..5161fa03a 100644
--- a/tools/test-server/test-server-main.cpp
+++ b/tools/test-server/test-server-main.cpp
@@ -446,12 +446,7 @@ SlangResult TestServer::_executeUnitTest(const JSONRPCCall& call)
     unitTestContext.workDirectory = "";
     unitTestContext.enabledApis = RenderApiFlags(args.enabledApis);
     unitTestContext.executableDirectory = m_exeDirectory.getBuffer();
-    // When using test server, any validation warning printed from the backend
-    // gets misinterpreted as the result from the test.
-    // This is due to the limitation that Slang RPC implementation expects only
-    // one time communication. Set enableDebugLayers to false to avoid Vulkan
-    // test failures when running on debug using test server.
-    unitTestContext.enableDebugLayers = false;
+    unitTestContext.enableDebugLayers = args.enableDebugLayers;
 
     auto testCount = testModule->getTestCount();
     SLANG_ASSERT(testIndex >= 0 && testIndex < testCount);