Use d3d12/vk debug layer when gfx debug layer is enabled. (#2411)

* Use d3d12/vk debug layer when gfx debug layer is enabled.

* Fix.

Co-authored-by: Yong He <yhe@nvidia.com>

7 files changed, 23 insertions, 7 deletions

diff --git a/tools/gfx/d3d12/d3d12-device.cpp b/tools/gfx/d3d12/d3d12-device.cpp
index efc1beb22..e682764f5 100644
--- a/tools/gfx/d3d12/d3d12-device.cpp
+++ b/tools/gfx/d3d12/d3d12-device.cpp
@@ -471,12 +471,15 @@ Result DeviceImpl::initialize(const Desc& desc)
         // TODO: we should probably provide a command-line option
         // to override UseDebug of default rather than leave it
         // up to each back-end to specify.
-#if ENABLE_DEBUG_LAYER
-        combiner.add(
-            DeviceCheckFlag::UseDebug, ChangeType::OnOff); ///< First try debug then non debug
-#else
-        combiner.add(DeviceCheckFlag::UseDebug, ChangeType::Off); ///< Don't bother with debug
-#endif
+        if (ENABLE_DEBUG_LAYER || isGfxDebugLayerEnabled())
+        {
+            combiner.add(
+                DeviceCheckFlag::UseDebug, ChangeType::OnOff); ///< First try debug then non debug
+        }
+        else
+        {
+            combiner.add(DeviceCheckFlag::UseDebug, ChangeType::Off); ///< Don't bother with debug
+        }
         combiner.add(
             DeviceCheckFlag::UseHardwareDevice,
             ChangeType::OnOff); ///< First try hardware, then reference
diff --git a/tools/gfx/debug-layer/debug-command-queue.cpp b/tools/gfx/debug-layer/debug-command-queue.cpp
index 253541abb..c196fa06a 100644
--- a/tools/gfx/debug-layer/debug-command-queue.cpp
+++ b/tools/gfx/debug-layer/debug-command-queue.cpp
@@ -47,6 +47,10 @@ void DebugCommandQueue::executeCommandBuffers(GfxCount count, ICommandBuffer* co
         }
     }
     baseObject->executeCommandBuffers(count, innerCommandBuffers.getBuffer(), getInnerObj(fence), valueToSignal);
+    if (fence)
+    {
+        getDebugObj(fence)->maxValueToSignal = Math::Max(getDebugObj(fence)->maxValueToSignal, valueToSignal);
+    }
 }
 
 void DebugCommandQueue::waitOnHost()
diff --git a/tools/gfx/debug-layer/debug-fence.cpp b/tools/gfx/debug-layer/debug-fence.cpp
index 7ecbfabd7..66c541618 100644
--- a/tools/gfx/debug-layer/debug-fence.cpp
+++ b/tools/gfx/debug-layer/debug-fence.cpp
@@ -31,6 +31,10 @@ Result DebugFence::getCurrentValue(uint64_t* outValue)
 Result DebugFence::setCurrentValue(uint64_t value)
 {
     SLANG_GFX_API_FUNC;
+    if (value < maxValueToSignal)
+    {
+        GFX_DIAGNOSE_ERROR_FORMAT("Cannot set fence value (%d) to lower than pending signal value (%d) on the fence.", value, maxValueToSignal);
+    }
     return baseObject->setCurrentValue(value);
 }
 
diff --git a/tools/gfx/debug-layer/debug-fence.h b/tools/gfx/debug-layer/debug-fence.h
index 980596d90..1f61fac33 100644
--- a/tools/gfx/debug-layer/debug-fence.h
+++ b/tools/gfx/debug-layer/debug-fence.h
@@ -18,6 +18,8 @@ public:
     virtual SLANG_NO_THROW Result SLANG_MCALL setCurrentValue(uint64_t value) override;
     virtual SLANG_NO_THROW Result SLANG_MCALL getSharedHandle(InteropHandle* outHandle) override;
     virtual SLANG_NO_THROW Result SLANG_MCALL getNativeHandle(InteropHandle* outNativeHandle) override;
+public:
+    uint64_t maxValueToSignal = 0;
 };
 
 } // namespace debug
diff --git a/tools/gfx/render.cpp b/tools/gfx/render.cpp
index e32ed8f6c..b19824671 100644
--- a/tools/gfx/render.cpp
+++ b/tools/gfx/render.cpp
@@ -16,6 +16,7 @@ Result SLANG_MCALL createCUDADevice(const IDevice::Desc* desc, IDevice** outDevi
 Result SLANG_MCALL createCPUDevice(const IDevice::Desc* desc, IDevice** outDevice);
 
 static bool debugLayerEnabled = false;
+bool isGfxDebugLayerEnabled() { return debugLayerEnabled; }
 
 /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Global Renderer Functions !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */
 
diff --git a/tools/gfx/renderer-shared.h b/tools/gfx/renderer-shared.h
index 0d71059fc..d0e4b52fb 100644
--- a/tools/gfx/renderer-shared.h
+++ b/tools/gfx/renderer-shared.h
@@ -45,6 +45,8 @@ struct GfxGUID
     static const Slang::Guid IID_ID3D12TransientResourceHeap;
 };
 
+bool isGfxDebugLayerEnabled();
+
 // We use a `BreakableReference` to avoid the cyclic reference situation in gfx implementation.
 // It is a common scenario where objects created from an `IDevice` implementation needs to hold
 // a strong reference to the device object that creates them. For example, a `Buffer` or a
diff --git a/tools/gfx/vulkan/vk-device.cpp b/tools/gfx/vulkan/vk-device.cpp
index c2d82f8a7..462c1b44c 100644
--- a/tools/gfx/vulkan/vk-device.cpp
+++ b/tools/gfx/vulkan/vk-device.cpp
@@ -662,7 +662,7 @@ SlangResult DeviceImpl::initialize(const Desc& desc)
             continue;
         descriptorSetAllocator.m_api = &m_api;
         initDeviceResult = initVulkanInstanceAndDevice(
-            desc.existingDeviceHandles.handles, ENABLE_VALIDATION_LAYER != 0);
+            desc.existingDeviceHandles.handles, ENABLE_VALIDATION_LAYER != 0 || isGfxDebugLayerEnabled());
         if (initDeviceResult == SLANG_OK)
             break;
     }