Fix gfx debug layer crashes. (#2416)

Co-authored-by: Yong He <yhe@nvidia.com>

5 files changed, 37 insertions, 50 deletions

diff --git a/tools/gfx/debug-layer/debug-command-buffer.cpp b/tools/gfx/debug-layer/debug-command-buffer.cpp
index 3785bcd43..aa9c6b8b8 100644
--- a/tools/gfx/debug-layer/debug-command-buffer.cpp
+++ b/tools/gfx/debug-layer/debug-command-buffer.cpp
@@ -30,10 +30,8 @@ void DebugCommandBuffer::encodeRenderCommands(
     SLANG_GFX_API_FUNC;
     checkCommandBufferOpenWhenCreatingEncoder();
     checkEncodersClosedBeforeNewEncoder();
-    auto innerRenderPass =
-        renderPass ? static_cast<DebugRenderPassLayout*>(renderPass)->baseObject : nullptr;
-    auto innerFramebuffer =
-        framebuffer ? static_cast<DebugFramebuffer*>(framebuffer)->baseObject : nullptr;
+    auto innerRenderPass = getInnerObj(renderPass);
+    auto innerFramebuffer = getInnerObj(framebuffer);
     m_renderCommandEncoder.isOpen = true;
     baseObject->encodeRenderCommands(
         innerRenderPass, innerFramebuffer, &m_renderCommandEncoder.baseObject);
diff --git a/tools/gfx/debug-layer/debug-command-encoder.cpp b/tools/gfx/debug-layer/debug-command-encoder.cpp
index c2d146a8b..b8b005338 100644
--- a/tools/gfx/debug-layer/debug-command-encoder.cpp
+++ b/tools/gfx/debug-layer/debug-command-encoder.cpp
@@ -30,7 +30,7 @@ Result DebugComputeCommandEncoder::bindPipeline(
 {
     SLANG_GFX_API_FUNC;
 
-    auto innerState = static_cast<DebugPipelineState*>(state)->baseObject;
+    auto innerState = getInnerObj(state);
     IShaderObject* innerRootObject = nullptr;
     commandBuffer->rootObject.reset();
     auto result = baseObject->bindPipeline(innerState, &innerRootObject);
@@ -72,7 +72,7 @@ Result DebugRenderCommandEncoder::bindPipeline(
 {
     SLANG_GFX_API_FUNC;
 
-    auto innerState = static_cast<DebugPipelineState*>(state)->baseObject;
+    auto innerState = getInnerObj(state);
     IShaderObject* innerRootObject = nullptr;
     commandBuffer->rootObject.reset();
     auto result = baseObject->bindPipeline(innerState, &innerRootObject);
diff --git a/tools/gfx/debug-layer/debug-command-queue.cpp b/tools/gfx/debug-layer/debug-command-queue.cpp
index c196fa06a..d094f3438 100644
--- a/tools/gfx/debug-layer/debug-command-queue.cpp
+++ b/tools/gfx/debug-layer/debug-command-queue.cpp
@@ -27,8 +27,8 @@ void DebugCommandQueue::executeCommandBuffers(GfxCount count, ICommandBuffer* co
     for (GfxIndex i = 0; i < count; i++)
     {
         auto cmdBufferIn = commandBuffers[i];
-        auto cmdBufferImpl = static_cast<DebugCommandBuffer*>(cmdBufferIn);
-        auto innerCmdBuffer = cmdBufferImpl->baseObject.get();
+        auto cmdBufferImpl = getDebugObj(cmdBufferIn);
+        auto innerCmdBuffer = getInnerObj(cmdBufferIn);
         innerCommandBuffers.add(innerCmdBuffer);
         if (cmdBufferImpl->isOpen)
         {
diff --git a/tools/gfx/debug-layer/debug-device.cpp b/tools/gfx/debug-layer/debug-device.cpp
index 356c875d4..7790149b6 100644
--- a/tools/gfx/debug-layer/debug-device.cpp
+++ b/tools/gfx/debug-layer/debug-device.cpp
@@ -191,7 +191,7 @@ Result DebugDevice::createTextureView(
 
     RefPtr<DebugResourceView> outObject = new DebugResourceView();
     auto result = baseObject->createTextureView(
-        static_cast<DebugTextureResource*>(texture)->baseObject,
+        getInnerObj(texture),
         desc,
         outObject->baseObject.writeRef());
     if (SLANG_FAILED(result))
@@ -210,8 +210,8 @@ Result DebugDevice::createBufferView(
 
     RefPtr<DebugResourceView> outObject = new DebugResourceView();
     auto result = baseObject->createBufferView(
-        static_cast<DebugBufferResource*>(buffer)->baseObject,
-        counterBuffer ? static_cast<DebugBufferResource*>(counterBuffer)->baseObject : nullptr,
+        getInnerObj(buffer),
+        getInnerObj(counterBuffer),
         desc,
         outObject->baseObject.writeRef());
     if (SLANG_FAILED(result))
@@ -263,19 +263,12 @@ Result DebugDevice::createFramebuffer(IFramebuffer::Desc const& desc, IFramebuff
     SLANG_GFX_API_FUNC;
 
     auto innerDesc = desc;
-    innerDesc.layout =
-        desc.layout ? static_cast<DebugFramebufferLayout*>(desc.layout)->baseObject.get() : nullptr;
-    innerDesc.depthStencilView =
-        desc.depthStencilView
-            ? static_cast<DebugResourceView*>(desc.depthStencilView)->baseObject.get()
-            : nullptr;
+    innerDesc.layout = getInnerObj(desc.layout);
+    innerDesc.depthStencilView = getInnerObj(desc.depthStencilView);
     List<IResourceView*> innerRenderTargets;
     for (GfxIndex i = 0; i < desc.renderTargetCount; i++)
     {
-        auto innerRenderTarget =
-            desc.renderTargetViews[i]
-                ? static_cast<DebugResourceView*>(desc.renderTargetViews[i])->baseObject.get()
-                : nullptr;
+        auto innerRenderTarget = getInnerObj(desc.renderTargetViews[i]);
         innerRenderTargets.add(innerRenderTarget);
     }
     innerDesc.renderTargetViews = innerRenderTargets.getBuffer();
@@ -295,9 +288,7 @@ Result DebugDevice::createRenderPassLayout(
     SLANG_GFX_API_FUNC;
 
     auto innerDesc = desc;
-    innerDesc.framebufferLayout =
-        desc.framebufferLayout? static_cast<DebugFramebufferLayout*>(desc.framebufferLayout)->baseObject.get()
-            : nullptr;
+    innerDesc.framebufferLayout = getInnerObj(desc.framebufferLayout);
     RefPtr<DebugRenderPassLayout> outObject = new DebugRenderPassLayout();
     auto result = baseObject->createRenderPassLayout(innerDesc, outObject->baseObject.writeRef());
     if (SLANG_FAILED(result))
@@ -314,9 +305,9 @@ Result DebugDevice::createSwapchain(
     SLANG_GFX_API_FUNC;
 
     auto innerDesc = desc;
-    innerDesc.queue = static_cast<DebugCommandQueue*>(desc.queue)->baseObject.get();
+    innerDesc.queue = getInnerObj(desc.queue);
     RefPtr<DebugSwapchain> outObject = new DebugSwapchain();
-    outObject->queue = static_cast<DebugCommandQueue*>(desc.queue);
+    outObject->queue = getDebugObj(desc.queue);
     auto result = baseObject->createSwapchain(innerDesc, window, outObject->baseObject.writeRef());
     if (SLANG_FAILED(result))
         return result;
@@ -479,14 +470,9 @@ Result DebugDevice::createGraphicsPipelineState(
     SLANG_GFX_API_FUNC;
 
     GraphicsPipelineStateDesc innerDesc = desc;
-    innerDesc.program =
-        desc.program ? static_cast<DebugShaderProgram*>(desc.program)->baseObject : nullptr;
-    innerDesc.inputLayout =
-        desc.inputLayout ? static_cast<DebugInputLayout*>(desc.inputLayout)->baseObject : nullptr;
-    innerDesc.framebufferLayout =
-        desc.framebufferLayout
-            ? static_cast<DebugFramebufferLayout*>(desc.framebufferLayout)->baseObject
-            : nullptr;
+    innerDesc.program = getInnerObj(desc.program);
+    innerDesc.inputLayout = getInnerObj(desc.inputLayout);
+    innerDesc.framebufferLayout = getInnerObj(desc.framebufferLayout);
     RefPtr<DebugPipelineState> outObject = new DebugPipelineState();
     auto result =
         baseObject->createGraphicsPipelineState(innerDesc, outObject->baseObject.writeRef());
@@ -503,7 +489,7 @@ Result DebugDevice::createComputePipelineState(
     SLANG_GFX_API_FUNC;
 
     ComputePipelineStateDesc innerDesc = desc;
-    innerDesc.program = static_cast<DebugShaderProgram*>(desc.program)->baseObject;
+    innerDesc.program = getInnerObj(desc.program);
 
     RefPtr<DebugPipelineState> outObject = new DebugPipelineState();
     auto result =
@@ -521,7 +507,7 @@ Result DebugDevice::createRayTracingPipelineState(
     SLANG_GFX_API_FUNC;
 
     RayTracingPipelineStateDesc innerDesc = desc;
-    innerDesc.program = static_cast<DebugShaderProgram*>(desc.program)->baseObject;
+    innerDesc.program = getInnerObj(desc.program);
 
     RefPtr<DebugPipelineState> outObject = new DebugPipelineState();
     auto result =
@@ -540,9 +526,8 @@ SlangResult DebugDevice::readTextureResource(
     size_t* outPixelSize)
 {
     SLANG_GFX_API_FUNC;
-    auto resourceImpl = static_cast<DebugTextureResource*>(resource);
     return baseObject->readTextureResource(
-        resourceImpl->baseObject, state, outBlob, outRowPitch, outPixelSize);
+        getInnerObj(resource), state, outBlob, outRowPitch, outPixelSize);
 }
 
 SlangResult DebugDevice::readBufferResource(
@@ -552,8 +537,7 @@ SlangResult DebugDevice::readBufferResource(
     ISlangBlob** outBlob)
 {
     SLANG_GFX_API_FUNC;
-    auto bufferImpl = static_cast<DebugBufferResource*>(buffer);
-    return baseObject->readBufferResource(bufferImpl->baseObject, offset, size, outBlob);
+    return baseObject->readBufferResource(getInnerObj(buffer), offset, size, outBlob);
 }
 
 const DeviceInfo& DebugDevice::getDeviceInfo() const
@@ -585,7 +569,12 @@ Result DebugDevice::waitForFences(
     GfxCount fenceCount, IFence** fences, uint64_t* values , bool waitForAll, uint64_t timeout)
 {
     SLANG_GFX_API_FUNC;
-    return baseObject->waitForFences(fenceCount, fences, values, waitForAll, timeout);
+    ShortList<IFence*> innerFences;
+    for (GfxCount i = 0; i < fenceCount; i++)
+    {
+        innerFences.add(getInnerObj(fences[i]));
+    }
+    return baseObject->waitForFences(fenceCount, innerFences.getArrayView().getBuffer(), values, waitForAll, timeout);
 }
 
 Result DebugDevice::getTextureAllocationInfo(
diff --git a/tools/gfx/debug-layer/debug-shader-object.cpp b/tools/gfx/debug-layer/debug-shader-object.cpp
index e6644a6f6..036e743ec 100644
--- a/tools/gfx/debug-layer/debug-shader-object.cpp
+++ b/tools/gfx/debug-layer/debug-shader-object.cpp
@@ -86,25 +86,25 @@ Result DebugShaderObject::getObject(ShaderOffset const& offset, IShaderObject**
 Result DebugShaderObject::setObject(ShaderOffset const& offset, IShaderObject* object)
 {
     SLANG_GFX_API_FUNC;
-    auto objectImpl = static_cast<DebugShaderObject*>(object);
+    auto objectImpl = getDebugObj(object);
     m_objects[ShaderOffsetKey{offset}] = objectImpl;
-    return baseObject->setObject(offset, objectImpl->baseObject.get());
+    return baseObject->setObject(offset, getInnerObj(object));
 }
 
 Result DebugShaderObject::setResource(ShaderOffset const& offset, IResourceView* resourceView)
 {
     SLANG_GFX_API_FUNC;
-    auto viewImpl = static_cast<DebugResourceView*>(resourceView);
+    auto viewImpl = getDebugObj(resourceView);
     m_resources[ShaderOffsetKey{offset}] = viewImpl;
-    return baseObject->setResource(offset, viewImpl->baseObject.get());
+    return baseObject->setResource(offset, getInnerObj(resourceView));
 }
 
 Result DebugShaderObject::setSampler(ShaderOffset const& offset, ISamplerState* sampler)
 {
     SLANG_GFX_API_FUNC;
-    auto samplerImpl = static_cast<DebugSamplerState*>(sampler);
+    auto samplerImpl = getDebugObj(sampler);
     m_samplers[ShaderOffsetKey{offset}] = samplerImpl;
-    return baseObject->setSampler(offset, samplerImpl->baseObject.get());
+    return baseObject->setSampler(offset, getInnerObj(sampler));
 }
 
 Result DebugShaderObject::setCombinedTextureSampler(
@@ -113,12 +113,12 @@ Result DebugShaderObject::setCombinedTextureSampler(
     ISamplerState* sampler)
 {
     SLANG_GFX_API_FUNC;
-    auto samplerImpl = static_cast<DebugSamplerState*>(sampler);
+    auto samplerImpl = getDebugObj(sampler);
     m_samplers[ShaderOffsetKey{offset}] = samplerImpl;
-    auto viewImpl = static_cast<DebugResourceView*>(textureView);
+    auto viewImpl = getDebugObj(textureView);
     m_resources[ShaderOffsetKey{offset}] = viewImpl;
     return baseObject->setCombinedTextureSampler(
-        offset, viewImpl->baseObject.get(), samplerImpl->baseObject.get());
+        offset, getInnerObj(viewImpl), getInnerObj(sampler));
 }
 
 Result DebugShaderObject::setSpecializationArgs(