[gfx] Metal graphics support (#4324)

* fix double semicolons

* fix another double semicolon

* wait for init data upload

* remove obsolete setData

* refactor swapchain to work on virtual back buffers

* buffer/texture use breakable device reference

* refactor input layout

* create render command encoder

* add todo

* refactor framebuffer layout

* refactor framebuffer

* refactor shader program

* translatePrimitiveType

* add more translate functions

* refactor framebuffer

* refactor render pass

* implement graphics pipeline state

* add depth stencil state

* initial render command encoder support

* comment

1 files changed, 131 insertions, 110 deletions

diff --git a/tools/gfx/metal/metal-command-encoder.cpp b/tools/gfx/metal/metal-command-encoder.cpp
index f1e834400..3e0fbd1f6 100644
--- a/tools/gfx/metal/metal-command-encoder.cpp
+++ b/tools/gfx/metal/metal-command-encoder.cpp
@@ -209,52 +209,59 @@ void ResourceCommandEncoder::endDebugEvent()
 
 void RenderCommandEncoder::beginPass(IRenderPassLayout* renderPass, IFramebuffer* framebuffer)
 {
-    FramebufferImpl* fb = static_cast<FramebufferImpl*>(framebuffer);
-    if (fb == nullptr)
+    m_renderPassLayout = static_cast<RenderPassLayoutImpl*>(renderPass);
+    m_framebuffer = static_cast<FramebufferImpl*>(framebuffer);
+    if (!m_framebuffer)
     {
+        // TODO use empty framebuffer
         return;
     }
-    RenderPassLayoutImpl* renderPassLayoutImpl = static_cast<RenderPassLayoutImpl*>(renderPass);
 
-    MTL::RenderPassDescriptor* rpd = renderPassLayoutImpl->m_renderPassDesc->copy();
+    // Create a copy of the render pass descriptor and fill in remaining information.
+    m_renderPassDesc = NS::TransferPtr(m_renderPassLayout->m_renderPassDesc->copy());
 
-    if (rpd->depthAttachment() && false)
+    m_renderPassDesc->setRenderTargetWidth(m_framebuffer->m_width);
+    m_renderPassDesc->setRenderTargetHeight(m_framebuffer->m_height);
+
+    for (Index i = 0; i < m_framebuffer->m_renderTargetViews.getCount(); ++i)
     {
-        TextureResourceViewImpl* depthView = static_cast<TextureResourceViewImpl*>(fb->depthStencilView.get());
-        rpd->depthAttachment()->setTexture(depthView->m_texture->m_texture.get());
+        TextureResourceViewImpl* renderTargetView = m_framebuffer->m_renderTargetViews[i];
+        MTL::RenderPassColorAttachmentDescriptor* colorAttachment = m_renderPassDesc->colorAttachments()->object(i);
+        colorAttachment->setTexture(renderTargetView->m_texture->m_texture.get());
+        colorAttachment->setLevel(renderTargetView->m_desc.subresourceRange.mipLevel);
+        colorAttachment->setSlice(renderTargetView->m_desc.subresourceRange.baseArrayLayer);
     }
-    const int colorTargetCount = fb->renderTargetViews.getCount();
-    for (int i = 0; i < colorTargetCount; ++i)
+
+    if (m_framebuffer->m_depthStencilView)
     {
-        TextureResourceViewImpl* texView = static_cast<TextureResourceViewImpl*>(fb->renderTargetViews[i].get());
-        MTL::Texture* tex = nullptr;
-        assert(texView->m_texture);
-        tex = texView->m_texture->m_texture.get();
-        rpd->colorAttachments()->object(i)->setTexture(tex);
-        rpd->colorAttachments()->object(i)->setClearColor(MTL::ClearColor(0.2, 0.4, 0.9, 1.0));
+        TextureResourceViewImpl* depthStencilView = m_framebuffer->m_depthStencilView.get();
+        MTL::PixelFormat pixelFormat = MetalUtil::translatePixelFormat(depthStencilView->m_desc.format);
+        if (MetalUtil::isDepthFormat(pixelFormat))
+        {
+            MTL::RenderPassDepthAttachmentDescriptor* depthAttachment = m_renderPassDesc->depthAttachment();
+            depthAttachment->setTexture(depthStencilView->m_texture->m_texture.get());
+            depthAttachment->setLevel(depthStencilView->m_desc.subresourceRange.mipLevel);
+            depthAttachment->setSlice(depthStencilView->m_desc.subresourceRange.baseArrayLayer);
+        }
+        if (MetalUtil::isStencilFormat(pixelFormat))
+        {
+            MTL::RenderPassStencilAttachmentDescriptor* stencilAttachment = m_renderPassDesc->stencilAttachment();
+            stencilAttachment->setTexture(depthStencilView->m_texture->m_texture.get());
+            stencilAttachment->setLevel(depthStencilView->m_desc.subresourceRange.mipLevel);
+            stencilAttachment->setSlice(depthStencilView->m_desc.subresourceRange.baseArrayLayer);
+        }
     }
-    rpd->setRenderTargetWidth(fb->m_width);
-    rpd->setRenderTargetHeight(fb->m_height);
-
-    m_encoder = m_metalCommandBuffer->renderCommandEncoder(rpd);
 }
 
 void RenderCommandEncoder::endEncoding()
 {
-    m_encoder->endEncoding();
+    PipelineCommandEncoder::endEncodingImpl();
 }
 
 Result RenderCommandEncoder::bindPipeline(
     IPipelineState* pipelineState, IShaderObject** outRootObject)
 {
-    m_currentPipeline = static_cast<PipelineStateImpl*>(pipelineState);
-    // Initialize the root object
-    SLANG_RETURN_ON_FAIL(m_commandBuffer->m_rootObject.init(m_commandBuffer->m_device,
-        m_currentPipeline->getProgram<ShaderProgramImpl>()->m_rootObjectLayout));
-    *outRootObject = &m_commandBuffer->m_rootObject;
-    //if (pPipelineState->m_renderState == nullptr) return SLANG_ERROR_INVALID_PARAMETER;
-    //m_encoder->setRenderPipelineState(pPipelineState->m_renderState);
-    return SLANG_OK;
+    return setPipelineStateImpl(pipelineState, outRootObject);
 }
 
 Result RenderCommandEncoder::bindPipelineWithRootObject(
@@ -265,44 +272,37 @@ Result RenderCommandEncoder::bindPipelineWithRootObject(
 
 void RenderCommandEncoder::setViewports(GfxCount count, const Viewport* viewports)
 {
-    static const int kMaxViewports = 8; // TODO: base on device caps
-    assert(count <= kMaxViewports);
-
     m_viewports.setCount(count);
     for (GfxIndex i = 0; i < count; ++i)
     {
-        const auto& inViewport = viewports[i];
-        auto& metalViewport = m_viewports[i];
-        metalViewport.height = inViewport.extentY;
-        metalViewport.width = inViewport.extentX;
-        metalViewport.originX = inViewport.originX;
-        metalViewport.originY = inViewport.originY;
-        metalViewport.znear = inViewport.minZ;
-        metalViewport.zfar = inViewport.maxZ;
+        const auto& viewport = viewports[i];
+        auto& mtlViewport = m_viewports[i];
+        mtlViewport.originX = viewport.originX;
+        mtlViewport.originY = viewport.originY;
+        mtlViewport.width = viewport.extentX;
+        mtlViewport.height = viewport.extentY;
+        mtlViewport.znear = viewport.minZ;
+        mtlViewport.zfar = viewport.maxZ;
     }
-    m_encoder->setViewports(m_viewports.begin(), count);
 }
 
 void RenderCommandEncoder::setScissorRects(GfxCount count, const ScissorRect* rects)
 {
-    static const int kMaxScissorRects = 9; // TODO:
-    assert(count < kMaxScissorRects);
-
     m_scissorRects.setCount(count);
     for (GfxIndex i = 0; i < count; ++i)
     {
-        const auto& inRect = rects[i];
-        auto& metalRect = m_scissorRects[i];
-        metalRect.height = inRect.maxX - inRect.minX;
-        metalRect.width = inRect.maxY - inRect.minY;
-        metalRect.x = inRect.minX;
-        metalRect.y = inRect.minY;
+        const auto& rect = rects[i];
+        auto& mtlRect = m_scissorRects[i];
+        mtlRect.x = rect.minX;
+        mtlRect.y = rect.minY;
+        mtlRect.width = rect.maxX - rect.minX;
+        mtlRect.height = rect.maxY - rect.minY;
     }
-    m_encoder->setScissorRects(m_scissorRects.begin(), count);
 }
 
 void RenderCommandEncoder::setPrimitiveTopology(PrimitiveTopology topology)
 {
+    m_primitiveType = MetalUtil::translatePrimitiveType(topology);
 }
 
 void RenderCommandEncoder::setVertexBuffers(
@@ -311,79 +311,104 @@ void RenderCommandEncoder::setVertexBuffers(
     IBufferResource* const* buffers,
     const Offset* offsets)
 {
-    for (GfxIndex i = 0; i < GfxIndex(slotCount); i++)
+    Index count = Math::Max(m_vertexBuffers.getCount(), Index(startSlot + slotCount));
+    m_vertexBuffers.setCount(count);
+    m_vertexBufferOffsets.setCount(count);
+
+    for (Index i = 0; i < Index(slotCount); i++)
     {
-        GfxIndex slotIndex = startSlot + i;
-        BufferResourceImpl* buffer = static_cast<BufferResourceImpl*>(buffers[i]);
-        if (buffer)
-        {
-            MTL::Buffer* vertexBuffers = {buffer->m_buffer.get()};
-            m_encoder->setVertexBuffer(buffer->m_buffer.get(), offsets[i], slotIndex);
-            // ...
-        }
+        Index slotIndex = startSlot + i;
+        m_vertexBuffers[slotIndex] = static_cast<BufferResourceImpl*>(buffers[i])->m_buffer.get();
+        m_vertexBufferOffsets[slotIndex] = offsets[i];
     }
 }
 
 void RenderCommandEncoder::setIndexBuffer(
     IBufferResource* buffer, Format indexFormat, Offset offset)
 {
+    m_indexBuffer = static_cast<BufferResourceImpl*>(buffer)->m_buffer.get();
+    m_indexBufferOffset = offset;
+
+    switch (indexFormat)
+    {
+    case Format::R16_UINT:
+        m_indexBufferType = MTL::IndexTypeUInt16;
+        break;
+    case Format::R32_UINT:
+        m_indexBufferType = MTL::IndexTypeUInt32;
+        break;
+    default:
+        assert(!"unsupported index format");
+    }
+}
+
+void RenderCommandEncoder::setStencilReference(uint32_t referenceValue)
+{
+    m_stencilReferenceValue = referenceValue;
 }
 
-Result RenderCommandEncoder::prepareDraw()
+Result RenderCommandEncoder::setSamplePositions(
+    GfxCount samplesPerPixel, GfxCount pixelCount, const SamplePosition* samplePositions)
+{
+    return SLANG_E_NOT_AVAILABLE;
+}
+
+Result RenderCommandEncoder::prepareDraw(MTL::RenderCommandEncoder*& encoder)
 {
-    // Bind render state, including JIT pipeline state object creation
     auto pipeline = static_cast<PipelineStateImpl*>(m_currentPipeline.Ptr());
-    if (!pipeline)
+    pipeline->ensureAPIPipelineStateCreated();
+
+    encoder = m_commandBuffer->getMetalRenderCommandEncoder(m_renderPassDesc.get());
+    encoder->setRenderPipelineState(pipeline->m_renderPipelineState.get());
+    // TODO only vertex stage bindings are set for now
+    VertexBindingContext bindingContext;
+    bindingContext.init(m_commandBuffer->m_device, encoder);
+    auto program = static_cast<ShaderProgramImpl*>(m_currentPipeline->m_program.get());
+    m_commandBuffer->m_rootObject.bindAsRoot(&bindingContext, program->m_rootObjectLayout);
+
+    for (Index i = 0; i < m_vertexBuffers.getCount(); ++i)
     {
-        return SLANG_FAIL;
+        encoder->setVertexBuffer(m_vertexBuffers[i], m_vertexBufferOffsets[i], m_currentPipeline->m_vertexBufferOffset + i);
     }
-    // TODO: specialization, binding, ...
-    SLANG_RETURN_ON_FAIL(pipeline->ensureAPIPipelineStateCreated());
-    return SLANG_OK;
-}
 
-static Result translatePrimitiveType(gfx::PrimitiveType primType, MTL::PrimitiveType& mtlType)
-{
-    switch (primType) 
+    encoder->setViewports(m_viewports.getArrayView().getBuffer(), m_viewports.getCount());
+    encoder->setScissorRects(m_scissorRects.getArrayView().getBuffer(), m_scissorRects.getCount());
+
+    const RasterizerDesc& rasterDesc = pipeline->desc.graphics.rasterizer;
+    const DepthStencilDesc& depthStencilDesc = pipeline->desc.graphics.depthStencil;
+    encoder->setFrontFacingWinding(MetalUtil::translateWinding(rasterDesc.frontFace));
+    encoder->setCullMode(MetalUtil::translateCullMode(rasterDesc.cullMode));
+    encoder->setDepthClipMode(rasterDesc.depthClipEnable ? MTL::DepthClipModeClip : MTL::DepthClipModeClamp); // TODO correct?
+    encoder->setDepthBias(rasterDesc.depthBias, rasterDesc.slopeScaledDepthBias, rasterDesc.depthBiasClamp);
+    encoder->setTriangleFillMode(MetalUtil::translateTriangleFillMode(rasterDesc.fillMode));
+    // encoder->setBlendColor(); // not supported by gfx
+    if (m_framebuffer->m_depthStencilView)
     {
-    case PrimitiveType::Triangle:
-        mtlType = MTL::PrimitiveTypeTriangle;
-        break;
-    case PrimitiveType::Line:
-        mtlType = MTL::PrimitiveTypeLine;
-        break;
-    case PrimitiveType::Point:
-        mtlType = MTL::PrimitiveTypePoint;
-        break;
-    case PrimitiveType::Patch:
-    default:
-        return SLANG_E_INVALID_ARG;
+        encoder->setDepthStencilState(pipeline->m_depthStencilState.get());
     }
+    encoder->setStencilReferenceValue(m_stencilReferenceValue);
+
     return SLANG_OK;
 }
 
 Result RenderCommandEncoder::draw(GfxCount vertexCount, GfxIndex startVertex)
 {
-    SLANG_RETURN_ON_FAIL(prepareDraw());
-
-    MTL::PrimitiveType primType;
-    Result res = translatePrimitiveType(m_currentPipeline->desc.graphics.primitiveType, primType);
-    if (res != SLANG_OK)
-        return res;
-    m_encoder->drawPrimitives(primType, startVertex, vertexCount);
+    MTL::RenderCommandEncoder* encoder;
+    SLANG_RETURN_ON_FAIL(prepareDraw(encoder));
+    encoder->drawPrimitives(m_primitiveType, startVertex, vertexCount);
     return SLANG_OK;
 }
 
 Result RenderCommandEncoder::drawIndexed(
     GfxCount indexCount, GfxIndex startIndex, GfxIndex baseVertex)
 {
+    MTL::RenderCommandEncoder* encoder;
+    SLANG_RETURN_ON_FAIL(prepareDraw(encoder));
+    // TODO baseVertex is not supported by Metal
+    encoder->drawIndexedPrimitives(m_primitiveType, indexCount, m_indexBufferType, m_indexBuffer, m_indexBufferOffset);
     return SLANG_E_NOT_IMPLEMENTED;
 }
 
-void RenderCommandEncoder::setStencilReference(uint32_t referenceValue)
-{
-}
-
 Result RenderCommandEncoder::drawIndirect(
     GfxCount maxDrawCount,
     IBufferResource* argBuffer,
@@ -404,20 +429,16 @@ Result RenderCommandEncoder::drawIndexedIndirect(
     return SLANG_E_NOT_IMPLEMENTED;
 }
 
-Result RenderCommandEncoder::setSamplePositions(
-    GfxCount samplesPerPixel, GfxCount pixelCount, const SamplePosition* samplePositions)
-{
-    return SLANG_E_NOT_AVAILABLE;
-}
-
 Result RenderCommandEncoder::drawInstanced(
     GfxCount vertexCount,
     GfxCount instanceCount,
     GfxIndex startVertex,
     GfxIndex startInstanceLocation)
 {
-    SLANG_RETURN_ON_FAIL(prepareDraw());
-    return SLANG_E_NOT_IMPLEMENTED;
+    MTL::RenderCommandEncoder* encoder;
+    SLANG_RETURN_ON_FAIL(prepareDraw(encoder));
+    encoder->drawPrimitives(m_primitiveType, startVertex, vertexCount, instanceCount, startInstanceLocation);
+    return SLANG_OK;
 }
 
 Result RenderCommandEncoder::drawIndexedInstanced(
@@ -427,13 +448,14 @@ Result RenderCommandEncoder::drawIndexedInstanced(
     GfxIndex baseVertexLocation,
     GfxIndex startInstanceLocation)
 {
-    SLANG_RETURN_ON_FAIL(prepareDraw());
-    return SLANG_E_NOT_IMPLEMENTED;
+    MTL::RenderCommandEncoder* encoder;
+    SLANG_RETURN_ON_FAIL(prepareDraw(encoder));
+    encoder->drawIndexedPrimitives(m_primitiveType, indexCount, m_indexBufferType, m_indexBuffer, startIndexLocation, instanceCount, baseVertexLocation, startIndexLocation);
+    return SLANG_OK;
 }
 
 Result RenderCommandEncoder::drawMeshTasks(int x, int y, int z)
 {
-    SLANG_RETURN_ON_FAIL(prepareDraw());
     return SLANG_E_NOT_IMPLEMENTED;
 }
 
@@ -459,16 +481,15 @@ Result ComputeCommandEncoder::dispatchCompute(int x, int y, int z)
     auto pipeline = static_cast<PipelineStateImpl*>(m_currentPipeline.Ptr());
     pipeline->ensureAPIPipelineStateCreated();
 
-    auto metalComputeCommandEncoder = m_commandBuffer->getMetalComputeCommandEncoder();
-    metalComputeCommandEncoder->setComputePipelineState(pipeline->m_computePipelineState.get());
+    MTL::ComputeCommandEncoder* encoder = m_commandBuffer->getMetalComputeCommandEncoder();
+    encoder->setComputePipelineState(pipeline->m_computePipelineState.get());
     ComputeBindingContext bindingContext;
-    bindingContext.init(m_commandBuffer->m_device, metalComputeCommandEncoder);
+    bindingContext.init(m_commandBuffer->m_device, encoder);
     auto program = static_cast<ShaderProgramImpl*>(m_currentPipeline->m_program.get());
     m_commandBuffer->m_rootObject.bindAsRoot(&bindingContext, program->m_rootObjectLayout);
-    metalComputeCommandEncoder->dispatchThreadgroups(MTL::Size(x, y, z), pipeline->m_threadGroupSize);
+    encoder->dispatchThreadgroups(MTL::Size(x, y, z), pipeline->m_threadGroupSize);
 
-    // Also create descriptor sets based on the given pipeline layout
-    return SLANG_E_NOT_IMPLEMENTED;
+    return SLANG_OK;
 }
 
 Result ComputeCommandEncoder::dispatchComputeIndirect(IBufferResource* argBuffer, Offset offset)