switch to vkCreateMetalSurfaceEXT and create metal layer in swapchain (#3627)

* switch to vkCreateMetalSurfaceEXT and create metal layer in swapchain

* fix window content size on macos

---------

Co-authored-by: Yong He <yonghe@outlook.com>

7 files changed, 41 insertions, 14 deletions

diff --git a/tools/gfx/apple/cocoa-util.h b/tools/gfx/apple/cocoa-util.h
index 80467d8a4..565783ee9 100644
--- a/tools/gfx/apple/cocoa-util.h
+++ b/tools/gfx/apple/cocoa-util.h
@@ -5,7 +5,10 @@ namespace gfx {
 // Utility functions for Cocoa
 struct CocoaUtil {
 
-    static void getNSViewRectSize(void* nsview, int* widthOut, int* heightOut);
+    static void getNSWindowContentSize(void* nswindow, int* widthOut, int* heightOut);
+
+    static void* createMetalLayer(void* nswindow);
+    static void destroyMetalLayer(void* metalLayer);
 
 };
 
diff --git a/tools/gfx/apple/cocoa-util.mm b/tools/gfx/apple/cocoa-util.mm
index 45b1c3df0..29c3056a9 100644
--- a/tools/gfx/apple/cocoa-util.mm
+++ b/tools/gfx/apple/cocoa-util.mm
@@ -1,15 +1,31 @@
 #include "cocoa-util.h"
 
 #import <Cocoa/Cocoa.h>
+#import <QuartzCore/CAMetalLayer.h>
 
 namespace gfx {
 
-void CocoaUtil::getNSViewRectSize(void* nsview, int* widthOut, int* heightOut)
+void CocoaUtil::getNSWindowContentSize(void* nswindow, int* widthOut, int* heightOut)
 {
-    NSView* view = (NSView*)nsview;
-    NSRect rect = [view frame];
-    *widthOut = rect.size.width;
-    *heightOut = rect.size.height;
+    NSWindow* window = (NSWindow*)nswindow;
+    const NSRect contentRect = [window.contentView frame];
+    *widthOut = contentRect.size.width;
+    *heightOut = contentRect.size.height;
+}
+
+void* CocoaUtil::createMetalLayer(void* nswindow)
+{
+    CAMetalLayer *layer = [CAMetalLayer layer];
+    NSWindow* window = (NSWindow*)nswindow;
+    window.contentView.layer = layer;
+    window.contentView.wantsLayer = YES;
+    return layer;
+}
+
+void CocoaUtil::destroyMetalLayer(void* metalLayer)
+{
+    CAMetalLayer* layer = (CAMetalLayer*)metalLayer;
+    [layer release];
 }
 
 }
\ No newline at end of file
diff --git a/tools/gfx/vulkan/vk-api.h b/tools/gfx/vulkan/vk-api.h
index 3d3303efa..f7523eb7f 100644
--- a/tools/gfx/vulkan/vk-api.h
+++ b/tools/gfx/vulkan/vk-api.h
@@ -143,7 +143,7 @@ namespace gfx {
     /* */
 #elif SLANG_APPLE_FAMILY
 #   define VK_API_INSTANCE_PLATFORM_KHR_PROCS(x)          \
-    x(vkCreateMacOSSurfaceMVK) \
+    x(vkCreateMetalSurfaceEXT) \
     /* */
 #elif SLANG_ENABLE_XLIB
 #   define VK_API_INSTANCE_PLATFORM_KHR_PROCS(x)          \
diff --git a/tools/gfx/vulkan/vk-device.cpp b/tools/gfx/vulkan/vk-device.cpp
index 46023281c..886e1ae75 100644
--- a/tools/gfx/vulkan/vk-device.cpp
+++ b/tools/gfx/vulkan/vk-device.cpp
@@ -189,7 +189,7 @@ Result DeviceImpl::initVulkanInstanceAndDevice(
 #if SLANG_WINDOWS_FAMILY
             instanceExtensions.add(VK_KHR_WIN32_SURFACE_EXTENSION_NAME);
 #elif SLANG_APPLE_FAMILY
-            instanceExtensions.add(VK_MVK_MACOS_SURFACE_EXTENSION_NAME);
+            instanceExtensions.add(VK_EXT_METAL_SURFACE_EXTENSION_NAME);
 #elif defined(SLANG_ENABLE_XLIB)
 
             instanceExtensions.add(VK_KHR_XLIB_SURFACE_EXTENSION_NAME);
diff --git a/tools/gfx/vulkan/vk-module.h b/tools/gfx/vulkan/vk-module.h
index 9b8fe0ed3..4677bbe50 100644
--- a/tools/gfx/vulkan/vk-module.h
+++ b/tools/gfx/vulkan/vk-module.h
@@ -8,7 +8,7 @@
 #if SLANG_WINDOWS_FAMILY
 #   define VK_USE_PLATFORM_WIN32_KHR 1
 #elif SLANG_APPLE_FAMILY
-#   define VK_USE_PLATFORM_MACOS_MVK 1
+#   define VK_USE_PLATFORM_METAL_EXT 1
 #else
 #   define VK_USE_PLATFORM_XLIB_KHR 1
 #endif
diff --git a/tools/gfx/vulkan/vk-swap-chain.cpp b/tools/gfx/vulkan/vk-swap-chain.cpp
index 856fa2489..7580e01b3 100644
--- a/tools/gfx/vulkan/vk-swap-chain.cpp
+++ b/tools/gfx/vulkan/vk-swap-chain.cpp
@@ -40,7 +40,7 @@ void SwapchainImpl::getWindowSize(int* widthOut, int* heightOut) const
     *widthOut = rc.right - rc.left;
     *heightOut = rc.bottom - rc.top;
 #elif SLANG_APPLE_FAMILY
-    CocoaUtil::getNSViewRectSize((void*)m_windowHandle.handleValues[0], widthOut, heightOut);
+    CocoaUtil::getNSWindowContentSize((void*)m_windowHandle.handleValues[0], widthOut, heightOut);
 #elif defined(SLANG_ENABLE_XLIB)
     XWindowAttributes winAttr = {};
     XGetWindowAttributes(
@@ -182,6 +182,9 @@ SwapchainImpl::~SwapchainImpl()
         m_surface = VK_NULL_HANDLE;
     }
     m_renderer->m_api.vkDestroySemaphore(m_renderer->m_api.m_device, m_nextImageSemaphore, nullptr);
+#if SLANG_APPLE_FAMILY
+    CocoaUtil::destroyMetalLayer(m_metalLayer);
+#endif
 }
 
 Index SwapchainImpl::_indexOfFormat(List<VkSurfaceFormatKHR>& formatsIn, VkFormat format)
@@ -225,11 +228,12 @@ Result SwapchainImpl::init(DeviceImpl* renderer, const ISwapchain::Desc& desc, W
     SLANG_VK_RETURN_ON_FAIL(
         m_api->vkCreateWin32SurfaceKHR(m_api->m_instance, &surfaceCreateInfo, nullptr, &m_surface));
 #elif SLANG_APPLE_FAMILY
-    VkMacOSSurfaceCreateInfoMVK surfaceCreateInfo = {};
-    surfaceCreateInfo.sType = VK_STRUCTURE_TYPE_MACOS_SURFACE_CREATE_INFO_MVK;
-    surfaceCreateInfo.pView = (void*)window.handleValues[0];
+    m_metalLayer = CocoaUtil::createMetalLayer((void*)window.handleValues[0]);
+    VkMetalSurfaceCreateInfoEXT surfaceCreateInfo = {};
+    surfaceCreateInfo.sType = VK_STRUCTURE_TYPE_METAL_SURFACE_CREATE_INFO_EXT;
+    surfaceCreateInfo.pLayer = (CAMetalLayer*)m_metalLayer;
     SLANG_VK_RETURN_ON_FAIL(
-        m_api->vkCreateMacOSSurfaceMVK(m_api->m_instance, &surfaceCreateInfo, nullptr, &m_surface));
+        m_api->vkCreateMetalSurfaceEXT(m_api->m_instance, &surfaceCreateInfo, nullptr, &m_surface));
 #elif SLANG_ENABLE_XLIB
     VkXlibSurfaceCreateInfoKHR surfaceCreateInfo = {};
     surfaceCreateInfo.sType = VK_STRUCTURE_TYPE_XLIB_SURFACE_CREATE_INFO_KHR;
diff --git a/tools/gfx/vulkan/vk-swap-chain.h b/tools/gfx/vulkan/vk-swap-chain.h
index 2c0fe62ed..50878cd7b 100644
--- a/tools/gfx/vulkan/vk-swap-chain.h
+++ b/tools/gfx/vulkan/vk-swap-chain.h
@@ -34,6 +34,10 @@ public:
     VulkanApi* m_api;
     uint32_t m_currentImageIndex = 0;
     WindowHandle m_windowHandle;
+#if SLANG_APPLE_FAMILY
+    void* m_metalLayer;
+#endif
+
     void destroySwapchainAndImages();
 
     void getWindowSize(int* widthOut, int* heightOut) const;