switch to vkCreateMetalSurfaceEXT and create metal layer in swapchain (#3627)

* switch to vkCreateMetalSurfaceEXT and create metal layer in swapchain

* fix window content size on macos

---------

Co-authored-by: Yong He <yonghe@outlook.com>

11 files changed, 51 insertions, 42 deletions

diff --git a/premake5.lua b/premake5.lua
index 38fe65889..19fc9c6fa 100644
--- a/premake5.lua
+++ b/premake5.lua
@@ -1208,7 +1208,7 @@ tool "gfx"
     end
 
     if os.target() == "macosx" then
-        links { "Cocoa.framework" }
+        links { "Cocoa.framework", "QuartzCore.framework" }
     end
 
     if enableXlib then
diff --git a/slang-gfx.h b/slang-gfx.h
index fa087b5fa..0ce61f9d2 100644
--- a/slang-gfx.h
+++ b/slang-gfx.h
@@ -1460,7 +1460,7 @@ struct WindowHandle
     {
         Unknown,
         Win32Handle,
-        NSViewHandle,
+        NSWindowHandle,
         XLibHandle,
     };
     Type type;
@@ -1472,11 +1472,11 @@ struct WindowHandle
         handle.handleValues[0] = (intptr_t)(hwnd);
         return handle;
     }
-    static WindowHandle FromNSView(void* nsview)
+    static WindowHandle FromNSWindow(void* nswindow)
     {
         WindowHandle handle = {};
-        handle.type = WindowHandle::Type::NSViewHandle;
-        handle.handleValues[0] = (intptr_t)(nsview);
+        handle.type = WindowHandle::Type::NSWindowHandle;
+        handle.handleValues[0] = (intptr_t)(nswindow);
         return handle;
     }
     static WindowHandle FromXWindow(void* xdisplay, uint32_t xwindow)
diff --git a/tools/gfx/apple/cocoa-util.h b/tools/gfx/apple/cocoa-util.h
index 80467d8a4..565783ee9 100644
--- a/tools/gfx/apple/cocoa-util.h
+++ b/tools/gfx/apple/cocoa-util.h
@@ -5,7 +5,10 @@ namespace gfx {
 // Utility functions for Cocoa
 struct CocoaUtil {
 
-    static void getNSViewRectSize(void* nsview, int* widthOut, int* heightOut);
+    static void getNSWindowContentSize(void* nswindow, int* widthOut, int* heightOut);
+
+    static void* createMetalLayer(void* nswindow);
+    static void destroyMetalLayer(void* metalLayer);
 
 };
 
diff --git a/tools/gfx/apple/cocoa-util.mm b/tools/gfx/apple/cocoa-util.mm
index 45b1c3df0..29c3056a9 100644
--- a/tools/gfx/apple/cocoa-util.mm
+++ b/tools/gfx/apple/cocoa-util.mm
@@ -1,15 +1,31 @@
 #include "cocoa-util.h"
 
 #import <Cocoa/Cocoa.h>
+#import <QuartzCore/CAMetalLayer.h>
 
 namespace gfx {
 
-void CocoaUtil::getNSViewRectSize(void* nsview, int* widthOut, int* heightOut)
+void CocoaUtil::getNSWindowContentSize(void* nswindow, int* widthOut, int* heightOut)
 {
-    NSView* view = (NSView*)nsview;
-    NSRect rect = [view frame];
-    *widthOut = rect.size.width;
-    *heightOut = rect.size.height;
+    NSWindow* window = (NSWindow*)nswindow;
+    const NSRect contentRect = [window.contentView frame];
+    *widthOut = contentRect.size.width;
+    *heightOut = contentRect.size.height;
+}
+
+void* CocoaUtil::createMetalLayer(void* nswindow)
+{
+    CAMetalLayer *layer = [CAMetalLayer layer];
+    NSWindow* window = (NSWindow*)nswindow;
+    window.contentView.layer = layer;
+    window.contentView.wantsLayer = YES;
+    return layer;
+}
+
+void CocoaUtil::destroyMetalLayer(void* metalLayer)
+{
+    CAMetalLayer* layer = (CAMetalLayer*)metalLayer;
+    [layer release];
 }
 
 }
\ No newline at end of file
diff --git a/tools/gfx/vulkan/vk-api.h b/tools/gfx/vulkan/vk-api.h
index 3d3303efa..f7523eb7f 100644
--- a/tools/gfx/vulkan/vk-api.h
+++ b/tools/gfx/vulkan/vk-api.h
@@ -143,7 +143,7 @@ namespace gfx {
     /* */
 #elif SLANG_APPLE_FAMILY
 #   define VK_API_INSTANCE_PLATFORM_KHR_PROCS(x)          \
-    x(vkCreateMacOSSurfaceMVK) \
+    x(vkCreateMetalSurfaceEXT) \
     /* */
 #elif SLANG_ENABLE_XLIB
 #   define VK_API_INSTANCE_PLATFORM_KHR_PROCS(x)          \
diff --git a/tools/gfx/vulkan/vk-device.cpp b/tools/gfx/vulkan/vk-device.cpp
index 46023281c..886e1ae75 100644
--- a/tools/gfx/vulkan/vk-device.cpp
+++ b/tools/gfx/vulkan/vk-device.cpp
@@ -189,7 +189,7 @@ Result DeviceImpl::initVulkanInstanceAndDevice(
 #if SLANG_WINDOWS_FAMILY
             instanceExtensions.add(VK_KHR_WIN32_SURFACE_EXTENSION_NAME);
 #elif SLANG_APPLE_FAMILY
-            instanceExtensions.add(VK_MVK_MACOS_SURFACE_EXTENSION_NAME);
+            instanceExtensions.add(VK_EXT_METAL_SURFACE_EXTENSION_NAME);
 #elif defined(SLANG_ENABLE_XLIB)
 
             instanceExtensions.add(VK_KHR_XLIB_SURFACE_EXTENSION_NAME);
diff --git a/tools/gfx/vulkan/vk-module.h b/tools/gfx/vulkan/vk-module.h
index 9b8fe0ed3..4677bbe50 100644
--- a/tools/gfx/vulkan/vk-module.h
+++ b/tools/gfx/vulkan/vk-module.h
@@ -8,7 +8,7 @@
 #if SLANG_WINDOWS_FAMILY
 #   define VK_USE_PLATFORM_WIN32_KHR 1
 #elif SLANG_APPLE_FAMILY
-#   define VK_USE_PLATFORM_MACOS_MVK 1
+#   define VK_USE_PLATFORM_METAL_EXT 1
 #else
 #   define VK_USE_PLATFORM_XLIB_KHR 1
 #endif
diff --git a/tools/gfx/vulkan/vk-swap-chain.cpp b/tools/gfx/vulkan/vk-swap-chain.cpp
index 856fa2489..7580e01b3 100644
--- a/tools/gfx/vulkan/vk-swap-chain.cpp
+++ b/tools/gfx/vulkan/vk-swap-chain.cpp
@@ -40,7 +40,7 @@ void SwapchainImpl::getWindowSize(int* widthOut, int* heightOut) const
     *widthOut = rc.right - rc.left;
     *heightOut = rc.bottom - rc.top;
 #elif SLANG_APPLE_FAMILY
-    CocoaUtil::getNSViewRectSize((void*)m_windowHandle.handleValues[0], widthOut, heightOut);
+    CocoaUtil::getNSWindowContentSize((void*)m_windowHandle.handleValues[0], widthOut, heightOut);
 #elif defined(SLANG_ENABLE_XLIB)
     XWindowAttributes winAttr = {};
     XGetWindowAttributes(
@@ -182,6 +182,9 @@ SwapchainImpl::~SwapchainImpl()
         m_surface = VK_NULL_HANDLE;
     }
     m_renderer->m_api.vkDestroySemaphore(m_renderer->m_api.m_device, m_nextImageSemaphore, nullptr);
+#if SLANG_APPLE_FAMILY
+    CocoaUtil::destroyMetalLayer(m_metalLayer);
+#endif
 }
 
 Index SwapchainImpl::_indexOfFormat(List<VkSurfaceFormatKHR>& formatsIn, VkFormat format)
@@ -225,11 +228,12 @@ Result SwapchainImpl::init(DeviceImpl* renderer, const ISwapchain::Desc& desc, W
     SLANG_VK_RETURN_ON_FAIL(
         m_api->vkCreateWin32SurfaceKHR(m_api->m_instance, &surfaceCreateInfo, nullptr, &m_surface));
 #elif SLANG_APPLE_FAMILY
-    VkMacOSSurfaceCreateInfoMVK surfaceCreateInfo = {};
-    surfaceCreateInfo.sType = VK_STRUCTURE_TYPE_MACOS_SURFACE_CREATE_INFO_MVK;
-    surfaceCreateInfo.pView = (void*)window.handleValues[0];
+    m_metalLayer = CocoaUtil::createMetalLayer((void*)window.handleValues[0]);
+    VkMetalSurfaceCreateInfoEXT surfaceCreateInfo = {};
+    surfaceCreateInfo.sType = VK_STRUCTURE_TYPE_METAL_SURFACE_CREATE_INFO_EXT;
+    surfaceCreateInfo.pLayer = (CAMetalLayer*)m_metalLayer;
     SLANG_VK_RETURN_ON_FAIL(
-        m_api->vkCreateMacOSSurfaceMVK(m_api->m_instance, &surfaceCreateInfo, nullptr, &m_surface));
+        m_api->vkCreateMetalSurfaceEXT(m_api->m_instance, &surfaceCreateInfo, nullptr, &m_surface));
 #elif SLANG_ENABLE_XLIB
     VkXlibSurfaceCreateInfoKHR surfaceCreateInfo = {};
     surfaceCreateInfo.sType = VK_STRUCTURE_TYPE_XLIB_SURFACE_CREATE_INFO_KHR;
diff --git a/tools/gfx/vulkan/vk-swap-chain.h b/tools/gfx/vulkan/vk-swap-chain.h
index 2c0fe62ed..50878cd7b 100644
--- a/tools/gfx/vulkan/vk-swap-chain.h
+++ b/tools/gfx/vulkan/vk-swap-chain.h
@@ -34,6 +34,10 @@ public:
     VulkanApi* m_api;
     uint32_t m_currentImageIndex = 0;
     WindowHandle m_windowHandle;
+#if SLANG_APPLE_FAMILY
+    void* m_metalLayer;
+#endif
+
     void destroySwapchainAndImages();
 
     void getWindowSize(int* widthOut, int* heightOut) const;
diff --git a/tools/platform/apple/cocoa-window.mm b/tools/platform/apple/cocoa-window.mm
index 450087dd2..b985cb4f3 100644
--- a/tools/platform/apple/cocoa-window.mm
+++ b/tools/platform/apple/cocoa-window.mm
@@ -50,8 +50,6 @@ class CocoaPlatformWindow : public Window
 public:
     NSWindow* window;
     WindowDelegate* delegate;
-    ContentView* view;
-    CAMetalLayer* layer;
     bool shouldClose = false;
 
     CocoaPlatformWindow(const WindowDesc& desc);
@@ -516,26 +514,14 @@ CocoaPlatformWindow::CocoaPlatformWindow(const WindowDesc& desc)
     else if (desc.style == WindowStyle::FixedSize)
         [window setStyleMask:NSWindowStyleMaskTitled | NSWindowStyleMaskClosable];
 
-    // Allocate view
-    rect = [window backingAlignedRect:rect options:NSAlignAllEdgesOutward];
-    view = [[ContentView alloc] initWithPlatformWindow:this];
-    [view setHidden:NO];
-    [view setNeedsDisplay:YES];
-    [view setWantsLayer:YES];
-
     delegate = [[WindowDelegate alloc] initWithPlatformWindow:this];
     [window setDelegate:delegate];
-    [window setContentView:view];
 
     NSString* title = [NSString stringWithUTF8String:desc.title];
     [window setTitle:title];
 
     [window center];
     [window makeKeyAndOrderFront:nil];
-
-    // Setup layer
-    layer = [[CAMetalLayer alloc] init];
-    [view setLayer:layer];
 }
 
 CocoaPlatformWindow::~CocoaPlatformWindow()
@@ -564,13 +550,9 @@ void CocoaPlatformWindow::close()
 {
     [window release];
     [delegate release];
-    [view release];
-    [layer release];
 
     window = nil;
     delegate = nil;
-    view = nil;
-    layer = nil;
 }
 
 bool CocoaPlatformWindow::getFocused()
@@ -585,7 +567,7 @@ bool CocoaPlatformWindow::getVisible()
 
 WindowHandle CocoaPlatformWindow::getNativeHandle()
 {
-    return WindowHandle::fromNSView(view);
+    return WindowHandle::fromNSWindow(window);
 }
 
 void CocoaPlatformWindow::setText(Slang::String text)
diff --git a/tools/platform/window.h b/tools/platform/window.h
index 7f78b8af9..efbe139a7 100644
--- a/tools/platform/window.h
+++ b/tools/platform/window.h
@@ -97,7 +97,7 @@ struct WindowHandle
     {
         Unknown,
         Win32Handle,
-        NSViewHandle,
+        NSWindowHandle,
         XLibHandle,
     };
     Type type;
@@ -109,11 +109,11 @@ struct WindowHandle
         handle.handleValues[0] = (intptr_t)(hwnd);
         return handle;
     }
-    static WindowHandle fromNSView(void* nsview)
+    static WindowHandle fromNSWindow(void* nswindow)
     {
         WindowHandle handle = {};
-        handle.type = WindowHandle::Type::NSViewHandle;
-        handle.handleValues[0] = (intptr_t)(nsview);
+        handle.type = WindowHandle::Type::NSWindowHandle;
+        handle.handleValues[0] = (intptr_t)(nswindow);
         return handle;
     }
     static WindowHandle fromXWindow(void* xdisplay, uint32_t xwindow)