Changes for vkd3d proton (#2813)

* Add some caches to .gitignore

* Remove appendWideChars

Use String::toWString instead

* s/Sleep/sleepCurrentThread

* formatting

* Expand set of shared libraries which have buggy dlclose

Work around
https://github.com/microsoft/DirectXShaderCompiler/issues/5119 and
https://github.com/doitsujin/dxvk/issues/3330 libdxcompiler.so invokes
UB on dlclose, the dxvk libs break GDB when closed

* Add assert for specialization failure on DX11

As a band aid for https://github.com/shader-slang/slang/issues/2805

* More fine grained selection of directx features

9 files changed, 75 insertions, 44 deletions

diff --git a/.gitignore b/.gitignore
index efbfbde24..549da7aa0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -77,3 +77,6 @@ build/**/*.log
 *.dll
 *.dxil
 /tests/serialization/*.map
+
+.cache
+vkd3d-proton.cache
diff --git a/slang.h b/slang.h
index 404d2ea23..732402441 100644
--- a/slang.h
+++ b/slang.h
@@ -166,6 +166,30 @@ Any platforms not detected by the above logic are now now explicitly zeroed out.
 #define SLANG_APPLE_FAMILY (SLANG_IOS || SLANG_OSX)                  /* equivalent to #if __APPLE__ */
 #define SLANG_UNIX_FAMILY (SLANG_LINUX_FAMILY || SLANG_APPLE_FAMILY) /* shortcut for unix/posix platforms */
 
+/* Macros concerning DirectX */
+#if SLANG_WINDOWS_FAMILY
+#    define SLANG_ENABLE_DIRECTX 1
+#    define SLANG_ENABLE_DXGI_DEBUG 1
+#    define SLANG_ENABLE_FXC 1
+#    define SLANG_ENABLE_PIX 1
+#    define SLANG_ENABLE_DXVK 0
+#    define SLANG_ENABLE_VKD3D_PROTON 0
+#elif SLANG_LINUX_FAMILY
+#    define SLANG_ENABLE_DIRECTX 0
+#    define SLANG_ENABLE_DXGI_DEBUG 0
+#    define SLANG_ENABLE_FXC 0
+#    define SLANG_ENABLE_PIX 0
+#    define SLANG_ENABLE_DXVK 1
+#    define SLANG_ENABLE_VKD3D_PROTON 1
+#else
+#    define SLANG_ENABLE_DIRECTX 0
+#    define SLANG_ENABLE_DXGI_DEBUG 0
+#    define SLANG_ENABLE_FXC 0
+#    define SLANG_ENABLE_PIX 0
+#    define SLANG_ENABLE_DXVK 0
+#    define SLANG_ENABLE_VKD3D_PROTON 0
+#endif
+
 /* Macro for declaring if a method is no throw. Should be set before the return parameter. */
 #ifndef SLANG_NO_THROW
 #   if SLANG_WINDOWS_FAMILY && !defined(SLANG_DISABLE_EXCEPTIONS)
diff --git a/source/core/slang-platform.cpp b/source/core/slang-platform.cpp
index 4f9b0698c..9185559a2 100644
--- a/source/core/slang-platform.cpp
+++ b/source/core/slang-platform.cpp
@@ -173,13 +173,24 @@ SLANG_COMPILE_TIME_ASSERT(E_OUTOFMEMORY == SLANG_E_OUT_OF_MEMORY);
 /* static */SlangResult SharedLibrary::loadWithPlatformPath(char const* platformFileName, Handle& handleOut)
 {
     handleOut = nullptr;
-    const auto dxcName = "libdxcompiler";
-    const bool isDXC = strncmp(platformFileName, dxcName, strlen(dxcName)) == 0;
+    // Work around
+    // https://github.com/microsoft/DirectXShaderCompiler/issues/5119 and
+    // https://github.com/doitsujin/dxvk/issues/3330
+    // libdxcompiler.so invokes UB on dlclose, the dxvk libs break GDB when
+    // closed
+    const auto unclosableLibNames = {"libdxcompiler", "libdxvk_d3d11", "libdxvk_dxgi"};
+    bool isUnclosable = false;
+    for(auto n : unclosableLibNames)
+    {
+        if(strncmp(platformFileName, n, strlen(n)) == 0)
+        {
+            isUnclosable = true;
+            break;
+        }
+    }
     if (strlen(platformFileName) == 0)
         platformFileName = nullptr;
-    // Work around https://github.com/microsoft/DirectXShaderCompiler/issues/5119
-    // libdxcompiler.so invokes UB on dlclose
-    const auto mode = RTLD_NOW | RTLD_GLOBAL | (isDXC ? RTLD_NODELETE : 0);
+    const auto mode = RTLD_NOW | RTLD_GLOBAL | (isUnclosable ? RTLD_NODELETE : 0);
     void *h = dlopen(platformFileName, mode);
     if (!h)
     {
diff --git a/tools/gfx/d3d/d3d-util.cpp b/tools/gfx/d3d/d3d-util.cpp
index eece71c16..c5092ddbd 100644
--- a/tools/gfx/d3d/d3d-util.cpp
+++ b/tools/gfx/d3d/d3d-util.cpp
@@ -2,9 +2,11 @@
 #include "d3d-util.h"
 
 #include <d3d12.h>
-#include <d3dcompiler.h>
 #include <dxgi1_4.h>
 #include <dxgidebug.h>
+#if SLANG_ENABLE_FXC
+#include <d3dcompiler.h>
+#endif
 
 // We will use the C standard library just for printing error messages.
 #include <stdio.h>
@@ -423,6 +425,9 @@ bool D3DUtil::isTypeless(DXGI_FORMAT format)
     // shader bytecode as part of an offline process, rather than doing it
     // on-the-fly like this
     //
+#if !SLANG_ENABLE_FXC
+    return SLANG_E_NOT_IMPLEMENTED;
+#else
     static pD3DCompile compileFunc = nullptr;
     if (!compileFunc)
     {
@@ -476,35 +481,15 @@ bool D3DUtil::isTypeless(DXGI_FORMAT format)
     SLANG_RETURN_ON_FAIL(hr);
     shaderBlobOut.swap(shaderBlob);
     return SLANG_OK;
-}
-
-/* static */void D3DUtil::appendWideChars(const char* in, List<wchar_t>& out)
-{
-    size_t len = ::strlen(in);
-
-    const DWORD dwFlags = 0;
-    int outSize = ::MultiByteToWideChar(CP_UTF8, dwFlags, in, int(len), nullptr, 0);
-
-    if (outSize > 0)
-    {
-        const Index prevSize = out.getCount();
-        out.setCount(prevSize + len + 1);
-
-        WCHAR* dst = out.getBuffer() + prevSize;
-        ::MultiByteToWideChar(CP_UTF8, dwFlags, in, int(len), dst, outSize);
-        // Make null terminated
-        dst[outSize] = 0;
-        // Remove terminating 0 from array
-        out.unsafeShrinkToCount(prevSize + outSize);
-    }
+#endif // SLANG_ENABLE_FXC
 }
 
 /* static */SharedLibrary::Handle D3DUtil::getDxgiModule()
 {
-#if SLANG_WINDOWS_FAMILY
-    const char* const libPath = "dxgi";
-#else
+#if SLANG_ENABLE_DXVK
     const char* const libPath = "dxvk_dxgi";
+#else
+    const char* const libPath = "dxgi";
 #endif
     static SharedLibrary::Handle s_dxgiModule = [&](){
         SharedLibrary::Handle h = nullptr;
@@ -849,6 +834,7 @@ D3D12_RESOURCE_STATES D3DUtil::getResourceState(ResourceState state)
 {
     static IDXGIDebug* dxgiDebug = nullptr;
 
+#if SLANG_ENABLE_DXGI_DEBUG
     if (!dxgiDebug)
     {
         HMODULE debugModule = LoadLibraryA("dxgidebug.dll");
@@ -861,6 +847,7 @@ D3D12_RESOURCE_STATES D3DUtil::getResourceState(ResourceState state)
             }
         }
     }
+#endif
 
     if (dxgiDebug)
     {
diff --git a/tools/gfx/d3d/d3d-util.h b/tools/gfx/d3d/d3d-util.h
index 1869d7de6..a35928f47 100644
--- a/tools/gfx/d3d/d3d-util.h
+++ b/tools/gfx/d3d/d3d-util.h
@@ -76,10 +76,6 @@ class D3DUtil
         /// Returns number of bits used for color channel for format (for channels with multiple sizes, returns smallest ie RGB565 -> 5)
     static Int getNumColorChannelBits(DXGI_FORMAT fmt);
 
-        /// Append text in in, into wide char array
-    static void appendWideChars(const char* in, Slang::List<wchar_t>& out);
-
-
     static SlangResult createFactory(DeviceCheckFlags flags, Slang::ComPtr<IDXGIFactory>& outFactory);
 
         /// Get the dxgiModule
diff --git a/tools/gfx/d3d11/d3d11-device.cpp b/tools/gfx/d3d11/d3d11-device.cpp
index e6a348dc1..4ab26725e 100644
--- a/tools/gfx/d3d11/d3d11-device.cpp
+++ b/tools/gfx/d3d11/d3d11-device.cpp
@@ -47,10 +47,10 @@ SlangResult DeviceImpl::initialize(const Desc& desc)
 
     // Rather than statically link against D3D, we load it dynamically.
     SharedLibrary::Handle d3dModule;
-#if SLANG_WINDOWS_FAMILY
-    const char* libName = "d3d11";
-#else
+#if SLANG_ENABLE_DXVK
     const char* libName = "dxvk_d3d11";
+#else
+    const char* libName = "d3d11";
 #endif
     if (SLANG_FAILED(SharedLibrary::load(libName, d3dModule)))
     {
@@ -1428,6 +1428,8 @@ void DeviceImpl::bindRootShaderObject(IShaderObject* shaderObject)
 {
     RootShaderObjectImpl* rootShaderObjectImpl = static_cast<RootShaderObjectImpl*>(shaderObject);
     RefPtr<PipelineStateBase> specializedPipeline;
+    // TODO: Do something less crappy than just asserting on failure here
+    SLANG_ASSERT_VOID_ON_FAIL(maybeSpecializePipeline(m_currentPipelineState, rootShaderObjectImpl, specializedPipeline));
     maybeSpecializePipeline(m_currentPipelineState, rootShaderObjectImpl, specializedPipeline);
     PipelineStateImpl* specializedPipelineImpl = static_cast<PipelineStateImpl*>(specializedPipeline.Ptr());
     setPipelineState(specializedPipelineImpl);
diff --git a/tools/gfx/d3d11/d3d11-query.cpp b/tools/gfx/d3d11/d3d11-query.cpp
index 37b8ddc25..c917ccbca 100644
--- a/tools/gfx/d3d11/d3d11-query.cpp
+++ b/tools/gfx/d3d11/d3d11-query.cpp
@@ -1,6 +1,8 @@
 // d3d11-query.cpp
 #include "d3d11-query.h"
 
+#include "core/slang-process.h"
+
 namespace gfx
 {
 
@@ -39,7 +41,7 @@ SLANG_NO_THROW Result SLANG_MCALL QueryPoolImpl::getResult(
     while (S_OK != m_device->m_immediateContext->GetData(
         m_device->m_disjointQuery, &disjointData, sizeof(D3D11_QUERY_DATA_TIMESTAMP_DISJOINT), 0))
     {
-        Sleep(1);
+        Process::sleepCurrentThread(1);
     }
     m_device->m_info.timestampFrequency = disjointData.Frequency;
 
diff --git a/tools/gfx/d3d12/d3d12-device.cpp b/tools/gfx/d3d12/d3d12-device.cpp
index ad71388c4..f00b4d8e0 100644
--- a/tools/gfx/d3d12/d3d12-device.cpp
+++ b/tools/gfx/d3d12/d3d12-device.cpp
@@ -463,6 +463,7 @@ Result DeviceImpl::initialize(const Desc& desc)
         return SLANG_FAIL;
     }
 
+#if SLANG_ENABLE_PIX
     HMODULE pixModule = LoadLibraryW(L"WinPixEventRuntime.dll");
     if (pixModule)
     {
@@ -471,6 +472,7 @@ Result DeviceImpl::initialize(const Desc& desc)
         m_EndEventOnCommandList =
             (PFN_EndEventOnCommandList)GetProcAddress(pixModule, "PIXEndEventOnCommandList");
     }
+#endif
 
     if (ENABLE_DEBUG_LAYER || isGfxDebugLayerEnabled())
     {
diff --git a/tools/gfx/render.cpp b/tools/gfx/render.cpp
index 2f10fa67c..11aacc741 100644
--- a/tools/gfx/render.cpp
+++ b/tools/gfx/render.cpp
@@ -247,13 +247,15 @@ extern "C"
 
         switch (type)
         {
-#if SLANG_WINDOWS_FAMILY
+#if SLANG_ENABLE_DIRECTX
         case DeviceType::DirectX11:
             SLANG_RETURN_ON_FAIL(getD3D11Adapters(adapters));
             break;
         case DeviceType::DirectX12:
             SLANG_RETURN_ON_FAIL(getD3D12Adapters(adapters));
             break;
+#endif
+#if SLANG_WINDOWS_FAMILY
         case DeviceType::OpenGl:
             return SLANG_E_NOT_IMPLEMENTED;
 #endif
@@ -283,7 +285,7 @@ extern "C"
     {
         switch (desc->deviceType)
         {
-#if SLANG_WINDOWS_FAMILY
+#if SLANG_ENABLE_DIRECTX
         case DeviceType::DirectX11:
             {
                 return createD3D11Device(desc, outDevice);
@@ -292,6 +294,8 @@ extern "C"
             {
                 return createD3D12Device(desc, outDevice);
             }
+#endif
+#if SLANG_WINDOWS_FAMILY
         case DeviceType::OpenGl:
             {
                 return createGLDevice(desc, outDevice);
@@ -326,9 +330,9 @@ extern "C"
         }
 #endif
         case DeviceType::CUDA:
-        {
-            return createCUDADevice(desc, outDevice);
-        }
+            {
+                return createCUDADevice(desc, outDevice);
+            }
         case DeviceType::CPU:
             {
                 return createCPUDevice(desc, outDevice);
@@ -361,7 +365,7 @@ extern "C"
     SLANG_GFX_API SlangResult SLANG_MCALL
         gfxReportLiveObjects()
     {
-#if SLANG_WINDOWS_FAMILY
+#if SLANG_ENABLE_DIRECTX
         SLANG_RETURN_ON_FAIL(reportD3DLiveObjects());
 #endif
         return SLANG_OK;