Enable SLANG_ENABLE_DXIL_SUPPORT on non-Windows platforms (#2750)

* Enable SLANG_ENABLE_DXIL_SUPPORT on non-Windows platforms

This currently grabs the DXC headers from the system, rather than from external/dxc

We should make this consistent by either pulling in the Windows adapter from DXC into external/dxc or by making the Windows builds use <dxc/dxcapi.h>

* Update dxcapi and add DXC's WinAdapter

* Use our copy of dxcapi.h for non-windows DXC usage

* Only set -fms-extensions where necessary

* Work around dxc dlclose bug

* Neaten and comment dxc-compiler.cpp

2 files changed, 37 insertions, 17 deletions

diff --git a/source/compiler-core/slang-dxc-compiler.cpp b/source/compiler-core/slang-dxc-compiler.cpp
index a39503b13..a275143f8 100644
--- a/source/compiler-core/slang-dxc-compiler.cpp
+++ b/source/compiler-core/slang-dxc-compiler.cpp
@@ -24,24 +24,39 @@
 #include "slang-artifact-diagnostic-util.h"
 #include "slang-artifact-desc-util.h"
 
+// Enable DXIL by default unless told not to
+#ifndef SLANG_ENABLE_DXIL_SUPPORT
+#   define SLANG_ENABLE_DXIL_SUPPORT 1
+#endif
+
 // Enable calling through to  `dxc` to
 // generate code on Windows.
-#ifdef _WIN32
-#   define WIN32_LEAN_AND_MEAN
-#   define NOMINMAX
-#   include <Windows.h>
-#   include <Unknwn.h>
-#   include "../../external/dxc/dxcapi.h"
-#   undef WIN32_LEAN_AND_MEAN
-#   undef NOMINMAX
-
-#   ifndef SLANG_ENABLE_DXIL_SUPPORT
-#       define SLANG_ENABLE_DXIL_SUPPORT 1
+#if SLANG_ENABLE_DXIL_SUPPORT
+
+#   ifdef _WIN32
+#       define WIN32_LEAN_AND_MEAN
+#       define NOMINMAX
+#       include <Windows.h>
+#       include <Unknwn.h>
+#       include "../../external/dxc/dxcapi.h"
+#       undef WIN32_LEAN_AND_MEAN
+#       undef NOMINMAX
+#   else
+#       include "../../external/dxc/dxcapi.h"
+
+#       ifdef __uuidof
+            // DXC's WinAdapter.h defines __uuidof(T) over types, but the existing
+            // usage in this file is over values (both are accepted on MSVC.)
+            // We also need to decay through Slang::ComPtr, hence the helper struct
+            template <typename T>
+            struct StripSlangComPtr { using type = T; };
+            template <typename T>
+            struct StripSlangComPtr<Slang::ComPtr<T>> { using type = T; };
+#           undef __uuidof
+#           define __uuidof(x) __emulated_uuidof<StripSlangComPtr<std::decay_t<decltype(x)>>::type>()
+#       endif
 #   endif
-#endif
 
-#ifndef SLANG_ENABLE_DXIL_SUPPORT
-#   define SLANG_ENABLE_DXIL_SUPPORT 0
 #endif
 
 namespace Slang
@@ -93,7 +108,7 @@ class DxcIncludeHandler : public IDxcIncludeHandler
 {
 public:
     // Implement IUnknown
-    SLANG_NO_THROW HRESULT SLANG_MCALL QueryInterface(const IID& uuid, void** out)
+    SLANG_NO_THROW HRESULT SLANG_MCALL QueryInterface(const IID& uuid, void** out) override
     {
         ISlangUnknown* intf = getInterface(reinterpret_cast<const Guid&>(uuid));
         if (intf)
@@ -739,7 +754,7 @@ SlangResult DXCDownstreamCompiler::getVersionString(slang::IBlob** outVersionStr
     else
     {
         // If we don't have the commitHash, we use the library timestamp, to uniquely identify.
-        versionString << " " << SharedLibraryUtils::getSharedLibraryTimestamp(m_createInstance);
+        versionString << " " << SharedLibraryUtils::getSharedLibraryTimestamp(reinterpret_cast<void*>(m_createInstance));
     }
 
     *outVersionString = StringBlob::moveCreate(versionString).detach();
diff --git a/source/core/slang-platform.cpp b/source/core/slang-platform.cpp
index dc2d563df..bb612932c 100644
--- a/source/core/slang-platform.cpp
+++ b/source/core/slang-platform.cpp
@@ -177,9 +177,14 @@ SLANG_COMPILE_TIME_ASSERT(E_OUTOFMEMORY == SLANG_E_OUT_OF_MEMORY);
 /* static */SlangResult SharedLibrary::loadWithPlatformPath(char const* platformFileName, Handle& handleOut)
 {
     handleOut = nullptr;
+    const auto dxcName = "libdxcompiler";
+    const bool isDXC = strncmp(platformFileName, dxcName, strlen(dxcName)) == 0;
     if (strlen(platformFileName) == 0)
         platformFileName = nullptr;
-    void *h = dlopen(platformFileName, RTLD_NOW | RTLD_GLOBAL);
+    // Work around https://github.com/microsoft/DirectXShaderCompiler/issues/5119
+    // libdxcompiler.so invokes UB on dlclose
+    const auto mode = RTLD_NOW | RTLD_GLOBAL | (isDXC ? RTLD_NODELETE : 0);
+    void *h = dlopen(platformFileName, mode);
     if (!h)
     {
 #if 0