Use LOAD_LIBRARY_SEARCH_DEFAULT_DIRS for LoadLibraryExW (#8491)

Before:
- Uses `LOAD_LIBRARY_SEARCH_USER_DIRS` in `LoadLibraryExW`, which might
cause exception if there is no pathes added by `AddDllDirectory()`
After:
- Use the composite flag `LOAD_LIBRARY_SEARCH_DEFAULT_DIRS`, which
searches for several locations.
- Will still search dir added by `AddDllDirectory()`, but avoids empty
path seraching if there is no AddDllDirectory() calls.

Related to https://github.com/shader-slang/slang/issues/8462

---------

Co-authored-by: slangbot <ellieh+slangbot@nvidia.com>
Co-authored-by: slangbot <186143334+slangbot@users.noreply.github.com>

1 files changed, 16 insertions, 11 deletions

diff --git a/source/core/slang-platform.cpp b/source/core/slang-platform.cpp
index f02534e7c..6f3669dbe 100644
--- a/source/core/slang-platform.cpp
+++ b/source/core/slang-platform.cpp
@@ -129,19 +129,24 @@ SLANG_COMPILE_TIME_ASSERT(E_OUTOFMEMORY == SLANG_E_OUT_OF_MEMORY);
     }
 
     // We try to search the DLL in two different attempts.
-    // First attempt tries on the directories explicitly specified with AddDllDirectory(),
-    // If it failed to find one, we will search over all PATH.
-    // Windows API made two approaches mutually exclusive and we need to try two times.
+    // First attempt - LoadLibraryExW()
+    // If it failed to find one, we will use LoadLibraryW() to search over all PATH.
+    // Search order: 1) The directory that contains the DLL (LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR).
+    //                  This directory is searched only for dependencies of the DLL being loaded.
+    //               2) Application directory
+    //               3) User directories (AddDllDirectory/SetDllDirectory)
+    //               4) System32
+    //               5) PATH environment variable (by the 2nd attempt with LoadLibraryW())
     // https://learn.microsoft.com/en-us/windows/win32/api/libloaderapi/nf-libloaderapi-loadlibraryexw
-    String platformFileNameStr(platformFileName);
-    HMODULE h =
-        LoadLibraryExW(platformFileNameStr.toWString(), nullptr, LOAD_LIBRARY_SEARCH_USER_DIRS);
-    // If LoadLibraryExW failed, try again with LoadLibraryW.
     // https://docs.microsoft.com/en-us/windows/desktop/api/libloaderapi/nf-libloaderapi-loadlibraryw
-    if (!h)
-        h = LoadLibraryW(platformFileNameStr.toWString());
+    String platformFileNameStr(platformFileName);
+    OSString wideFileName = platformFileNameStr.toWString();
+    HMODULE handle = LoadLibraryExW(wideFileName, nullptr, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS);
+
+    if (!handle)
+        handle = LoadLibraryW(wideFileName);
     // If still not found, return an error.
-    if (!h)
+    if (!handle)
     {
         const DWORD lastError = GetLastError();
         switch (lastError)
@@ -164,7 +169,7 @@ SLANG_COMPILE_TIME_ASSERT(E_OUTOFMEMORY == SLANG_E_OUT_OF_MEMORY);
         // Turn to Result, if not one of the well known errors
         return HRESULT_FROM_WIN32(lastError);
     }
-    handleOut = (Handle)h;
+    handleOut = (Handle)handle;
     return SLANG_OK;
 }