Refactor shader cache (#2558)

* Fix a bug in Path::find

* Fix code formatting

* Fix LockFile and add LockFileGuard

* Add PersistentCache and unit test

* Replace file path dependency list with source file dependency list

* Add note on ordering in Module/FileDependencyList

* Remove old shader cache code

* Refactor shader cache implementation

* Temporarily skip unit tests reading/writing files

* Fix warning

* Reenable lock file test

* Rename shader cache tests and disable crashing test

* Testing

* Stop using Path::getCanonical

* Fix persistent cache lock and test

* Fix threading issues

* Move adding file dependency hashes to getEntryPointHash()

* Fix handling of #include files

* Allow specifying additional search paths for gfx testing device

* Work on shader cache tests

* Update project files

* Revive shader cache graphics tests

* Split graphics pipeline test

* Fix compilation

10 files changed, 32 insertions, 16 deletions

diff --git a/build/visual-studio/core/core.vcxproj b/build/visual-studio/core/core.vcxproj
index e171a7d68..ecd6b54fe 100644
--- a/build/visual-studio/core/core.vcxproj
+++ b/build/visual-studio/core/core.vcxproj
@@ -297,6 +297,7 @@
     <ClInclude Include="..\..\..\source\core\slang-memory-arena.h" />
     <ClInclude Include="..\..\..\source\core\slang-memory-file-system.h" />
     <ClInclude Include="..\..\..\source\core\slang-offset-container.h" />
+    <ClInclude Include="..\..\..\source\core\slang-persistent-cache.h" />
     <ClInclude Include="..\..\..\source\core\slang-platform.h" />
     <ClInclude Include="..\..\..\source\core\slang-process-util.h" />
     <ClInclude Include="..\..\..\source\core\slang-process.h" />
@@ -353,6 +354,7 @@
     <ClCompile Include="..\..\..\source\core\slang-memory-arena.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-memory-file-system.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-offset-container.cpp" />
+    <ClCompile Include="..\..\..\source\core\slang-persistent-cache.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-platform.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-process-util.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-random-generator.cpp" />
diff --git a/build/visual-studio/core/core.vcxproj.filters b/build/visual-studio/core/core.vcxproj.filters
index 06d151349..144c5259f 100644
--- a/build/visual-studio/core/core.vcxproj.filters
+++ b/build/visual-studio/core/core.vcxproj.filters
@@ -123,6 +123,9 @@
     <ClInclude Include="..\..\..\source\core\slang-offset-container.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\..\source\core\slang-persistent-cache.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\..\source\core\slang-platform.h">
       <Filter>Header Files</Filter>
     </ClInclude>
@@ -287,6 +290,9 @@
     <ClCompile Include="..\..\..\source\core\slang-offset-container.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\source\core\slang-persistent-cache.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\source\core\slang-platform.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
diff --git a/build/visual-studio/gfx-unit-test-tool/gfx-unit-test-tool.vcxproj b/build/visual-studio/gfx-unit-test-tool/gfx-unit-test-tool.vcxproj
index b0879ccfe..869b0fd27 100644
--- a/build/visual-studio/gfx-unit-test-tool/gfx-unit-test-tool.vcxproj
+++ b/build/visual-studio/gfx-unit-test-tool/gfx-unit-test-tool.vcxproj
@@ -317,16 +317,17 @@
     <None Include="..\..\..\tools\gfx-unit-test\compute-trivial.slang" />

     <None Include="..\..\..\tools\gfx-unit-test\format-test-shaders.slang" />

     <None Include="..\..\..\tools\gfx-unit-test\graphics-smoke.slang" />

-    <None Include="..\..\..\tools\gfx-unit-test\multiple-entry-point-shader-cache-shader.slang" />

     <None Include="..\..\..\tools\gfx-unit-test\mutable-shader-object.slang" />

     <None Include="..\..\..\tools\gfx-unit-test\nested-parameter-block.slang" />

     <None Include="..\..\..\tools\gfx-unit-test\ray-tracing-test-shaders.slang" />

     <None Include="..\..\..\tools\gfx-unit-test\resolve-resource-shader.slang" />

     <None Include="..\..\..\tools\gfx-unit-test\root-shader-parameter.slang" />

     <None Include="..\..\..\tools\gfx-unit-test\sampler-array.slang" />

+    <None Include="..\..\..\tools\gfx-unit-test\shader-cache-graphics-fragment.slang" />

+    <None Include="..\..\..\tools\gfx-unit-test\shader-cache-graphics-vertex.slang" />

     <None Include="..\..\..\tools\gfx-unit-test\shader-cache-graphics.slang" />

-    <None Include="..\..\..\tools\gfx-unit-test\split-graphics-fragment.slang" />

-    <None Include="..\..\..\tools\gfx-unit-test\split-graphics-vertex.slang" />

+    <None Include="..\..\..\tools\gfx-unit-test\shader-cache-multiple-entry-points.slang" />

+    <None Include="..\..\..\tools\gfx-unit-test\shader-cache-specialization.slang" />

     <None Include="..\..\..\tools\gfx-unit-test\swapchain-shader.slang" />

     <None Include="..\..\..\tools\gfx-unit-test\trivial-copy-textures.slang" />

     <None Include="..\..\..\tools\gfx-unit-test\trivial-copy.slang" />

diff --git a/build/visual-studio/gfx-unit-test-tool/gfx-unit-test-tool.vcxproj.filters b/build/visual-studio/gfx-unit-test-tool/gfx-unit-test-tool.vcxproj.filters
index 00fd1de16..58045ccee 100644
--- a/build/visual-studio/gfx-unit-test-tool/gfx-unit-test-tool.vcxproj.filters
+++ b/build/visual-studio/gfx-unit-test-tool/gfx-unit-test-tool.vcxproj.filters
@@ -121,9 +121,6 @@
     <None Include="..\..\..\tools\gfx-unit-test\graphics-smoke.slang">

       <Filter>Source Files</Filter>

     </None>

-    <None Include="..\..\..\tools\gfx-unit-test\multiple-entry-point-shader-cache-shader.slang">

-      <Filter>Source Files</Filter>

-    </None>

     <None Include="..\..\..\tools\gfx-unit-test\mutable-shader-object.slang">

       <Filter>Source Files</Filter>

     </None>

@@ -142,13 +139,19 @@
     <None Include="..\..\..\tools\gfx-unit-test\sampler-array.slang">

       <Filter>Source Files</Filter>

     </None>

+    <None Include="..\..\..\tools\gfx-unit-test\shader-cache-graphics-fragment.slang">

+      <Filter>Source Files</Filter>

+    </None>

+    <None Include="..\..\..\tools\gfx-unit-test\shader-cache-graphics-vertex.slang">

+      <Filter>Source Files</Filter>

+    </None>

     <None Include="..\..\..\tools\gfx-unit-test\shader-cache-graphics.slang">

       <Filter>Source Files</Filter>

     </None>

-    <None Include="..\..\..\tools\gfx-unit-test\split-graphics-fragment.slang">

+    <None Include="..\..\..\tools\gfx-unit-test\shader-cache-multiple-entry-points.slang">

       <Filter>Source Files</Filter>

     </None>

-    <None Include="..\..\..\tools\gfx-unit-test\split-graphics-vertex.slang">

+    <None Include="..\..\..\tools\gfx-unit-test\shader-cache-specialization.slang">

       <Filter>Source Files</Filter>

     </None>

     <None Include="..\..\..\tools\gfx-unit-test\swapchain-shader.slang">

diff --git a/build/visual-studio/gfx/gfx.vcxproj b/build/visual-studio/gfx/gfx.vcxproj
index 8f200be8e..476f6a808 100644
--- a/build/visual-studio/gfx/gfx.vcxproj
+++ b/build/visual-studio/gfx/gfx.vcxproj
@@ -419,7 +419,6 @@ IF EXIST "$(SolutionDir)tools\gfx\slang.slang"\ (xcopy /Q /E /Y /I "$(SolutionDi
     <ClInclude Include="..\..\..\tools\gfx\nvapi\nvapi-include.h" />

     <ClInclude Include="..\..\..\tools\gfx\nvapi\nvapi-util.h" />

     <ClInclude Include="..\..\..\tools\gfx\open-gl\render-gl.h" />

-    <ClInclude Include="..\..\..\tools\gfx\persistent-shader-cache.h" />

     <ClInclude Include="..\..\..\tools\gfx\renderer-shared.h" />

     <ClInclude Include="..\..\..\tools\gfx\resource-desc-utils.h" />

     <ClInclude Include="..\..\..\tools\gfx\simple-render-pass-layout.h" />

@@ -530,7 +529,6 @@ IF EXIST "$(SolutionDir)tools\gfx\slang.slang"\ (xcopy /Q /E /Y /I "$(SolutionDi
     <ClCompile Include="..\..\..\tools\gfx\immediate-renderer-base.cpp" />

     <ClCompile Include="..\..\..\tools\gfx\nvapi\nvapi-util.cpp" />

     <ClCompile Include="..\..\..\tools\gfx\open-gl\render-gl.cpp" />

-    <ClCompile Include="..\..\..\tools\gfx\persistent-shader-cache.cpp" />

     <ClCompile Include="..\..\..\tools\gfx\render.cpp" />

     <ClCompile Include="..\..\..\tools\gfx\renderer-shared.cpp" />

     <ClCompile Include="..\..\..\tools\gfx\resource-desc-utils.cpp" />

diff --git a/build/visual-studio/gfx/gfx.vcxproj.filters b/build/visual-studio/gfx/gfx.vcxproj.filters
index c708450d5..1a7ed3d03 100644
--- a/build/visual-studio/gfx/gfx.vcxproj.filters
+++ b/build/visual-studio/gfx/gfx.vcxproj.filters
@@ -306,9 +306,6 @@
     <ClInclude Include="..\..\..\tools\gfx\open-gl\render-gl.h">

       <Filter>Header Files</Filter>

     </ClInclude>

-    <ClInclude Include="..\..\..\tools\gfx\persistent-shader-cache.h">

-      <Filter>Header Files</Filter>

-    </ClInclude>

     <ClInclude Include="..\..\..\tools\gfx\renderer-shared.h">

       <Filter>Header Files</Filter>

     </ClInclude>

@@ -635,9 +632,6 @@
     <ClCompile Include="..\..\..\tools\gfx\open-gl\render-gl.cpp">

       <Filter>Source Files</Filter>

     </ClCompile>

-    <ClCompile Include="..\..\..\tools\gfx\persistent-shader-cache.cpp">

-      <Filter>Source Files</Filter>

-    </ClCompile>

     <ClCompile Include="..\..\..\tools\gfx\render.cpp">

       <Filter>Source Files</Filter>

     </ClCompile>

diff --git a/build/visual-studio/slang-rt/slang-rt.vcxproj b/build/visual-studio/slang-rt/slang-rt.vcxproj
index 03d3852fd..92191667c 100644
--- a/build/visual-studio/slang-rt/slang-rt.vcxproj
+++ b/build/visual-studio/slang-rt/slang-rt.vcxproj
@@ -309,6 +309,7 @@
     <ClInclude Include="..\..\..\source\core\slang-memory-arena.h" />
     <ClInclude Include="..\..\..\source\core\slang-memory-file-system.h" />
     <ClInclude Include="..\..\..\source\core\slang-offset-container.h" />
+    <ClInclude Include="..\..\..\source\core\slang-persistent-cache.h" />
     <ClInclude Include="..\..\..\source\core\slang-platform.h" />
     <ClInclude Include="..\..\..\source\core\slang-process-util.h" />
     <ClInclude Include="..\..\..\source\core\slang-process.h" />
@@ -366,6 +367,7 @@
     <ClCompile Include="..\..\..\source\core\slang-memory-arena.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-memory-file-system.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-offset-container.cpp" />
+    <ClCompile Include="..\..\..\source\core\slang-persistent-cache.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-platform.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-process-util.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-random-generator.cpp" />
diff --git a/build/visual-studio/slang-rt/slang-rt.vcxproj.filters b/build/visual-studio/slang-rt/slang-rt.vcxproj.filters
index df9f8c2ca..b99c077ae 100644
--- a/build/visual-studio/slang-rt/slang-rt.vcxproj.filters
+++ b/build/visual-studio/slang-rt/slang-rt.vcxproj.filters
@@ -123,6 +123,9 @@
     <ClInclude Include="..\..\..\source\core\slang-offset-container.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\..\source\core\slang-persistent-cache.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\..\source\core\slang-platform.h">
       <Filter>Header Files</Filter>
     </ClInclude>
@@ -290,6 +293,9 @@
     <ClCompile Include="..\..\..\source\core\slang-offset-container.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\source\core\slang-persistent-cache.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\source\core\slang-platform.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
diff --git a/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj b/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj
index deab210ee..5eec9ec82 100644
--- a/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj
+++ b/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj
@@ -296,6 +296,7 @@
     <ClCompile Include="..\..\..\tools\slang-unit-test\unit-test-memory-arena.cpp" />

     <ClCompile Include="..\..\..\tools\slang-unit-test\unit-test-offset-container.cpp" />

     <ClCompile Include="..\..\..\tools\slang-unit-test\unit-test-path.cpp" />

+    <ClCompile Include="..\..\..\tools\slang-unit-test\unit-test-persistent-cache.cpp" />

     <ClCompile Include="..\..\..\tools\slang-unit-test\unit-test-process.cpp" />

     <ClCompile Include="..\..\..\tools\slang-unit-test\unit-test-riff.cpp" />

     <ClCompile Include="..\..\..\tools\slang-unit-test\unit-test-rtti.cpp" />

diff --git a/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj.filters b/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj.filters
index a33dc44cc..c350b6f24 100644
--- a/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj.filters
+++ b/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj.filters
@@ -62,6 +62,9 @@
     <ClCompile Include="..\..\..\tools\slang-unit-test\unit-test-path.cpp">

       <Filter>Source Files</Filter>

     </ClCompile>

+    <ClCompile Include="..\..\..\tools\slang-unit-test\unit-test-persistent-cache.cpp">

+      <Filter>Source Files</Filter>

+    </ClCompile>

     <ClCompile Include="..\..\..\tools\slang-unit-test\unit-test-process.cpp">

       <Filter>Source Files</Filter>

     </ClCompile>