From f0cd62b37c5dfbbdb3fb205f1be2b8beba0dfed4 Mon Sep 17 00:00:00 2001
From: lucy96chen <47800040+lucy96chen@users.noreply.github.com>
Date: Wed, 12 Oct 2022 09:55:09 -0700
Subject: Shader caching (#2432)

* Changed all getEntryPointCode calls to use RendererBase::getEntryPointCodeFromShaderCache

* Hashing hooked up, tests pass but need to add more to fully test functionality

* checkpoint

* Checkpoint: File system creation seems functional, saving is broken

* checkpoint: Fixed filename generation from MD5 hash, shader blob might be going missing ahead of pipeline state creation

* Fixed a lot of bugs related to hash code generation, shader cache is likely working but needs further testing

* Added workaround for module loading by re-creating the test device, shader cache test functional

* Vulkan shader caching bug fixed, checkpoint commit before more refinement

* pre-ToT merge checkpoint

* checkpoint commit, improving cache keys

* Significantly expanded items included in the dependency hash for Module; Added dependency hash functions to SpecializedComponentType and RenamedEntryPointComponentType

* Temporarily disable shader cache test

* Mid cleanup changes, solution successfully builds

* Added several helper update functions to slang-md5 to help simplify usage; Added a function under ISession to compute a hash for all linkage-related items; Function renames and cleaned up some comments

* Ran premake.bat; Renamed getASTBasedHashCode to computeASTBasedHash

* Added slang unit tests for Checksum and MD5; Extended gfx shader cache test to test with multiple shader files and one shader file with multiple entry points

* Solution builds and shader cache tests pass, but at least a couple other tests now failing

* ran premake.bat

* More cleanup changes

* Added shaderCachePath field to IDevice desc in gfx.slang, gfx-smoke.slang should be functional

* ran premake

* cleanup changes; Adding test printf to getEntryPointCodeFromShaderCache to see if output can be seen in CI

* Removed debugging printfs; Added handling for getEntryPointCode() failing

* Cleanup changes; Jonathan's fixes to SerialWriter to zero initialize otherwise uninitialized memory; Change to SwizzleExpr creation to zero initialize elementCount

* Changed enable_if_t to enable_if

* Fixed enable_if

* Added test for import vs include and changes to included and imported files; Fixed build errors in CUDA; Renamed shader cache statistics fields

* cleanup changes

* Readd removed file

* Restructured computeDependencyBasedHash calls, added computeDependencyBasedHashImpl to all classes dervied from ComponentType

* Applied same restructuring to the AST hash functions

* Cleanup changes; Moved HashBuilder out to slang-digest.h and added some helper functions to streamline the process of adding items to a hash

* Cleanup; Fixed incorrect expected results for shader import and include test
---
 build/visual-studio/core/core.vcxproj              |  3 +++
 build/visual-studio/core/core.vcxproj.filters      |  9 ++++++++
 .../gfx-unit-test-tool/gfx-unit-test-tool.vcxproj  |  8 ++++++++
 .../gfx-unit-test-tool.vcxproj.filters             | 24 ++++++++++++++++++++++
 build/visual-studio/slang-rt/slang-rt.vcxproj      |  3 +++
 .../slang-rt/slang-rt.vcxproj.filters              |  9 ++++++++
 .../slang-unit-test-tool.vcxproj                   |  2 ++
 .../slang-unit-test-tool.vcxproj.filters           |  6 ++++++
 build/visual-studio/slang/slang.vcxproj            |  1 +
 build/visual-studio/slang/slang.vcxproj.filters    |  3 +++
 10 files changed, 68 insertions(+)

(limited to 'build')

diff --git a/build/visual-studio/core/core.vcxproj b/build/visual-studio/core/core.vcxproj
index a49ccd040..f84695880 100644
--- a/build/visual-studio/core/core.vcxproj
+++ b/build/visual-studio/core/core.vcxproj
@@ -270,6 +270,7 @@
     <ClInclude Include="..\..\..\source\core\slang-deflate-compression-system.h" />
     <ClInclude Include="..\..\..\source\core\slang-destroyable.h" />
     <ClInclude Include="..\..\..\source\core\slang-dictionary.h" />
+    <ClInclude Include="..\..\..\source\core\slang-digest.h" />
     <ClInclude Include="..\..\..\source\core\slang-exception.h" />
     <ClInclude Include="..\..\..\source\core\slang-file-system.h" />
     <ClInclude Include="..\..\..\source\core\slang-free-list.h" />
@@ -284,6 +285,7 @@
     <ClInclude Include="..\..\..\source\core\slang-list.h" />
     <ClInclude Include="..\..\..\source\core\slang-lz4-compression-system.h" />
     <ClInclude Include="..\..\..\source\core\slang-math.h" />
+    <ClInclude Include="..\..\..\source\core\slang-md5.h" />
     <ClInclude Include="..\..\..\source\core\slang-memory-arena.h" />
     <ClInclude Include="..\..\..\source\core\slang-memory-file-system.h" />
     <ClInclude Include="..\..\..\source\core\slang-offset-container.h" />
@@ -339,6 +341,7 @@
     <ClCompile Include="..\..\..\source\core\slang-io.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-lazy-castable-list.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-lz4-compression-system.cpp" />
+    <ClCompile Include="..\..\..\source\core\slang-md5.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-memory-arena.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-memory-file-system.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-offset-container.cpp" />
diff --git a/build/visual-studio/core/core.vcxproj.filters b/build/visual-studio/core/core.vcxproj.filters
index 9266db917..14d866c12 100644
--- a/build/visual-studio/core/core.vcxproj.filters
+++ b/build/visual-studio/core/core.vcxproj.filters
@@ -69,6 +69,9 @@
     <ClInclude Include="..\..\..\source\core\slang-dictionary.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\..\source\core\slang-digest.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\..\source\core\slang-exception.h">
       <Filter>Header Files</Filter>
     </ClInclude>
@@ -111,6 +114,9 @@
     <ClInclude Include="..\..\..\source\core\slang-math.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\..\source\core\slang-md5.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\..\source\core\slang-memory-arena.h">
       <Filter>Header Files</Filter>
     </ClInclude>
@@ -272,6 +278,9 @@
     <ClCompile Include="..\..\..\source\core\slang-lz4-compression-system.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\source\core\slang-md5.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\source\core\slang-memory-arena.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
diff --git a/build/visual-studio/gfx-unit-test-tool/gfx-unit-test-tool.vcxproj b/build/visual-studio/gfx-unit-test-tool/gfx-unit-test-tool.vcxproj
index a346900c6..7553119d3 100644
--- a/build/visual-studio/gfx-unit-test-tool/gfx-unit-test-tool.vcxproj
+++ b/build/visual-studio/gfx-unit-test-tool/gfx-unit-test-tool.vcxproj
@@ -295,6 +295,7 @@
     <ClCompile Include="..\..\..\tools\gfx-unit-test\root-mutable-shader-object.cpp" />
     <ClCompile Include="..\..\..\tools\gfx-unit-test\root-shader-parameter.cpp" />
     <ClCompile Include="..\..\..\tools\gfx-unit-test\sampler-array.cpp" />
+    <ClCompile Include="..\..\..\tools\gfx-unit-test\shader-cache-tests.cpp" />
     <ClCompile Include="..\..\..\tools\gfx-unit-test\shared-buffers-tests.cpp" />
     <ClCompile Include="..\..\..\tools\gfx-unit-test\shared-textures-tests.cpp" />
     <ClCompile Include="..\..\..\tools\gfx-unit-test\swap-chain-resize-test.cpp" />
@@ -307,12 +308,19 @@
     <None Include="..\..\..\tools\gfx-unit-test\compute-trivial.slang" />
     <None Include="..\..\..\tools\gfx-unit-test\format-test-shaders.slang" />
     <None Include="..\..\..\tools\gfx-unit-test\graphics-smoke.slang" />
+    <None Include="..\..\..\tools\gfx-unit-test\imported.slang" />
+    <None Include="..\..\..\tools\gfx-unit-test\importing-shader-cache-shader.slang" />
+    <None Include="..\..\..\tools\gfx-unit-test\multiple-entry-point-shader-cache-shader.slang" />
     <None Include="..\..\..\tools\gfx-unit-test\mutable-shader-object.slang" />
     <None Include="..\..\..\tools\gfx-unit-test\nested-parameter-block.slang" />
     <None Include="..\..\..\tools\gfx-unit-test\ray-tracing-test-shaders.slang" />
     <None Include="..\..\..\tools\gfx-unit-test\resolve-resource-shader.slang" />
     <None Include="..\..\..\tools\gfx-unit-test\root-shader-parameter.slang" />
     <None Include="..\..\..\tools\gfx-unit-test\sampler-array.slang" />
+    <None Include="..\..\..\tools\gfx-unit-test\shader-cache-shader-A.slang" />
+    <None Include="..\..\..\tools\gfx-unit-test\shader-cache-shader-B.slang" />
+    <None Include="..\..\..\tools\gfx-unit-test\shader-cache-shader-C.slang" />
+    <None Include="..\..\..\tools\gfx-unit-test\shader-cache-shader.slang" />
     <None Include="..\..\..\tools\gfx-unit-test\swapchain-shader.slang" />
     <None Include="..\..\..\tools\gfx-unit-test\trivial-copy-textures.slang" />
     <None Include="..\..\..\tools\gfx-unit-test\trivial-copy.slang" />
diff --git a/build/visual-studio/gfx-unit-test-tool/gfx-unit-test-tool.vcxproj.filters b/build/visual-studio/gfx-unit-test-tool/gfx-unit-test-tool.vcxproj.filters
index 157e1e554..90dc02b69 100644
--- a/build/visual-studio/gfx-unit-test-tool/gfx-unit-test-tool.vcxproj.filters
+++ b/build/visual-studio/gfx-unit-test-tool/gfx-unit-test-tool.vcxproj.filters
@@ -86,6 +86,9 @@
     <ClCompile Include="..\..\..\tools\gfx-unit-test\sampler-array.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\tools\gfx-unit-test\shader-cache-tests.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\tools\gfx-unit-test\shared-buffers-tests.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
@@ -118,6 +121,15 @@
     <None Include="..\..\..\tools\gfx-unit-test\graphics-smoke.slang">
       <Filter>Source Files</Filter>
     </None>
+    <None Include="..\..\..\tools\gfx-unit-test\imported.slang">
+      <Filter>Source Files</Filter>
+    </None>
+    <None Include="..\..\..\tools\gfx-unit-test\importing-shader-cache-shader.slang">
+      <Filter>Source Files</Filter>
+    </None>
+    <None Include="..\..\..\tools\gfx-unit-test\multiple-entry-point-shader-cache-shader.slang">
+      <Filter>Source Files</Filter>
+    </None>
     <None Include="..\..\..\tools\gfx-unit-test\mutable-shader-object.slang">
       <Filter>Source Files</Filter>
     </None>
@@ -136,6 +148,18 @@
     <None Include="..\..\..\tools\gfx-unit-test\sampler-array.slang">
       <Filter>Source Files</Filter>
     </None>
+    <None Include="..\..\..\tools\gfx-unit-test\shader-cache-shader-A.slang">
+      <Filter>Source Files</Filter>
+    </None>
+    <None Include="..\..\..\tools\gfx-unit-test\shader-cache-shader-B.slang">
+      <Filter>Source Files</Filter>
+    </None>
+    <None Include="..\..\..\tools\gfx-unit-test\shader-cache-shader-C.slang">
+      <Filter>Source Files</Filter>
+    </None>
+    <None Include="..\..\..\tools\gfx-unit-test\shader-cache-shader.slang">
+      <Filter>Source Files</Filter>
+    </None>
     <None Include="..\..\..\tools\gfx-unit-test\swapchain-shader.slang">
       <Filter>Source Files</Filter>
     </None>
diff --git a/build/visual-studio/slang-rt/slang-rt.vcxproj b/build/visual-studio/slang-rt/slang-rt.vcxproj
index 55c69a6c2..c30f5f077 100644
--- a/build/visual-studio/slang-rt/slang-rt.vcxproj
+++ b/build/visual-studio/slang-rt/slang-rt.vcxproj
@@ -282,6 +282,7 @@
     <ClInclude Include="..\..\..\source\core\slang-deflate-compression-system.h" />
     <ClInclude Include="..\..\..\source\core\slang-destroyable.h" />
     <ClInclude Include="..\..\..\source\core\slang-dictionary.h" />
+    <ClInclude Include="..\..\..\source\core\slang-digest.h" />
     <ClInclude Include="..\..\..\source\core\slang-exception.h" />
     <ClInclude Include="..\..\..\source\core\slang-file-system.h" />
     <ClInclude Include="..\..\..\source\core\slang-free-list.h" />
@@ -296,6 +297,7 @@
     <ClInclude Include="..\..\..\source\core\slang-list.h" />
     <ClInclude Include="..\..\..\source\core\slang-lz4-compression-system.h" />
     <ClInclude Include="..\..\..\source\core\slang-math.h" />
+    <ClInclude Include="..\..\..\source\core\slang-md5.h" />
     <ClInclude Include="..\..\..\source\core\slang-memory-arena.h" />
     <ClInclude Include="..\..\..\source\core\slang-memory-file-system.h" />
     <ClInclude Include="..\..\..\source\core\slang-offset-container.h" />
@@ -352,6 +354,7 @@
     <ClCompile Include="..\..\..\source\core\slang-io.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-lazy-castable-list.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-lz4-compression-system.cpp" />
+    <ClCompile Include="..\..\..\source\core\slang-md5.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-memory-arena.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-memory-file-system.cpp" />
     <ClCompile Include="..\..\..\source\core\slang-offset-container.cpp" />
diff --git a/build/visual-studio/slang-rt/slang-rt.vcxproj.filters b/build/visual-studio/slang-rt/slang-rt.vcxproj.filters
index d5726e5f9..65f4d5221 100644
--- a/build/visual-studio/slang-rt/slang-rt.vcxproj.filters
+++ b/build/visual-studio/slang-rt/slang-rt.vcxproj.filters
@@ -69,6 +69,9 @@
     <ClInclude Include="..\..\..\source\core\slang-dictionary.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\..\source\core\slang-digest.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\..\source\core\slang-exception.h">
       <Filter>Header Files</Filter>
     </ClInclude>
@@ -111,6 +114,9 @@
     <ClInclude Include="..\..\..\source\core\slang-math.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\..\source\core\slang-md5.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\..\source\core\slang-memory-arena.h">
       <Filter>Header Files</Filter>
     </ClInclude>
@@ -275,6 +281,9 @@
     <ClCompile Include="..\..\..\source\core\slang-lz4-compression-system.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\source\core\slang-md5.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\source\core\slang-memory-arena.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
diff --git a/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj b/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj
index a6a5dad7d..83681488f 100644
--- a/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj
+++ b/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj
@@ -272,6 +272,7 @@
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\..\..\tools\slang-unit-test\unit-test-byte-encode.cpp" />
+    <ClCompile Include="..\..\..\tools\slang-unit-test\unit-test-checksum.cpp" />
     <ClCompile Include="..\..\..\tools\slang-unit-test\unit-test-chunked-list.cpp" />
     <ClCompile Include="..\..\..\tools\slang-unit-test\unit-test-com-host-callable.cpp" />
     <ClCompile Include="..\..\..\tools\slang-unit-test\unit-test-command-line-args.cpp" />
@@ -282,6 +283,7 @@
     <ClCompile Include="..\..\..\tools\slang-unit-test\unit-test-io.cpp" />
     <ClCompile Include="..\..\..\tools\slang-unit-test\unit-test-json-native.cpp" />
     <ClCompile Include="..\..\..\tools\slang-unit-test\unit-test-json.cpp" />
+    <ClCompile Include="..\..\..\tools\slang-unit-test\unit-test-md5.cpp" />
     <ClCompile Include="..\..\..\tools\slang-unit-test\unit-test-memory-arena.cpp" />
     <ClCompile Include="..\..\..\tools\slang-unit-test\unit-test-offset-container.cpp" />
     <ClCompile Include="..\..\..\tools\slang-unit-test\unit-test-path.cpp" />
diff --git a/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj.filters b/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj.filters
index c84d21b30..b83a18b76 100644
--- a/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj.filters
+++ b/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj.filters
@@ -17,6 +17,9 @@
     <ClCompile Include="..\..\..\tools\slang-unit-test\unit-test-byte-encode.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\tools\slang-unit-test\unit-test-checksum.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\tools\slang-unit-test\unit-test-chunked-list.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
@@ -47,6 +50,9 @@
     <ClCompile Include="..\..\..\tools\slang-unit-test\unit-test-json.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\tools\slang-unit-test\unit-test-md5.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\tools\slang-unit-test\unit-test-memory-arena.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
diff --git a/build/visual-studio/slang/slang.vcxproj b/build/visual-studio/slang/slang.vcxproj
index 7436f319e..4e8a9e70a 100644
--- a/build/visual-studio/slang/slang.vcxproj
+++ b/build/visual-studio/slang/slang.vcxproj
@@ -327,6 +327,7 @@ IF EXIST ..\..\..\external\slang-glslang\bin\windows-aarch64\release\slang-glsla
     <ClInclude Include="..\..\..\source\slang\slang-emit-precedence.h" />
     <ClInclude Include="..\..\..\source\slang\slang-emit-source-writer.h" />
     <ClInclude Include="..\..\..\source\slang\slang-glsl-extension-tracker.h" />
+    <ClInclude Include="..\..\..\source\slang\slang-hash-utils.h" />
     <ClInclude Include="..\..\..\source\slang\slang-hlsl-intrinsic-set.h" />
     <ClInclude Include="..\..\..\source\slang\slang-image-format-defs.h" />
     <ClInclude Include="..\..\..\source\slang\slang-intrinsic-expand.h" />
diff --git a/build/visual-studio/slang/slang.vcxproj.filters b/build/visual-studio/slang/slang.vcxproj.filters
index 02ef9de2e..8e6763e15 100644
--- a/build/visual-studio/slang/slang.vcxproj.filters
+++ b/build/visual-studio/slang/slang.vcxproj.filters
@@ -114,6 +114,9 @@
     <ClInclude Include="..\..\..\source\slang\slang-glsl-extension-tracker.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\..\source\slang\slang-hash-utils.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\..\source\slang\slang-hlsl-intrinsic-set.h">
       <Filter>Header Files</Filter>
     </ClInclude>
-- 
cgit v1.2.3