Shader caching (#2432)

* Changed all getEntryPointCode calls to use RendererBase::getEntryPointCodeFromShaderCache

* Hashing hooked up, tests pass but need to add more to fully test functionality

* checkpoint

* Checkpoint: File system creation seems functional, saving is broken

* checkpoint: Fixed filename generation from MD5 hash, shader blob might be going missing ahead of pipeline state creation

* Fixed a lot of bugs related to hash code generation, shader cache is likely working but needs further testing

* Added workaround for module loading by re-creating the test device, shader cache test functional

* Vulkan shader caching bug fixed, checkpoint commit before more refinement

* pre-ToT merge checkpoint

* checkpoint commit, improving cache keys

* Significantly expanded items included in the dependency hash for Module; Added dependency hash functions to SpecializedComponentType and RenamedEntryPointComponentType

* Temporarily disable shader cache test

* Mid cleanup changes, solution successfully builds

* Added several helper update functions to slang-md5 to help simplify usage; Added a function under ISession to compute a hash for all linkage-related items; Function renames and cleaned up some comments

* Ran premake.bat; Renamed getASTBasedHashCode to computeASTBasedHash

* Added slang unit tests for Checksum and MD5; Extended gfx shader cache test to test with multiple shader files and one shader file with multiple entry points

* Solution builds and shader cache tests pass, but at least a couple other tests now failing

* ran premake.bat

* More cleanup changes

* Added shaderCachePath field to IDevice desc in gfx.slang, gfx-smoke.slang should be functional

* ran premake

* cleanup changes; Adding test printf to getEntryPointCodeFromShaderCache to see if output can be seen in CI

* Removed debugging printfs; Added handling for getEntryPointCode() failing

* Cleanup changes; Jonathan's fixes to SerialWriter to zero initialize otherwise uninitialized memory; Change to SwizzleExpr creation to zero initialize elementCount

* Changed enable_if_t to enable_if

* Fixed enable_if

* Added test for import vs include and changes to included and imported files; Fixed build errors in CUDA; Renamed shader cache statistics fields

* cleanup changes

* Readd removed file

* Restructured computeDependencyBasedHash calls, added computeDependencyBasedHashImpl to all classes dervied from ComponentType

* Applied same restructuring to the AST hash functions

* Cleanup changes; Moved HashBuilder out to slang-digest.h and added some helper functions to streamline the process of adding items to a hash

* Cleanup; Fixed incorrect expected results for shader import and include test

1 files changed, 117 insertions, 2 deletions

diff --git a/source/slang/slang-compiler.h b/source/slang/slang-compiler.h
index 46be07fa3..016c8fefa 100755
--- a/source/slang/slang-compiler.h
+++ b/source/slang/slang-compiler.h
@@ -3,6 +3,7 @@
 
 #include "../core/slang-basic.h"
 #include "../core/slang-shared-library.h"
+#include "../core/slang-digest.h"
 
 #include "../compiler-core/slang-downstream-compiler.h"
 #include "../compiler-core/slang-downstream-compiler-util.h"
@@ -282,14 +283,23 @@ namespace Slang
             const char* newName,
             slang::IComponentType** outEntryPoint) SLANG_OVERRIDE;
         SLANG_NO_THROW SlangResult SLANG_MCALL link(
-            slang::IComponentType**         outLinkedComponentType,
-            ISlangBlob**                    outDiagnostics) SLANG_OVERRIDE;
+            slang::IComponentType** outLinkedComponentType,
+            ISlangBlob**            outDiagnostics) SLANG_OVERRIDE;
         SLANG_NO_THROW SlangResult SLANG_MCALL getEntryPointHostCallable(
             int                     entryPointIndex,
             int                     targetIndex,
             ISlangSharedLibrary**   outSharedLibrary,
             slang::IBlob**          outDiagnostics) SLANG_OVERRIDE;
 
+            /// ComponentType is the only class inheriting from IComponentType that provides a
+            /// meaningful implementation for these two functions. All others should forward these
+            /// and implement updateDependencyBasedHash and updateASTBasedHash instead.
+        SLANG_NO_THROW void SLANG_MCALL computeDependencyBasedHash(
+            SlangInt entryPointIndex,
+            SlangInt targetIndex,
+            slang::Digest* outHash) SLANG_OVERRIDE;
+        SLANG_NO_THROW void SLANG_MCALL computeASTBasedHash(slang::Digest* outHash) SLANG_OVERRIDE;
+
             /// Get the linkage (aka "session" in the public API) for this component type.
         Linkage* getLinkage() { return m_linkage; }
 
@@ -298,6 +308,16 @@ namespace Slang
             /// The `target` must be a target on the `Linkage` that was used to create this program.
         TargetProgram* getTargetProgram(TargetRequest* target);
 
+            /// Update the hash builder with the dependencies for this component type. 
+        virtual void updateDependencyBasedHash(
+            DigestBuilder& hashBuilder,
+            SlangInt entryPointIndex) = 0;
+
+            /// Update the hash builder with the AST contents for this component type.
+            /// The AST is associated with a Module component, so most derived ComponentType classes
+            /// will simply do nothing with this.
+        virtual void updateASTBasedHash(DigestBuilder& hashBuilder) = 0;
+
             /// Get the number of entry points linked into this component type.
         virtual Index getEntryPointCount() = 0;
 
@@ -495,6 +515,12 @@ namespace Slang
             Linkage*                            linkage,
             List<RefPtr<ComponentType>> const&  childComponents);
 
+        virtual void updateDependencyBasedHash(
+            DigestBuilder& hashBuilder,
+            SlangInt entryPointIndex) override;
+
+        virtual void updateASTBasedHash(DigestBuilder& hashBuilder) override;
+
         List<RefPtr<ComponentType>> const& getChildComponents() { return m_childComponents; };
         Index getChildComponentCount() { return m_childComponents.getCount(); }
         RefPtr<ComponentType> getChildComponent(Index index) { return m_childComponents[index]; }
@@ -571,6 +597,15 @@ namespace Slang
             List<SpecializationArg> const&  specializationArgs,
             DiagnosticSink*                 sink);
 
+        virtual void updateDependencyBasedHash(
+            DigestBuilder& hashBuilder,
+            SlangInt entryPointIndex) override;
+
+        virtual void updateASTBasedHash(DigestBuilder& hashBuilder) override
+        {
+            SLANG_UNUSED(hashBuilder);
+        }
+
             /// Get the base (unspecialized) component type that is being specialized.
         RefPtr<ComponentType> getBaseComponentType() { return m_base; }
 
@@ -754,6 +789,16 @@ namespace Slang
 
         void acceptVisitor(ComponentTypeVisitor* visitor, SpecializationInfo* specializationInfo)
             SLANG_OVERRIDE;
+
+        virtual void updateDependencyBasedHash(
+            DigestBuilder& hashBuilder,
+            SlangInt entryPointIndex) override;
+
+        virtual void updateASTBasedHash(DigestBuilder& hashBuilder) override
+        {
+            SLANG_UNUSED(hashBuilder);
+        }
+
     private:
         RefPtr<ComponentType> m_base;
         String m_entryPointNameOverride;
@@ -846,6 +891,28 @@ namespace Slang
             return Super::getEntryPointHostCallable(entryPointIndex, targetIndex, outSharedLibrary, outDiagnostics);
         }
 
+        SLANG_NO_THROW void SLANG_MCALL computeDependencyBasedHash(
+            SlangInt entryPointIndex,
+            SlangInt targetIndex,
+            slang::Digest* outHash) SLANG_OVERRIDE
+        {
+            return Super::computeDependencyBasedHash(entryPointIndex, targetIndex, outHash);
+        }
+
+        SLANG_NO_THROW void SLANG_MCALL computeASTBasedHash(slang::Digest* outHash)
+        {
+            return Super::computeASTBasedHash(outHash);
+        }
+
+        virtual void updateDependencyBasedHash(
+            DigestBuilder& hashBuilder,
+            SlangInt entryPointIndex) override;
+
+        virtual void updateASTBasedHash(DigestBuilder& hashBuilder) override
+        {
+            SLANG_UNUSED(hashBuilder);
+        }
+
             /// Create an entry point that refers to the given function.
         static RefPtr<EntryPoint> create(
             Linkage*            linkage,
@@ -1051,6 +1118,28 @@ namespace Slang
                 entryPointIndex, targetIndex, outSharedLibrary, outDiagnostics);
         }
 
+        SLANG_NO_THROW void SLANG_MCALL computeDependencyBasedHash(
+            SlangInt entryPointIndex,
+            SlangInt targetIndex,
+            slang::Digest* outHash) SLANG_OVERRIDE
+        {
+            return Super::computeDependencyBasedHash(entryPointIndex, targetIndex, outHash);
+        }
+
+        SLANG_NO_THROW void SLANG_MCALL computeASTBasedHash(slang::Digest* outHash)
+        {
+            return Super::computeASTBasedHash(outHash);
+        }
+
+        virtual void updateDependencyBasedHash(
+            DigestBuilder& hashBuilder,
+            SlangInt entryPointIndex) override;
+
+        virtual void updateASTBasedHash(DigestBuilder& hashBuilder) override
+        {
+            SLANG_UNUSED(hashBuilder);
+        }
+
         List<Module*> const& getModuleDependencies() SLANG_OVERRIDE;
         List<String> const& getFilePathDependencies() SLANG_OVERRIDE;
 
@@ -1225,6 +1314,25 @@ namespace Slang
 
         //
 
+        SLANG_NO_THROW void SLANG_MCALL computeDependencyBasedHash(
+            SlangInt entryPointIndex,
+            SlangInt targetIndex,
+            slang::Digest* outHash) SLANG_OVERRIDE
+        {
+            return Super::computeDependencyBasedHash(entryPointIndex, targetIndex, outHash);
+        }
+
+        SLANG_NO_THROW void SLANG_MCALL computeASTBasedHash(slang::Digest* outHash)
+        {
+            return Super::computeASTBasedHash(outHash);
+        }
+
+        virtual void updateDependencyBasedHash(
+            DigestBuilder& hashBuilder,
+            SlangInt entryPointIndex) override;
+
+        virtual void updateASTBasedHash(DigestBuilder& hashBuilder) override;
+
             /// Create a module (initially empty).
         Module(Linkage* linkage, ASTBuilder* astBuilder = nullptr);
 
@@ -1657,6 +1765,13 @@ namespace Slang
         SLANG_NO_THROW SlangResult SLANG_MCALL createCompileRequest(
             SlangCompileRequest**   outCompileRequest) override;
 
+        // Updates the supplied has builder with linkage-related information, which includes preprocessor
+        // defines, the compiler version, and other compiler options. This is then merged with the hash
+        // produced for the program to produce a key that can be used with the shader cache.
+        void updateDependencyBasedHash(
+            DigestBuilder& builder,
+            SlangInt targetIndex);
+
         void addTarget(
             slang::TargetDesc const& desc);
         SlangResult addSearchPath(