Shader caching (#2432)

* Changed all getEntryPointCode calls to use RendererBase::getEntryPointCodeFromShaderCache

* Hashing hooked up, tests pass but need to add more to fully test functionality

* checkpoint

* Checkpoint: File system creation seems functional, saving is broken

* checkpoint: Fixed filename generation from MD5 hash, shader blob might be going missing ahead of pipeline state creation

* Fixed a lot of bugs related to hash code generation, shader cache is likely working but needs further testing

* Added workaround for module loading by re-creating the test device, shader cache test functional

* Vulkan shader caching bug fixed, checkpoint commit before more refinement

* pre-ToT merge checkpoint

* checkpoint commit, improving cache keys

* Significantly expanded items included in the dependency hash for Module; Added dependency hash functions to SpecializedComponentType and RenamedEntryPointComponentType

* Temporarily disable shader cache test

* Mid cleanup changes, solution successfully builds

* Added several helper update functions to slang-md5 to help simplify usage; Added a function under ISession to compute a hash for all linkage-related items; Function renames and cleaned up some comments

* Ran premake.bat; Renamed getASTBasedHashCode to computeASTBasedHash

* Added slang unit tests for Checksum and MD5; Extended gfx shader cache test to test with multiple shader files and one shader file with multiple entry points

* Solution builds and shader cache tests pass, but at least a couple other tests now failing

* ran premake.bat

* More cleanup changes

* Added shaderCachePath field to IDevice desc in gfx.slang, gfx-smoke.slang should be functional

* ran premake

* cleanup changes; Adding test printf to getEntryPointCodeFromShaderCache to see if output can be seen in CI

* Removed debugging printfs; Added handling for getEntryPointCode() failing

* Cleanup changes; Jonathan's fixes to SerialWriter to zero initialize otherwise uninitialized memory; Change to SwizzleExpr creation to zero initialize elementCount

* Changed enable_if_t to enable_if

* Fixed enable_if

* Added test for import vs include and changes to included and imported files; Fixed build errors in CUDA; Renamed shader cache statistics fields

* cleanup changes

* Readd removed file

* Restructured computeDependencyBasedHash calls, added computeDependencyBasedHashImpl to all classes dervied from ComponentType

* Applied same restructuring to the AST hash functions

* Cleanup changes; Moved HashBuilder out to slang-digest.h and added some helper functions to streamline the process of adding items to a hash

* Cleanup; Fixed incorrect expected results for shader import and include test

1 files changed, 172 insertions, 3 deletions

diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp
index a3fd91ce0..a6dcf8ab2 100644
--- a/source/slang/slang.cpp
+++ b/source/slang/slang.cpp
@@ -44,6 +44,9 @@
 
 #include "slang-check-impl.h"
 
+#include "../core/slang-md5.h"
+#include "slang-hash-utils.h"
+
 #include "../../slang-tag-version.h"
 
 // Used to print exception type names in internal-compiler-error messages
@@ -1318,6 +1321,54 @@ SLANG_NO_THROW SlangResult SLANG_MCALL Linkage::createCompileRequest(
     return SLANG_OK;
 }
 
+void Linkage::updateDependencyBasedHash(
+    DigestBuilder& builder,
+    SlangInt targetIndex)
+{
+    // Add the Slang compiler version to the hash
+    auto version = String(getBuildTagString());
+    builder.addToDigest(version);
+
+    // Add the search directory paths to the hash
+    auto searchDirectoryList = getSearchDirectories().searchDirectories;
+    for (auto& searchDir : searchDirectoryList)
+    {
+        auto searchPath = searchDir.path;
+        builder.addToDigest(searchPath);
+    }
+
+    // Add the preprocessor definitions to the hash
+    for (auto& key : preprocessorDefinitions)
+    {
+        builder.addToDigest(key.Key);
+        builder.addToDigest(key.Value);
+    }
+
+    // Add the target specified by targetIndex
+    auto targetReq = targets[targetIndex];
+    builder.addToDigest(targetReq->getTarget());
+    builder.addToDigest(targetReq->getTargetFlags());
+    builder.addToDigest(targetReq->getFloatingPointMode());
+    builder.addToDigest(targetReq->getLineDirectiveMode());
+    builder.addToDigest(targetReq->shouldDumpIntermediates());
+    builder.addToDigest(targetReq->getForceGLSLScalarBufferLayout());
+    builder.addToDigest(targetReq->shouldTrackLiveness());
+
+    auto targetProfile = targetReq->getTargetProfile();
+    builder.addToDigest(targetProfile.getStage());
+    builder.addToDigest(targetProfile.getVersion());
+    builder.addToDigest(targetProfile.getFamily());
+
+    auto targetProfileName = String(targetProfile.getName());
+    builder.addToDigest(targetProfileName);
+
+    auto cookedCapabilities = targetReq->getTargetCaps().getExpandedAtoms();
+    for (auto& capability : cookedCapabilities)
+    {
+        builder.addToDigest(capability);
+    }
+}
+
 SlangResult Linkage::addSearchPath(
     char const* path)
 {
@@ -1618,10 +1669,20 @@ void TranslationUnitRequest::_addSourceFile(SourceFile* sourceFile)
     // an associated path and/or wasn't from a file.
 
     auto pathInfo = sourceFile->getPathInfo();
-    if (pathInfo.hasFileFoundPath())
+    if (pathInfo.hasFoundPath())
     {
         getModule()->addFilePathDependency(pathInfo.foundPath);
     }
+    else
+    {
+        // No path exists for this source, so we generate a new string to use as a
+        // fake path in the list of file path dependencies. This is needed to account
+        // for non-file-based dependencies later when shader files are being hashed for
+        // the shader cache.
+
+        slang::Digest sourceHash = computeHashForStringSlice(sourceFile->getContent());
+        getModule()->addFilePathDependency(hashToString(sourceHash));
+    }
 }
 
 List<SourceFile*> const& TranslationUnitRequest::getSourceFiles() 
@@ -3149,8 +3210,6 @@ void FilePathDependencyList::addDependency(Module* module)
     }
 }
 
-
-
 //
 // Module
 //
@@ -3178,6 +3237,22 @@ ISlangUnknown* Module::getInterface(const Guid& guid)
     return Super::getInterface(guid);
 }
 
+void Module::updateDependencyBasedHash(
+    DigestBuilder& builder,
+    SlangInt entryPointIndex)
+{
+    // CompositeComponentType will have already hashed this Module's file
+    // dependencies.
+    SLANG_UNUSED(builder);
+    SLANG_UNUSED(entryPointIndex);
+}
+
+void Module::updateASTBasedHash(DigestBuilder& builder)
+{
+    auto serializedAST = ASTSerialUtil::serializeAST(getModuleDecl());
+    builder.addToDigest(serializedAST);
+}
+
 void Module::addModuleDependency(Module* module)
 {
     m_moduleDependencyList.addDependency(module);
@@ -3376,6 +3451,52 @@ SLANG_NO_THROW SlangResult SLANG_MCALL ComponentType::getEntryPointCode(
     return artifact->loadBlob(ArtifactKeep::Yes, outCode);
 }
 
+SLANG_NO_THROW void SLANG_MCALL ComponentType::computeDependencyBasedHash(
+    SlangInt entryPointIndex,
+    SlangInt targetIndex,
+    slang::Digest* outHash)
+{
+    DigestBuilder builder;
+
+    // A note on enums that may be hashed in as part of the following two function calls:
+    // 
+    // While enums are not guaranteed to be encoded the same way across all versions of
+    // the compiler, part of hashing the linkage is hashing in the compiler version.
+    // Consequently, any encoding differences as a result of different compiler versions
+    // will already be reflected in the resulting hash.
+    getLinkage()->updateDependencyBasedHash(builder, targetIndex);
+    updateDependencyBasedHash(builder, entryPointIndex);
+
+    // Add file path dependencies to the hash - all child components
+    // will have file path dependencies that are a subset of this list.
+    auto fileDeps = getFilePathDependencies();
+    for (auto& file : fileDeps)
+    {
+        builder.addToDigest(file);
+    }
+
+    // Add the name and name override for the specified entry point
+    // to the hash.
+    auto entryPointName = getEntryPoint(entryPointIndex)->getName()->text;
+    builder.addToDigest(entryPointName);
+    auto entryPointNameOverride = getEntryPointNameOverride(entryPointIndex);
+    builder.addToDigest(entryPointNameOverride);
+
+    slang::Digest hash;
+    builder.finalize(&hash);
+    *outHash = hash;
+}
+
+SLANG_NO_THROW void SLANG_MCALL ComponentType::computeASTBasedHash(slang::Digest* outHash)
+{
+    DigestBuilder builder;
+    updateASTBasedHash(builder);
+
+    slang::Digest hash;
+    builder.finalize(&hash);
+    *outHash = hash;
+}
+
 SLANG_NO_THROW SlangResult SLANG_MCALL ComponentType::getEntryPointHostCallable(
     int                     entryPointIndex,
     int                     targetIndex,
@@ -3708,6 +3829,28 @@ CompositeComponentType::CompositeComponentType(
     }
 }
 
+void CompositeComponentType::updateDependencyBasedHash(
+    DigestBuilder& builder,
+    SlangInt entryPointIndex)
+{
+    auto componentCount = getChildComponentCount();
+
+    for (Index i = 0; i < componentCount; ++i)
+    {
+        getChildComponent(i)->updateDependencyBasedHash(builder, entryPointIndex);
+    }
+}
+
+void CompositeComponentType::updateASTBasedHash(DigestBuilder& builder)
+{
+    auto componentCount = getChildComponentCount();
+
+    for (Index i = 0; i < componentCount; ++i)
+    {
+        getChildComponent(i)->updateASTBasedHash(builder);
+    }
+}
+
 Index CompositeComponentType::getEntryPointCount()
 {
     return m_entryPoints.getCount();
@@ -4187,6 +4330,22 @@ SpecializedComponentType::SpecializedComponentType(
     collector.visitSpecialized(this);
 }
 
+void SpecializedComponentType::updateDependencyBasedHash(
+    DigestBuilder& builder,
+    SlangInt entryPointIndex)
+{
+    auto specializationArgCount = getSpecializationArgCount();
+    for (Index i = 0; i < specializationArgCount; ++i)
+    {
+        auto specializationArg = getSpecializationArg(i);
+        auto argString = specializationArg.val->toString();
+        builder.addToDigest(argString);
+    }
+
+    slang::Digest baseHash;
+    getBaseComponentType()->updateDependencyBasedHash(builder, entryPointIndex);
+}
+
 void SpecializedComponentType::acceptVisitor(ComponentTypeVisitor* visitor, SpecializationInfo* specializationInfo)
 {
     SLANG_ASSERT(specializationInfo == nullptr);
@@ -4231,6 +4390,16 @@ void RenamedEntryPointComponentType::acceptVisitor(
         this, as<EntryPoint::EntryPointSpecializationInfo>(specializationInfo));
 }
 
+void RenamedEntryPointComponentType::updateDependencyBasedHash(
+    DigestBuilder& builder,
+    SlangInt entryPointIndex)
+{
+    // CompositeComponentType will have already hashed the name override and file
+    // dependencies for this entry point.
+    SLANG_UNUSED(entryPointIndex);
+    SLANG_UNUSED(builder);
+}
+
 void ComponentTypeVisitor::visitChildren(CompositeComponentType* composite, CompositeComponentType::CompositeSpecializationInfo* specializationInfo)
 {
     auto childCount = composite->getChildComponentCount();