From 8add41a6e37994577d928bc312801ddfa1c33173 Mon Sep 17 00:00:00 2001 From: lucy96chen <47800040+lucy96chen@users.noreply.github.com> Date: Mon, 17 Oct 2022 17:38:59 -0700 Subject: Shader cache index implementation (#2452) --- build/visual-studio/core/core.vcxproj | 4 +- build/visual-studio/core/core.vcxproj.filters | 8 +- build/visual-studio/gfx/gfx.vcxproj | 2 + build/visual-studio/gfx/gfx.vcxproj.filters | 6 + build/visual-studio/slang-rt/slang-rt.vcxproj | 4 +- .../slang-rt/slang-rt.vcxproj.filters | 8 +- .../slang-unit-test-tool.vcxproj | 2 +- .../slang-unit-test-tool.vcxproj.filters | 6 +- build/visual-studio/slang/slang.vcxproj | 2 - build/visual-studio/slang/slang.vcxproj.filters | 6 - slang.h | 13 ++ source/compiler-core/slang-dxc-compiler.cpp | 2 +- source/compiler-core/slang-glslang-compiler.cpp | 2 +- source/core/slang-digest-builder.h | 35 ++++ source/core/slang-digest-util.cpp | 66 ++++++++ source/core/slang-digest-util.h | 43 +++++ source/core/slang-digest.h | 35 ---- source/core/slang-linked-list.h | 44 +++-- source/slang/slang-ast-base.h | 2 - source/slang/slang-ast-val.h | 1 - source/slang/slang-compiler.h | 8 +- source/slang/slang-hash-utils.h | 49 ------ source/slang/slang-shader-cache-index.h | 60 ------- source/slang/slang.cpp | 6 +- tools/gfx/persistent-shader-cache.cpp | 182 +++++++++++++++++++++ tools/gfx/persistent-shader-cache.h | 79 +++++++++ tools/gfx/renderer-shared.cpp | 7 +- tools/gfx/renderer-shared.h | 2 + tools/slang-unit-test/unit-test-checksum.cpp | 32 ---- tools/slang-unit-test/unit-test-digest-utils.cpp | 58 +++++++ tools/slang-unit-test/unit-test-md5.cpp | 12 +- 31 files changed, 558 insertions(+), 228 deletions(-) create mode 100644 source/core/slang-digest-builder.h create mode 100644 source/core/slang-digest-util.cpp create mode 100644 source/core/slang-digest-util.h delete mode 100644 source/core/slang-digest.h delete mode 100644 source/slang/slang-hash-utils.h delete mode 100644 source/slang/slang-shader-cache-index.h create mode 100644 tools/gfx/persistent-shader-cache.cpp create mode 100644 tools/gfx/persistent-shader-cache.h delete mode 100644 tools/slang-unit-test/unit-test-checksum.cpp create mode 100644 tools/slang-unit-test/unit-test-digest-utils.cpp diff --git a/build/visual-studio/core/core.vcxproj b/build/visual-studio/core/core.vcxproj index f84695880..e375b52a6 100644 --- a/build/visual-studio/core/core.vcxproj +++ b/build/visual-studio/core/core.vcxproj @@ -270,7 +270,8 @@ - + + @@ -333,6 +334,7 @@ + diff --git a/build/visual-studio/core/core.vcxproj.filters b/build/visual-studio/core/core.vcxproj.filters index 14d866c12..068023108 100644 --- a/build/visual-studio/core/core.vcxproj.filters +++ b/build/visual-studio/core/core.vcxproj.filters @@ -69,7 +69,10 @@ Header Files - + + Header Files + + Header Files @@ -254,6 +257,9 @@ Source Files + + Source Files + Source Files diff --git a/build/visual-studio/gfx/gfx.vcxproj b/build/visual-studio/gfx/gfx.vcxproj index db0dfc5e4..d7e9b9577 100644 --- a/build/visual-studio/gfx/gfx.vcxproj +++ b/build/visual-studio/gfx/gfx.vcxproj @@ -410,6 +410,7 @@ IF EXIST "$(SolutionDir)tools\gfx\slang.slang"\ (xcopy /Q /E /Y /I "$(SolutionDi + @@ -520,6 +521,7 @@ IF EXIST "$(SolutionDir)tools\gfx\slang.slang"\ (xcopy /Q /E /Y /I "$(SolutionDi + diff --git a/build/visual-studio/gfx/gfx.vcxproj.filters b/build/visual-studio/gfx/gfx.vcxproj.filters index 1a7ed3d03..c708450d5 100644 --- a/build/visual-studio/gfx/gfx.vcxproj.filters +++ b/build/visual-studio/gfx/gfx.vcxproj.filters @@ -306,6 +306,9 @@ Header Files + + Header Files + Header Files @@ -632,6 +635,9 @@ Source Files + + Source Files + Source Files diff --git a/build/visual-studio/slang-rt/slang-rt.vcxproj b/build/visual-studio/slang-rt/slang-rt.vcxproj index c30f5f077..3a8352337 100644 --- a/build/visual-studio/slang-rt/slang-rt.vcxproj +++ b/build/visual-studio/slang-rt/slang-rt.vcxproj @@ -282,7 +282,8 @@ - + + @@ -346,6 +347,7 @@ + diff --git a/build/visual-studio/slang-rt/slang-rt.vcxproj.filters b/build/visual-studio/slang-rt/slang-rt.vcxproj.filters index 65f4d5221..dd72f5178 100644 --- a/build/visual-studio/slang-rt/slang-rt.vcxproj.filters +++ b/build/visual-studio/slang-rt/slang-rt.vcxproj.filters @@ -69,7 +69,10 @@ Header Files - + + Header Files + + Header Files @@ -257,6 +260,9 @@ Source Files + + Source Files + Source Files diff --git a/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj b/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj index 83681488f..9253b2930 100644 --- a/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj +++ b/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj @@ -272,11 +272,11 @@ - + diff --git a/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj.filters b/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj.filters index b83a18b76..b4ce5c734 100644 --- a/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj.filters +++ b/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj.filters @@ -17,9 +17,6 @@ Source Files - - Source Files - Source Files @@ -32,6 +29,9 @@ Source Files + + Source Files + Source Files diff --git a/build/visual-studio/slang/slang.vcxproj b/build/visual-studio/slang/slang.vcxproj index a6cf0846d..7436f319e 100644 --- a/build/visual-studio/slang/slang.vcxproj +++ b/build/visual-studio/slang/slang.vcxproj @@ -327,7 +327,6 @@ IF EXIST ..\..\..\external\slang-glslang\bin\windows-aarch64\release\slang-glsla - @@ -446,7 +445,6 @@ IF EXIST ..\..\..\external\slang-glslang\bin\windows-aarch64\release\slang-glsla - diff --git a/build/visual-studio/slang/slang.vcxproj.filters b/build/visual-studio/slang/slang.vcxproj.filters index ff315e0a7..02ef9de2e 100644 --- a/build/visual-studio/slang/slang.vcxproj.filters +++ b/build/visual-studio/slang/slang.vcxproj.filters @@ -114,9 +114,6 @@ Header Files - - Header Files - Header Files @@ -471,9 +468,6 @@ Header Files - - Header Files - Header Files diff --git a/slang.h b/slang.h index 08279a1de..8a0813f7b 100644 --- a/slang.h +++ b/slang.h @@ -4139,6 +4139,19 @@ namespace slang struct Digest { uint32_t values[4] = { 0 }; + + bool operator==(const Digest& rhs) + { + return values[0] == rhs.values[0] + && values[1] == rhs.values[1] + && values[2] == rhs.values[2] + && values[3] == rhs.values[3]; + } + + uint32_t getHashCode() + { + return values[0]; + } }; /** A session provides a scope for code that is loaded. diff --git a/source/compiler-core/slang-dxc-compiler.cpp b/source/compiler-core/slang-dxc-compiler.cpp index 52933eaa7..06dfc74b0 100644 --- a/source/compiler-core/slang-dxc-compiler.cpp +++ b/source/compiler-core/slang-dxc-compiler.cpp @@ -14,7 +14,7 @@ #include "../core/slang-semantic-version.h" #include "../core/slang-char-util.h" -#include "../core/slang-digest.h" +#include "../core/slang-digest-builder.h" #include "slang-include-system.h" #include "slang-source-loc.h" diff --git a/source/compiler-core/slang-glslang-compiler.cpp b/source/compiler-core/slang-glslang-compiler.cpp index 26cbf3292..6e11d9814 100644 --- a/source/compiler-core/slang-glslang-compiler.cpp +++ b/source/compiler-core/slang-glslang-compiler.cpp @@ -14,7 +14,7 @@ #include "../core/slang-semantic-version.h" #include "../core/slang-char-util.h" -#include "../core/slang-digest.h" +#include "../core/slang-digest-builder.h" #include "slang-artifact-associated-impl.h" #include "slang-artifact-desc-util.h" diff --git a/source/core/slang-digest-builder.h b/source/core/slang-digest-builder.h new file mode 100644 index 000000000..47c0cd8f7 --- /dev/null +++ b/source/core/slang-digest-builder.h @@ -0,0 +1,35 @@ +#pragma once +#include "slang-md5.h" +#include "../../slang.h" + +namespace Slang +{ + using slang::Digest; + + // Wrapper struct that holds objects necessary for hashing. + struct DigestBuilder + { + public: + DigestBuilder() + { + hashGen.init(&context); + } + + template + void addToDigest(T item) + { + hashGen.update(&context, item); + } + + Digest finalize() + { + Digest hash; + hashGen.finalize(&context, &hash); + return hash; + } + + private: + MD5HashGen hashGen; + MD5Context context; + }; +} diff --git a/source/core/slang-digest-util.cpp b/source/core/slang-digest-util.cpp new file mode 100644 index 000000000..d235f2cf0 --- /dev/null +++ b/source/core/slang-digest-util.cpp @@ -0,0 +1,66 @@ +// string-digest-util.cpp +#include "slang-digest-util.h" + +#include "../core/slang-basic.h" +#include "../core/slang-digest-builder.h" +#include "../core/slang-md5.h" +#include "../core/slang-char-util.h" + +namespace Slang +{ + +/*static*/ Digest DigestUtil::computeDigestForStringSlice(UnownedStringSlice text) +{ + DigestBuilder builder; + builder.addToDigest(text); + return builder.finalize(); +} + +/*static*/ Digest DigestUtil::combine(const Digest& digestA, const Digest& digestB) +{ + DigestBuilder builder; + builder.addToDigest(digestA); + builder.addToDigest(digestB); + return builder.finalize(); +} + +/*static*/ String DigestUtil::toString(const Digest& digest) +{ + StringBuilder hashString; + + uint8_t* uint8Hash = (uint8_t*)digest.values; + + for (Index i = 0; i < 16; ++i) + { + auto hashSegmentString = String(uint8Hash[i], 16); + + if (hashSegmentString.getLength() == 1) + { + hashString.append("0"); + } + hashString.append(hashSegmentString.getBuffer()); + } + + return hashString; +} + +/*static*/ Digest DigestUtil::fromString(UnownedStringSlice hashString) +{ + uint8_t uint8Hash[16]; + + // When the hash is converted to a String, ReverseInternalAscii is called + // at the very end. Since there is no way to get a char* for hashString to pass + // to ReverseInternalAscii to flip the string back, we instead loop starting from + // the end and work backwards towards the beginning. + for (Index i = 0; i < 16; i++) + { + uint8Hash[i] = (uint8_t)CharUtil::getHexDigitValue(hashString[i * 2]) * 16 + + (uint8_t)CharUtil::getHexDigitValue(hashString[i * 2 + 1]); + } + + Digest digest; + memcpy(digest.values, uint8Hash, 16); + return digest; +} + +} diff --git a/source/core/slang-digest-util.h b/source/core/slang-digest-util.h new file mode 100644 index 000000000..1e272fd5e --- /dev/null +++ b/source/core/slang-digest-util.h @@ -0,0 +1,43 @@ +// slang-digest-utils.h - Utility functions specifically designed to be used with slang::Digest +#pragma once +#include "../../slang.h" +#include "slang-string.h" + +namespace Slang +{ + using slang::Digest; + + struct DigestUtil + { + // Compute the digest for an UnownedStringSlice + static Digest computeDigestForStringSlice(UnownedStringSlice text); + + // Combines the two provided digests. + static Digest combine(const Digest& digestA, const Digest& digestB); + + // Returns the hash stored in digest as a String. + static String toString(const Digest& digest); + + // Returns the hash represented by hashString as a Digest. + static Digest fromString(UnownedStringSlice hashString); + }; + + inline StringBuilder& operator<<(StringBuilder& sb, const Digest& d) + { + // Must cast to uint8_t* first in order to correctly account for + // endianness. + uint8_t* uint8Hash = (uint8_t*)d.values; + + for (Index i = 0; i < 16; ++i) + { + int hashSegment = uint8Hash[i]; + // Check if we need to append a leading zero. + if (hashSegment < 16) + { + sb << "0"; + } + sb.append(hashSegment, 16); + } + return sb; + } +} diff --git a/source/core/slang-digest.h b/source/core/slang-digest.h deleted file mode 100644 index 47c0cd8f7..000000000 --- a/source/core/slang-digest.h +++ /dev/null @@ -1,35 +0,0 @@ -#pragma once -#include "slang-md5.h" -#include "../../slang.h" - -namespace Slang -{ - using slang::Digest; - - // Wrapper struct that holds objects necessary for hashing. - struct DigestBuilder - { - public: - DigestBuilder() - { - hashGen.init(&context); - } - - template - void addToDigest(T item) - { - hashGen.update(&context, item); - } - - Digest finalize() - { - Digest hash; - hashGen.finalize(&context, &hash); - return hash; - } - - private: - MD5HashGen hashGen; - MD5Context context; - }; -} diff --git a/source/core/slang-linked-list.h b/source/core/slang-linked-list.h index 7c9987397..38da2ccce 100644 --- a/source/core/slang-linked-list.h +++ b/source/core/slang-linked-list.h @@ -243,6 +243,12 @@ public: { LinkedNode* n = new LinkedNode(this); n->Value = nData; + AddFirst(n); + count++; + return n; + }; + void AddFirst(LinkedNode* n) + { n->prev = 0; n->next = head; if (head) @@ -250,24 +256,12 @@ public: head = n; if (!tail) tail = n; - count++; - return n; - }; - void Delete(LinkedNode* n, int Count = 1) + } + void RemoveFromList(LinkedNode* n) { - LinkedNode*n1, *n2 = 0, *tn; + LinkedNode*n1, *n2 = 0; n1 = n->prev; - tn = n; - int numDeleted = 0; - for (int i = 0; i < Count; i++) - { - n2 = tn->next; - delete tn; - tn = n2; - numDeleted++; - if (tn == 0) - break; - } + n2 = n->next; if (n1) n1->next = n2; else @@ -276,6 +270,24 @@ public: n2->prev = n1; else tail = n1; + n->prev = nullptr; + n->next = nullptr; + } + void Delete(LinkedNode* n, int Count = 1) + { + LinkedNode*cur, *next; + cur = n; + int numDeleted = 0; + for (int i = 0; i < Count; i++) + { + next = cur->next; + RemoveFromList(cur); + delete cur; + cur = next; + numDeleted++; + if (cur == 0) + break; + } count -= numDeleted; } void Clear() diff --git a/source/slang/slang-ast-base.h b/source/slang/slang-ast-base.h index 04788340a..dea02afbb 100644 --- a/source/slang/slang-ast-base.h +++ b/source/slang/slang-ast-base.h @@ -9,8 +9,6 @@ #include "slang-serialize-reflection.h" -#include "../core/slang-digest.h" - // This file defines the primary base classes for the hierarchy of // AST nodes and related objects. For example, this is where the // basic `Decl`, `Stmt`, `Expr`, `type`, etc. definitions come from. diff --git a/source/slang/slang-ast-val.h b/source/slang/slang-ast-val.h index 49189f65c..5f72e58c8 100644 --- a/source/slang/slang-ast-val.h +++ b/source/slang/slang-ast-val.h @@ -3,7 +3,6 @@ #pragma once #include "slang-ast-base.h" -#include "../core/slang-digest.h" namespace Slang { diff --git a/source/slang/slang-compiler.h b/source/slang/slang-compiler.h index 016c8fefa..bd7cd4a18 100755 --- a/source/slang/slang-compiler.h +++ b/source/slang/slang-compiler.h @@ -3,7 +3,7 @@ #include "../core/slang-basic.h" #include "../core/slang-shared-library.h" -#include "../core/slang-digest.h" +#include "../core/slang-digest-builder.h" #include "../compiler-core/slang-downstream-compiler.h" #include "../compiler-core/slang-downstream-compiler-util.h" @@ -899,7 +899,7 @@ namespace Slang return Super::computeDependencyBasedHash(entryPointIndex, targetIndex, outHash); } - SLANG_NO_THROW void SLANG_MCALL computeASTBasedHash(slang::Digest* outHash) + SLANG_NO_THROW void SLANG_MCALL computeASTBasedHash(slang::Digest* outHash) SLANG_OVERRIDE { return Super::computeASTBasedHash(outHash); } @@ -1126,7 +1126,7 @@ namespace Slang return Super::computeDependencyBasedHash(entryPointIndex, targetIndex, outHash); } - SLANG_NO_THROW void SLANG_MCALL computeASTBasedHash(slang::Digest* outHash) + SLANG_NO_THROW void SLANG_MCALL computeASTBasedHash(slang::Digest* outHash) SLANG_OVERRIDE { return Super::computeASTBasedHash(outHash); } @@ -1322,7 +1322,7 @@ namespace Slang return Super::computeDependencyBasedHash(entryPointIndex, targetIndex, outHash); } - SLANG_NO_THROW void SLANG_MCALL computeASTBasedHash(slang::Digest* outHash) + SLANG_NO_THROW void SLANG_MCALL computeASTBasedHash(slang::Digest* outHash) SLANG_OVERRIDE { return Super::computeASTBasedHash(outHash); } diff --git a/source/slang/slang-hash-utils.h b/source/slang/slang-hash-utils.h deleted file mode 100644 index c354e43f0..000000000 --- a/source/slang/slang-hash-utils.h +++ /dev/null @@ -1,49 +0,0 @@ -// slang-hash-utils.h - Utility functions specifically designed to be used with slang::Digest -#pragma once -#include "../../slang.h" -#include "../core/slang-basic.h" -#include "../core/slang-md5.h" -#include "../core/slang-digest.h" - -namespace Slang -{ - // Compute the hash for an UnownedStringSlice - inline slang::Digest computeHashForStringSlice(UnownedStringSlice text) - { - DigestBuilder builder; - builder.addToDigest(text); - return builder.finalize(); - } - - // Combines the two provided hashes. - inline slang::Digest combineHashes(const slang::Digest& hashA, const slang::Digest& hashB) - { - DigestBuilder builder; - builder.addToDigest(hashA); - builder.addToDigest(hashB); - return builder.finalize(); - } - - // Returns the stored hash in checksum as a String. - inline StringBuilder hashToString(const slang::Digest& hash) - { - StringBuilder filename; - - uint8_t* uint8Hash = (uint8_t*)hash.values; - - for (Index i = 0; i < 16; ++i) - { - auto hashSegmentString = String(uint8Hash[i], 16); - - if (hashSegmentString.getLength() == 1) - { - filename.append("0"); - } - filename.append(hashSegmentString.getBuffer()); - } - - return filename; - } - - // TODO: fromString implementation? -} diff --git a/source/slang/slang-shader-cache-index.h b/source/slang/slang-shader-cache-index.h deleted file mode 100644 index 530aacf11..000000000 --- a/source/slang/slang-shader-cache-index.h +++ /dev/null @@ -1,60 +0,0 @@ -#pragma once -#include "../../slang.h" - -#include "../core/slang-string.h" -#include "../core/slang-linked-list.h" -#include "../core/slang-dictionary.h" - -namespace Slang -{ - class ShaderCacheIndex - { - public: - struct ShaderCacheEntry - { - slang::Digest dependencyBasedDigest; - slang::Digest astBasedDigest; - }; - - ShaderCacheIndex(SlangInt size) - : entryCountLimit(size) - {} - - // Load a previous cache index saved to disk. If not found, create a new cache index - // and save it to disk as filename. - SlangResult loadCacheIndexFromFile(String filename); - - // Fetch the cache entry corresponding to the provided key. If found, move the entry to - // the front of entries and return the entry. Else, return nullptr. - ShaderCacheEntry* findEntry(const slang::Digest& key); - - // Add an entry to the cache with the provided key and contents hashes. If - // adding an entry causes the cache to exceed size limitations, this will also - // delete the least recently used entry. - void addEntry(const slang::Digest& dependencyDigest, const slang::Digest& astDigest); - - // Update the contents hash for the specified entry in the cache and update the - // corresponding file on disk. - void updateEntry(const slang::Digest& dependencyDigest, const slang::Digest& astDigest); - - private: - // Update the cache index on disk. This should be called any time an entry changes. - SlangResult saveCacheIndexToFile(); - - // Delete the last entry from entries and remove its key/value pair from keyToContents - // as well as remove the corresponding file on disk. This should only be used when the - // cache has reached the size limit specified by entryLimit to create space for a new entry. - void deleteEntry(); - - // Dictionary mapping each shader's key to its corresponding node (entry) in the list - // of entries. - Dictionary*> keyToEntry; - - // Linked list containing the entries stored in the shader cache in order - // of most to least recently used. - LinkedList entries; - - // The maximum number of cache entries allowed. - SlangInt entryCountLimit = -1; - } -} diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp index 3fb89c549..c7351d6a8 100644 --- a/source/slang/slang.cpp +++ b/source/slang/slang.cpp @@ -45,7 +45,7 @@ #include "slang-check-impl.h" #include "../core/slang-md5.h" -#include "slang-hash-utils.h" +#include "../core/slang-digest-util.h" #include "../../slang-tag-version.h" @@ -1692,8 +1692,8 @@ void TranslationUnitRequest::_addSourceFile(SourceFile* sourceFile) // for non-file-based dependencies later when shader files are being hashed for // the shader cache. - slang::Digest sourceHash = computeHashForStringSlice(sourceFile->getContent()); - getModule()->addFilePathDependency(hashToString(sourceHash)); + slang::Digest sourceHash = DigestUtil::computeDigestForStringSlice(sourceFile->getContent()); + getModule()->addFilePathDependency(DigestUtil::toString(sourceHash)); } } diff --git a/tools/gfx/persistent-shader-cache.cpp b/tools/gfx/persistent-shader-cache.cpp new file mode 100644 index 000000000..a378e0f4f --- /dev/null +++ b/tools/gfx/persistent-shader-cache.cpp @@ -0,0 +1,182 @@ +// slang-shader-cache-index.cpp +#include "persistent-shader-cache.h" + +#include "../../source/core/slang-digest-util.h" +#include "../../source/core/slang-io.h" +#include "../../source/core/slang-string-util.h" +#include "../../source/core/slang-file-system.h" + +namespace Slang +{ + +PersistentShaderCache::PersistentShaderCache(const Desc& inDesc) +{ + desc = inDesc; + + if (!desc.shaderCachePath.getBuffer()) + { + if (!desc.shaderCacheFileSystem) + { + // Only a path was provided, so we get a mutable file system + // using OSFileSystem::getMutableSingleton. + desc.shaderCacheFileSystem = OSFileSystem::getMutableSingleton(); + } + desc.shaderCacheFileSystem = new RelativeFileSystem(desc.shaderCacheFileSystem, desc.shaderCachePath); + } + + // If our shader cache has an underlying file system, check if it's mutable. If so, store a pointer + // to the mutable version in order to save new entries later. + if (desc.shaderCacheFileSystem) + { + desc.shaderCacheFileSystem->queryInterface(ISlangMutableFileSystem::getTypeGuid(), (void**)mutableShaderCacheFileSystem.writeRef()); + } + + loadCacheFromFile(); +} + +// Load a previous cache index saved to disk. If not found, create a new cache index +// and save it to disk as filename. +void PersistentShaderCache::loadCacheFromFile() +{ + ComPtr indexBlob; + if (SLANG_FAILED(desc.shaderCacheFileSystem->loadFile(desc.cacheFilename.getBuffer(), indexBlob.writeRef()))) + { + // Cache index not found, so we'll create and save a new one. + if (mutableShaderCacheFileSystem) + { + mutableShaderCacheFileSystem->saveFile(desc.cacheFilename.getBuffer(), nullptr, 0); + return; + } + // Cache index not found and we can't save a new one due to the file system being immutable. + return; + } + + String indexString; + File::readAllText(desc.cacheFilename, indexString); + + List lines; + StringUtil::calcLines(indexString.getUnownedSlice(), lines); + for (auto line : lines) + { + List digests; + StringUtil::split(line, ' ', digests); // This will return our two hashes as two elements in digests. + auto dependencyDigest = DigestUtil::fromString(digests[0]); + auto astDigest = DigestUtil::fromString(digests[1]); + + ShaderCacheEntry entry = { dependencyDigest, astDigest }; + auto entryNode = entries.AddLast(entry); + keyToEntry.Add(dependencyDigest, entryNode); + } +} + +LinkedNode* PersistentShaderCache::findEntry(const slang::Digest& key, ISlangBlob** outCompiledCode) +{ + LinkedNode* entryNode; + if (!keyToEntry.TryGetValue(key, entryNode)) + { + // The key was not found in the cache, so we return nullptr. + *outCompiledCode = nullptr; + return nullptr; + } + + // If the key is found, load the stored contents from disk. We then move the corresponding + // entry to the front of the linked list and update the cache file on disk + desc.shaderCacheFileSystem->loadFile(DigestUtil::toString(key).getBuffer(), outCompiledCode); + if (entries.FirstNode() != entryNode) + { + entries.RemoveFromList(entryNode); + entries.AddFirst(entryNode); + if (mutableShaderCacheFileSystem) + { + saveCacheToFile(); + } + } + return entryNode; +} + +void PersistentShaderCache::addEntry(const slang::Digest& dependencyDigest, const slang::Digest& astDigest, ISlangBlob* compiledCode) +{ + if (!mutableShaderCacheFileSystem) + { + // Should not save new entries if the underlying file system isn't mutable. + return; + } + + // Check that we do not exceed the cache's size limit by adding another entry. If so, + // remove the least recently used entry first. + // + // In theory, this could loop infinitely since deleteLRUEntry() immediately returns if + // mutableShaderCacheFileSystem is not set. However, this situation is functionally impossible + // because we check immediately before this as well. + while (desc.entryCountLimit > 0 && entries.Count() >= desc.entryCountLimit) + { + deleteLRUEntry(); + } + + ShaderCacheEntry entry = { dependencyDigest, astDigest }; + auto entryNode = entries.AddFirst(entry); + keyToEntry.Add(dependencyDigest, entryNode); + + mutableShaderCacheFileSystem->saveFileBlob(DigestUtil::toString(dependencyDigest).getBuffer(), compiledCode); + + saveCacheToFile(); +} + +void PersistentShaderCache::updateEntry( + LinkedNode* entryNode, + const slang::Digest& dependencyDigest, + const slang::Digest& astDigest, + ISlangBlob* updatedCode) +{ + if (!mutableShaderCacheFileSystem) + { + // Updating entries requires saving to disk in order to overwrite the old shader file + // on disk, so we return if the underlying file system isn't mutable. + return; + } + + entryNode->Value.astBasedDigest = astDigest; + mutableShaderCacheFileSystem->saveFileBlob(DigestUtil::toString(dependencyDigest).getBuffer(), updatedCode); + + saveCacheToFile(); +} + +void PersistentShaderCache::saveCacheToFile() +{ + if (!mutableShaderCacheFileSystem) + { + // Cannot save the index to disk if the underlying file system isn't mutable. + return; + } + + StringBuilder indexSb; + for (auto& entry : entries) + { + indexSb << entry.dependencyBasedDigest; + indexSb << " "; + indexSb << entry.astBasedDigest; + indexSb << "\n"; + } + + mutableShaderCacheFileSystem->saveFile(desc.cacheFilename.getBuffer(), indexSb.getBuffer(), indexSb.getLength()); +} + +void PersistentShaderCache::deleteLRUEntry() +{ + if (!mutableShaderCacheFileSystem) + { + // This is here as a safety precaution but should never be hit as + // addEntry() is the only function that should call this. + return; + } + + auto lruEntry = entries.LastNode(); + auto shaderKey = lruEntry->Value.dependencyBasedDigest; + + keyToEntry.Remove(shaderKey); + mutableShaderCacheFileSystem->remove(DigestUtil::toString(shaderKey).getBuffer()); + + entries.Delete(lruEntry); +} + +} diff --git a/tools/gfx/persistent-shader-cache.h b/tools/gfx/persistent-shader-cache.h new file mode 100644 index 000000000..55738a9af --- /dev/null +++ b/tools/gfx/persistent-shader-cache.h @@ -0,0 +1,79 @@ +// slang-shader-cache-index.h +#pragma once +#include "../../slang.h" +#include "../../slang-com-ptr.h" + +#include "../../source/core/slang-string.h" +#include "../../source/core/slang-dictionary.h" +#include "../../source/core/slang-linked-list.h" + +namespace Slang +{ + +struct ShaderCacheEntry +{ + slang::Digest dependencyBasedDigest; + slang::Digest astBasedDigest; +}; + +class PersistentShaderCache +{ +public: + // TODO: Remove in integration PR in favor of new ShaderCacheDesc in slang-gfx.h + struct Desc + { + String cacheFilename; + String shaderCachePath; + SlangInt entryCountLimit = 1000; + ISlangFileSystem* shaderCacheFileSystem = nullptr; + }; + + PersistentShaderCache(const Desc& inDesc); + + // Fetch the cache entry corresponding to the provided key. If found, move the entry to + // the front of entries and return the entry and the corresponding compiled code in + // outCompiledCode. Else, return nullptr. + LinkedNode* findEntry(const slang::Digest& key, ISlangBlob** outCompiledCode); + + // Add an entry to the cache with the provided key and contents hashes. If + // adding an entry causes the cache to exceed size limitations, this will also + // delete the least recently used entry. + void addEntry(const slang::Digest& dependencyDigest, const slang::Digest& astDigest, ISlangBlob* compiledCode); + + // Update the contents hash for the specified entry in the cache and update the + // corresponding file on disk. + void updateEntry( + LinkedNode* entryNode, + const slang::Digest& dependencyDigest, + const slang::Digest& astDigest, + ISlangBlob* updatedCode); + +private: + // Load a previous cache index saved to disk. If not found, create a new cache index + // and save it to disk as filename. + void loadCacheFromFile(); + + // Update the cache index on disk. This should be called any time an entry changes. + void saveCacheToFile(); + + // Delete the last entry (the least recently used) from entries, remove its key/value pair + // from keyToEntry, and remove the corresponding file on disk. This should only be called + // by addEntry() when the cache reaches maximum capacity. + void deleteLRUEntry(); + + // The shader cache's description. + Desc desc; + + // Dictionary mapping each shader's key to its corresponding node (entry) in the list + // of entries. + Dictionary*> keyToEntry; + + // Linked list containing the entries stored in the shader cache in order + // of most to least recently used. + LinkedList entries; + + // The underlying file system used for the shader cache. + ComPtr mutableShaderCacheFileSystem = nullptr; +}; + +} diff --git a/tools/gfx/renderer-shared.cpp b/tools/gfx/renderer-shared.cpp index f76fa2325..48cd6907f 100644 --- a/tools/gfx/renderer-shared.cpp +++ b/tools/gfx/renderer-shared.cpp @@ -6,7 +6,7 @@ #include "../../source/core/slang-file-system.h" #include "../../slang.h" -#include "../../source/slang/slang-hash-utils.h" +#include "../../source/core/slang-digest-util.h" using namespace Slang; @@ -368,7 +368,7 @@ Result RendererBase::getEntryPointCodeFromShaderCache( slang::Digest shaderKeyHash; program->computeDependencyBasedHash(entryPointIndex, targetIndex, &shaderKeyHash); - StringBuilder shaderKey = hashToString(shaderKeyHash); + String shaderKey = DigestUtil::toString(shaderKeyHash); // Produce a hash using the AST for this program - This is needed to check whether a cache entry is effectively dirty, // or to save along with the compiled code into an entry so the entry can be checked if fetched later on. @@ -463,6 +463,9 @@ IDevice* gfx::RendererBase::getInterface(const Guid& guid) SLANG_NO_THROW Result SLANG_MCALL RendererBase::initialize(const Desc& desc) { + // TODO: This logic for initalizing the shader cache has been replicated inside + // the constructor for ShaderCacheIndex. Remove when ShaderCacheIndex is integrated in. + // If a shader cache file system was provided, use the provided system. if (desc.shaderCacheFileSystem) { diff --git a/tools/gfx/renderer-shared.h b/tools/gfx/renderer-shared.h index b140e1f40..fc1c35d20 100644 --- a/tools/gfx/renderer-shared.h +++ b/tools/gfx/renderer-shared.h @@ -1372,6 +1372,8 @@ public: SlangContext slangContext; ShaderCache shaderCache; + // TODO: These should be removed when ShaderCacheIndex is ready to be integrated. ShaderCacheIndex + // will be responsible for keeping track of the underlying filesystem for the cache. ISlangFileSystem* shaderCacheFileSystem = nullptr; ComPtr mutableShaderCacheFileSystem = nullptr; diff --git a/tools/slang-unit-test/unit-test-checksum.cpp b/tools/slang-unit-test/unit-test-checksum.cpp deleted file mode 100644 index 90f426f94..000000000 --- a/tools/slang-unit-test/unit-test-checksum.cpp +++ /dev/null @@ -1,32 +0,0 @@ -// unit-test-checksum.cpp - -#include "tools/unit-test/slang-unit-test.h" - -#include "../../source/slang/slang-hash-utils.h" - -using namespace Slang; - -SLANG_UNIT_TEST(checksum) -{ - { - slang::Digest testA; - testA.values[0] = 1; - testA.values[1] = 2; - testA.values[2] = 3; - testA.values[3] = 4; - - String testAString = hashToString(testA); - SLANG_CHECK(testAString.equals(String("01000000020000000300000004000000"))); - } - - { - slang::Digest testC; - testC.values[0] = 0x11111111; - testC.values[1] = 0x22222222; - testC.values[2] = 0x33333333; - testC.values[3] = 0x44444444; - - String testCString = hashToString(testC); - SLANG_CHECK(testCString.equals(String("11111111222222223333333344444444"))); - } -} diff --git a/tools/slang-unit-test/unit-test-digest-utils.cpp b/tools/slang-unit-test/unit-test-digest-utils.cpp new file mode 100644 index 000000000..a463fe07f --- /dev/null +++ b/tools/slang-unit-test/unit-test-digest-utils.cpp @@ -0,0 +1,58 @@ +// unit-test-digest-utils.cpp + +#include "tools/unit-test/slang-unit-test.h" + +#include "../../source/core/slang-digest-util.h" + +using namespace Slang; + +SLANG_UNIT_TEST(digestUtils) +{ + { + slang::Digest testA; + testA.values[0] = 1; + testA.values[1] = 2; + testA.values[2] = 3; + testA.values[3] = 4; + + String testAString = DigestUtil::toString(testA); + SLANG_CHECK(testAString.equals(String("01000000020000000300000004000000"))); + } + + { + slang::Digest testC; + testC.values[0] = 0x11111111; + testC.values[1] = 0x22222222; + testC.values[2] = 0x33333333; + testC.values[3] = 0x44444444; + + String testCString = DigestUtil::toString(testC); + SLANG_CHECK(testCString.equals(String("11111111222222223333333344444444"))); + } + + { + auto digestString = UnownedStringSlice("5D6CC58E1824A4DFD0CF57395B603316"); + slang::Digest digest = DigestUtil::fromString(digestString); + auto resultString = DigestUtil::toString(digest); + SLANG_CHECK(resultString == digestString); + } + + { + auto digestString = UnownedStringSlice("01000000020000000300000004000000"); + slang::Digest digest = DigestUtil::fromString(digestString); + auto resultString = DigestUtil::toString(digest); + SLANG_CHECK(resultString == digestString); + } + + { + slang::Digest testD; + testD.values[0] = 1; + testD.values[1] = 2; + testD.values[2] = 3; + testD.values[3] = 4; + + StringBuilder testDSb; + testDSb << testD; + SLANG_CHECK(testDSb.equals(String("01000000020000000300000004000000"))); + } +} diff --git a/tools/slang-unit-test/unit-test-md5.cpp b/tools/slang-unit-test/unit-test-md5.cpp index 0e7fef57c..41f3d6cf6 100644 --- a/tools/slang-unit-test/unit-test-md5.cpp +++ b/tools/slang-unit-test/unit-test-md5.cpp @@ -3,7 +3,7 @@ #include "../../source/core/slang-md5.h" #include "../../source/core/slang-string.h" -#include "../../source/slang/slang-hash-utils.h" +#include "../../source/core/slang-digest-util.h" using namespace Slang; @@ -23,7 +23,7 @@ SLANG_UNIT_TEST(md5hash) slang::Digest testA; testHashGen.finalize(&testCtx, &testA); - String testAString = hashToString(testA); + String testAString = DigestUtil::toString(testA); SLANG_CHECK(testAString.equals(String("5BA171E20898BDD205639013746F2679"))); } @@ -43,7 +43,7 @@ SLANG_UNIT_TEST(md5hash) slang::Digest testB; testHashGen.finalize(&testCtx, &testB); - String testBString = hashToString(testB); + String testBString = DigestUtil::toString(testB); SLANG_CHECK(testBString.equals(String("4352D88A78AA39750BF70CD6F27BCAA5"))); } @@ -59,7 +59,7 @@ SLANG_UNIT_TEST(md5hash) slang::Digest testC; testHashGen.finalize(&testCtx, &testC); - String testCString = hashToString(testC); + String testCString = DigestUtil::toString(testC); SLANG_CHECK(testCString.equals(String("5D6CC58E1824A4DFD0CF57395B603316"))); } @@ -75,7 +75,7 @@ SLANG_UNIT_TEST(md5hash) slang::Digest testD; testHashGen.finalize(&testCtx, &testD); - String testDString = hashToString(testD); + String testDString = DigestUtil::toString(testD); SLANG_CHECK(testDString.equals(String("DF5A79CC2170C7401CF0A506CEB0CE24"))); } @@ -91,7 +91,7 @@ SLANG_UNIT_TEST(md5hash) slang::Digest testE; testHashGen.finalize(&testCtx, &testE); - String testEString = hashToString(testE); + String testEString = DigestUtil::toString(testE); SLANG_CHECK(testEString.equals(String("4AE71336E44BF9BF79D2752E234818A5"))); } } -- cgit v1.2.3