From f0cd62b37c5dfbbdb3fb205f1be2b8beba0dfed4 Mon Sep 17 00:00:00 2001
From: lucy96chen <47800040+lucy96chen@users.noreply.github.com>
Date: Wed, 12 Oct 2022 09:55:09 -0700
Subject: Shader caching (#2432)
* Changed all getEntryPointCode calls to use RendererBase::getEntryPointCodeFromShaderCache
* Hashing hooked up, tests pass but need to add more to fully test functionality
* checkpoint
* Checkpoint: File system creation seems functional, saving is broken
* checkpoint: Fixed filename generation from MD5 hash, shader blob might be going missing ahead of pipeline state creation
* Fixed a lot of bugs related to hash code generation, shader cache is likely working but needs further testing
* Added workaround for module loading by re-creating the test device, shader cache test functional
* Vulkan shader caching bug fixed, checkpoint commit before more refinement
* pre-ToT merge checkpoint
* checkpoint commit, improving cache keys
* Significantly expanded items included in the dependency hash for Module; Added dependency hash functions to SpecializedComponentType and RenamedEntryPointComponentType
* Temporarily disable shader cache test
* Mid cleanup changes, solution successfully builds
* Added several helper update functions to slang-md5 to help simplify usage; Added a function under ISession to compute a hash for all linkage-related items; Function renames and cleaned up some comments
* Ran premake.bat; Renamed getASTBasedHashCode to computeASTBasedHash
* Added slang unit tests for Checksum and MD5; Extended gfx shader cache test to test with multiple shader files and one shader file with multiple entry points
* Solution builds and shader cache tests pass, but at least a couple other tests now failing
* ran premake.bat
* More cleanup changes
* Added shaderCachePath field to IDevice desc in gfx.slang, gfx-smoke.slang should be functional
* ran premake
* cleanup changes; Adding test printf to getEntryPointCodeFromShaderCache to see if output can be seen in CI
* Removed debugging printfs; Added handling for getEntryPointCode() failing
* Cleanup changes; Jonathan's fixes to SerialWriter to zero initialize otherwise uninitialized memory; Change to SwizzleExpr creation to zero initialize elementCount
* Changed enable_if_t to enable_if
* Fixed enable_if
* Added test for import vs include and changes to included and imported files; Fixed build errors in CUDA; Renamed shader cache statistics fields
* cleanup changes
* Readd removed file
* Restructured computeDependencyBasedHash calls, added computeDependencyBasedHashImpl to all classes dervied from ComponentType
* Applied same restructuring to the AST hash functions
* Cleanup changes; Moved HashBuilder out to slang-digest.h and added some helper functions to streamline the process of adding items to a hash
* Cleanup; Fixed incorrect expected results for shader import and include test
---
build/visual-studio/core/core.vcxproj | 3 +
build/visual-studio/core/core.vcxproj.filters | 9 +
.../gfx-unit-test-tool/gfx-unit-test-tool.vcxproj | 8 +
.../gfx-unit-test-tool.vcxproj.filters | 24 +
build/visual-studio/slang-rt/slang-rt.vcxproj | 3 +
.../slang-rt/slang-rt.vcxproj.filters | 9 +
.../slang-unit-test-tool.vcxproj | 2 +
.../slang-unit-test-tool.vcxproj.filters | 6 +
build/visual-studio/slang/slang.vcxproj | 1 +
build/visual-studio/slang/slang.vcxproj.filters | 3 +
slang-gfx.h | 24 +-
slang.h | 38 ++
source/core/slang-digest.h | 31 +
source/core/slang-md5.cpp | 312 +++++++++++
source/core/slang-md5.h | 82 +++
source/slang/slang-ast-builder.h | 15 +-
source/slang/slang-check-expr.cpp | 4 +
source/slang/slang-compiler.cpp | 24 +
source/slang/slang-compiler.h | 119 +++-
source/slang/slang-hash-utils.h | 55 ++
source/slang/slang-serialize-ast.cpp | 25 +
source/slang/slang-serialize-ast.h | 2 +
source/slang/slang-serialize-type-info.h | 12 +
source/slang/slang-serialize.cpp | 17 +-
source/slang/slang-serialize.h | 25 +-
source/slang/slang.cpp | 175 +++++-
tools/gfx-unit-test/gfx-test-util.cpp | 14 +-
tools/gfx-unit-test/gfx-test-util.h | 6 +-
tools/gfx-unit-test/imported.slang | 8 +
.../importing-shader-cache-shader.slang | 15 +
.../multiple-entry-point-shader-cache-shader.slang | 28 +
tools/gfx-unit-test/shader-cache-shader-A.slang | 10 +
tools/gfx-unit-test/shader-cache-shader-B.slang | 10 +
tools/gfx-unit-test/shader-cache-shader-C.slang | 10 +
tools/gfx-unit-test/shader-cache-shader.slang | 10 +
tools/gfx-unit-test/shader-cache-tests.cpp | 624 +++++++++++++++++++++
tools/gfx/cuda/cuda-device.cpp | 2 +-
tools/gfx/d3d11/d3d11-device.cpp | 2 +-
tools/gfx/d3d12/d3d12-pipeline-state.cpp | 4 +-
tools/gfx/gfx.slang | 2 +
tools/gfx/open-gl/render-gl.cpp | 2 +-
tools/gfx/renderer-shared.cpp | 176 +++++-
tools/gfx/renderer-shared.h | 31 +-
tools/gfx/vulkan/vk-pipeline-state.cpp | 6 +-
tools/slang-unit-test/unit-test-checksum.cpp | 32 ++
tools/slang-unit-test/unit-test-md5.cpp | 97 ++++
46 files changed, 2085 insertions(+), 32 deletions(-)
create mode 100644 source/core/slang-digest.h
create mode 100644 source/core/slang-md5.cpp
create mode 100644 source/core/slang-md5.h
create mode 100644 source/slang/slang-hash-utils.h
create mode 100644 tools/gfx-unit-test/imported.slang
create mode 100644 tools/gfx-unit-test/importing-shader-cache-shader.slang
create mode 100644 tools/gfx-unit-test/multiple-entry-point-shader-cache-shader.slang
create mode 100644 tools/gfx-unit-test/shader-cache-shader-A.slang
create mode 100644 tools/gfx-unit-test/shader-cache-shader-B.slang
create mode 100644 tools/gfx-unit-test/shader-cache-shader-C.slang
create mode 100644 tools/gfx-unit-test/shader-cache-shader.slang
create mode 100644 tools/gfx-unit-test/shader-cache-tests.cpp
create mode 100644 tools/slang-unit-test/unit-test-checksum.cpp
create mode 100644 tools/slang-unit-test/unit-test-md5.cpp
diff --git a/build/visual-studio/core/core.vcxproj b/build/visual-studio/core/core.vcxproj
index a49ccd040..f84695880 100644
--- a/build/visual-studio/core/core.vcxproj
+++ b/build/visual-studio/core/core.vcxproj
@@ -270,6 +270,7 @@
+
@@ -284,6 +285,7 @@
+
@@ -339,6 +341,7 @@
+
diff --git a/build/visual-studio/core/core.vcxproj.filters b/build/visual-studio/core/core.vcxproj.filters
index 9266db917..14d866c12 100644
--- a/build/visual-studio/core/core.vcxproj.filters
+++ b/build/visual-studio/core/core.vcxproj.filters
@@ -69,6 +69,9 @@
Header Files
+
+ Header Files
+
Header Files
@@ -111,6 +114,9 @@
Header Files
+
+ Header Files
+
Header Files
@@ -272,6 +278,9 @@
Source Files
+
+ Source Files
+
Source Files
diff --git a/build/visual-studio/gfx-unit-test-tool/gfx-unit-test-tool.vcxproj b/build/visual-studio/gfx-unit-test-tool/gfx-unit-test-tool.vcxproj
index a346900c6..7553119d3 100644
--- a/build/visual-studio/gfx-unit-test-tool/gfx-unit-test-tool.vcxproj
+++ b/build/visual-studio/gfx-unit-test-tool/gfx-unit-test-tool.vcxproj
@@ -295,6 +295,7 @@
+
@@ -307,12 +308,19 @@
+
+
+
+
+
+
+
diff --git a/build/visual-studio/gfx-unit-test-tool/gfx-unit-test-tool.vcxproj.filters b/build/visual-studio/gfx-unit-test-tool/gfx-unit-test-tool.vcxproj.filters
index 157e1e554..90dc02b69 100644
--- a/build/visual-studio/gfx-unit-test-tool/gfx-unit-test-tool.vcxproj.filters
+++ b/build/visual-studio/gfx-unit-test-tool/gfx-unit-test-tool.vcxproj.filters
@@ -86,6 +86,9 @@
Source Files
+
+ Source Files
+
Source Files
@@ -118,6 +121,15 @@
Source Files
+
+ Source Files
+
+
+ Source Files
+
+
+ Source Files
+
Source Files
@@ -136,6 +148,18 @@
Source Files
+
+ Source Files
+
+
+ Source Files
+
+
+ Source Files
+
+
+ Source Files
+
Source Files
diff --git a/build/visual-studio/slang-rt/slang-rt.vcxproj b/build/visual-studio/slang-rt/slang-rt.vcxproj
index 55c69a6c2..c30f5f077 100644
--- a/build/visual-studio/slang-rt/slang-rt.vcxproj
+++ b/build/visual-studio/slang-rt/slang-rt.vcxproj
@@ -282,6 +282,7 @@
+
@@ -296,6 +297,7 @@
+
@@ -352,6 +354,7 @@
+
diff --git a/build/visual-studio/slang-rt/slang-rt.vcxproj.filters b/build/visual-studio/slang-rt/slang-rt.vcxproj.filters
index d5726e5f9..65f4d5221 100644
--- a/build/visual-studio/slang-rt/slang-rt.vcxproj.filters
+++ b/build/visual-studio/slang-rt/slang-rt.vcxproj.filters
@@ -69,6 +69,9 @@
Header Files
+
+ Header Files
+
Header Files
@@ -111,6 +114,9 @@
Header Files
+
+ Header Files
+
Header Files
@@ -275,6 +281,9 @@
Source Files
+
+ Source Files
+
Source Files
diff --git a/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj b/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj
index a6a5dad7d..83681488f 100644
--- a/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj
+++ b/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj
@@ -272,6 +272,7 @@
+
@@ -282,6 +283,7 @@
+
diff --git a/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj.filters b/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj.filters
index c84d21b30..b83a18b76 100644
--- a/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj.filters
+++ b/build/visual-studio/slang-unit-test-tool/slang-unit-test-tool.vcxproj.filters
@@ -17,6 +17,9 @@
Source Files
+
+ Source Files
+
Source Files
@@ -47,6 +50,9 @@
Source Files
+
+ Source Files
+
Source Files
diff --git a/build/visual-studio/slang/slang.vcxproj b/build/visual-studio/slang/slang.vcxproj
index 7436f319e..4e8a9e70a 100644
--- a/build/visual-studio/slang/slang.vcxproj
+++ b/build/visual-studio/slang/slang.vcxproj
@@ -327,6 +327,7 @@ IF EXIST ..\..\..\external\slang-glslang\bin\windows-aarch64\release\slang-glsla
+
diff --git a/build/visual-studio/slang/slang.vcxproj.filters b/build/visual-studio/slang/slang.vcxproj.filters
index 02ef9de2e..8e6763e15 100644
--- a/build/visual-studio/slang/slang.vcxproj.filters
+++ b/build/visual-studio/slang/slang.vcxproj.filters
@@ -114,6 +114,9 @@
Header Files
+
+ Header Files
+
Header Files
diff --git a/slang-gfx.h b/slang-gfx.h
index 233d963e9..18395e3f2 100644
--- a/slang-gfx.h
+++ b/slang-gfx.h
@@ -2052,6 +2052,25 @@ public:
0xbe91ba6c, 0x784, 0x4308, { 0xa1, 0x0, 0x19, 0xc3, 0x66, 0x83, 0x44, 0xb2 } \
}
+// These are exclusively used to track hit/miss counts for shader cache entries. Entry hit and
+// miss counts specifically indicate if the file containing relevant shader code was found in
+// the cache, while the general hit and miss counts indicate whether the file was both found and
+// up-to-date.
+class IShaderCacheStatistics : public ISlangUnknown
+{
+public:
+ virtual SLANG_NO_THROW GfxCount SLANG_MCALL getCacheMissCount() = 0;
+ virtual SLANG_NO_THROW GfxCount SLANG_MCALL getCacheHitCount() = 0;
+ virtual SLANG_NO_THROW GfxCount SLANG_MCALL getCacheEntryDirtyCount() = 0;
+
+ virtual SLANG_NO_THROW Result SLANG_MCALL resetCacheStatistics() = 0;
+};
+
+#define SLANG_UUID_IShaderCacheStatistics \
+ { \
+ 0x8eccc8ec, 0x5c04, 0x4a51, { 0x99, 0x75, 0x13, 0xf8, 0xfe, 0xa1, 0x59, 0xf3 } \
+ }
+
struct DeviceInfo
{
DeviceType deviceType;
@@ -2089,7 +2108,7 @@ public:
handleMessage(DebugMessageType type, DebugMessageSource source, const char* message) = 0;
};
-class IDevice: public ISlangUnknown
+class IDevice : public ISlangUnknown
{
public:
struct SlangDesc
@@ -2134,6 +2153,8 @@ public:
ISlangUnknown* apiCommandDispatcher = nullptr;
// The slot (typically UAV) used to identify NVAPI intrinsics. If >=0 NVAPI is required.
GfxIndex nvapiExtnSlot = -1;
+ // The root directory for the shader cache.
+ const char* shaderCachePath = nullptr;
// The file system for loading cached shader kernels. The layer does not maintain a strong reference to the object,
// instead the user is responsible for holding the object alive during the lifetime of an `IDevice`.
ISlangFileSystem* shaderCacheFileSystem = nullptr;
@@ -2473,7 +2494,6 @@ public:
0x715bdf26, 0x5135, 0x11eb, { 0xAE, 0x93, 0x02, 0x42, 0xAC, 0x13, 0x00, 0x02 } \
}
-
class IPipelineCreationAPIDispatcher : public ISlangUnknown
{
public:
diff --git a/slang.h b/slang.h
index 42825f5c4..08279a1de 100644
--- a/slang.h
+++ b/slang.h
@@ -4134,6 +4134,13 @@ namespace slang
None, UnsizedArray, StructuredBuffer, ConstantBuffer, ParameterBlock
};
+ // A struct storing a single hash represented as a four element uint32_t array.
+ // This is intended to be used with the current MD5 hashing implementation.
+ struct Digest
+ {
+ uint32_t values[4] = { 0 };
+ };
+
/** A session provides a scope for code that is loaded.
A session can be used to load modules of Slang source code,
@@ -4421,6 +4428,37 @@ namespace slang
IBlob** outCode,
IBlob** outDiagnostics = nullptr) = 0;
+ /** Compute the hash code of all dependencies for this component type. This generally means file path
+ dependencies but can also include the component's name or sub-components. The dependency-based
+ hash effectively represents all the files that may be included/imported by a component type along with
+ any non-code-specific that helps define a component. This can be useful to simply check for a component
+ type without needing to inspect the code. For example, a shader cache might key its entries using the
+ dependency-based hash in order to determine at a glance if a particular shader is present, with no
+ regard for the shader's contents.
+
+ This function should only have a meaningful implementation in ComponentType. All other types derived from
+ ComponentType that also inherit from IComponentType should do nothing.
+ */
+ virtual SLANG_NO_THROW void SLANG_MCALL computeDependencyBasedHash(
+ SlangInt entryPointIndex,
+ SlangInt targetIndex,
+ Digest* outHash) = 0;
+
+ /** Compute the hash code of this component type's AST. This hash effectively represents
+ the contents of the code covered by this component type, making its use ideal when we need
+ to confirm whether shader code changes have occurred. For example, a shader cache needs to be
+ able to check when a cache entry contains out-of-date code, which can be easily detected by
+ comparing the AST-based hashes since any change to the shader's code will be reflected in the
+ AST, and subsequently, the AST-based hash.
+
+ Not all component types will store an AST, and consequently, not all component types will have a
+ meaningful implementation for this function.
+
+ This function should only have a meaningful implementation in ComponentType. All other types derived
+ from ComponentType that also inherit from IComponentType should do nothing.
+ */
+ virtual SLANG_NO_THROW void SLANG_MCALL computeASTBasedHash(Digest* outHash) = 0;
+
/** Specialize the component by binding its specialization parameters to concrete arguments.
The `specializationArgs` array must have `specializationArgCount` entries, and
diff --git a/source/core/slang-digest.h b/source/core/slang-digest.h
new file mode 100644
index 000000000..d1bc80b40
--- /dev/null
+++ b/source/core/slang-digest.h
@@ -0,0 +1,31 @@
+#pragma once
+#include "slang-md5.h"
+#include "../../slang.h"
+
+namespace Slang
+{
+ // Wrapper struct that holds objects necessary for hashing.
+ struct DigestBuilder
+ {
+ public:
+ DigestBuilder()
+ {
+ hashGen.init(&context);
+ }
+
+ template
+ void addToDigest(T item)
+ {
+ hashGen.update(&context, item);
+ }
+
+ void finalize(slang::Digest* outHash)
+ {
+ hashGen.finalize(&context, outHash);
+ }
+
+ private:
+ MD5HashGen hashGen;
+ MD5Context context;
+ };
+}
diff --git a/source/core/slang-md5.cpp b/source/core/slang-md5.cpp
new file mode 100644
index 000000000..95b4816bf
--- /dev/null
+++ b/source/core/slang-md5.cpp
@@ -0,0 +1,312 @@
+/*
+ * This is an OpenSSL-compatible implementation of the RSA Data Security, Inc.
+ * MD5 Message-Digest Algorithm (RFC 1321).
+ *
+ * Homepage:
+ * http://openwall.info/wiki/people/solar/software/public-domain-source-code/md5
+ *
+ * Author:
+ * Alexander Peslyak, better known as Solar Designer
+ *
+ * This software was written by Alexander Peslyak in 2001. No copyright is
+ * claimed, and the software is hereby placed in the public domain.
+ * In case this attempt to disclaim copyright and place the software in the
+ * public domain is deemed null and void, then the software is
+ * Copyright (c) 2001 Alexander Peslyak and it is hereby released to the
+ * general public under the following terms:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted.
+ *
+ * There's ABSOLUTELY NO WARRANTY, express or implied.
+ *
+ * (This is a heavily cut-down "BSD license".)
+ *
+ * This differs from Colin Plumb's older public domain implementation in that
+ * no exactly 32-bit integer data type is required (any 32-bit or wider
+ * unsigned integer data type will do), there's no compile-time endianness
+ * configuration, and the function prototypes match OpenSSL's. No code from
+ * Colin Plumb's implementation has been reused; this comment merely compares
+ * the properties of the two independent implementations.
+ *
+ * The primary goals of this implementation are portability and ease of use.
+ * It is meant to be fast, but not as fast as possible. Some known
+ * optimizations are not included to reduce source code size and avoid
+ * compile-time configuration.
+ */
+
+#ifndef HAVE_OPENSSL
+
+#include
+
+#include "slang-md5.h"
+
+namespace Slang
+{
+ /*
+ * The basic MD5 functions.
+ *
+ * F and G are optimized compared to their RFC 1321 definitions for
+ * architectures that lack an AND-NOT instruction, just like in Colin Plumb's
+ * implementation.
+ */
+ #define F(x, y, z) ((z) ^ ((x) & ((y) ^ (z))))
+ #define G(x, y, z) ((y) ^ ((z) & ((x) ^ (y))))
+ #define H(x, y, z) (((x) ^ (y)) ^ (z))
+ #define H2(x, y, z) ((x) ^ ((y) ^ (z)))
+ #define I(x, y, z) ((y) ^ ((x) | ~(z)))
+
+ /*
+ * The MD5 transformation for all four rounds.
+ */
+ #define STEP(f, a, b, c, d, x, t, s) \
+ (a) += f((b), (c), (d)) + (x) + (t); \
+ (a) = (((a) << (s)) | (((a) & 0xffffffff) >> (32 - (s)))); \
+ (a) += (b);
+
+ /*
+ * SET reads 4 input bytes in little-endian byte order and stores them in a
+ * properly aligned word in host byte order.
+ *
+ * The check for little-endian architectures that tolerate unaligned memory
+ * accesses is just an optimization. Nothing will break if it fails to detect
+ * a suitable architecture.
+ *
+ * Unfortunately, this optimization may be a C strict aliasing rules violation
+ * if the caller's data buffer has effective type that cannot be aliased by
+ * MD5_u32plus. In practice, this problem may occur if these MD5 routines are
+ * inlined into a calling function, or with future and dangerously advanced
+ * link-time optimizations. For the time being, keeping these MD5 routines in
+ * their own translation unit avoids the problem.
+ */
+ #if defined(__i386__) || defined(__x86_64__) || defined(__vax__)
+ #define SET(n) \
+ (*(MD5_u32plus *)&ptr[(n) * 4])
+ #define GET(n) \
+ SET(n)
+ #else
+ #define SET(n) \
+ (ctx->block[(n)] = \
+ (MD5_u32plus)ptr[(n) * 4] | \
+ ((MD5_u32plus)ptr[(n) * 4 + 1] << 8) | \
+ ((MD5_u32plus)ptr[(n) * 4 + 2] << 16) | \
+ ((MD5_u32plus)ptr[(n) * 4 + 3] << 24))
+ #define GET(n) \
+ (ctx->block[(n)])
+ #endif
+
+ /*
+ * This processes one or more 64-byte data blocks, but does NOT update the bit
+ * counters. There are no alignment requirements.
+ */
+ /*static*/const void* MD5HashGen::body(MD5Context* ctx, const void* data, SlangInt size)
+ {
+ const unsigned char* ptr;
+ MD5_u32plus a, b, c, d;
+ MD5_u32plus saved_a, saved_b, saved_c, saved_d;
+
+ ptr = (const unsigned char*)data;
+
+ a = ctx->a;
+ b = ctx->b;
+ c = ctx->c;
+ d = ctx->d;
+
+ do {
+ saved_a = a;
+ saved_b = b;
+ saved_c = c;
+ saved_d = d;
+
+ /* Round 1 */
+ STEP(F, a, b, c, d, SET(0), 0xd76aa478, 7)
+ STEP(F, d, a, b, c, SET(1), 0xe8c7b756, 12)
+ STEP(F, c, d, a, b, SET(2), 0x242070db, 17)
+ STEP(F, b, c, d, a, SET(3), 0xc1bdceee, 22)
+ STEP(F, a, b, c, d, SET(4), 0xf57c0faf, 7)
+ STEP(F, d, a, b, c, SET(5), 0x4787c62a, 12)
+ STEP(F, c, d, a, b, SET(6), 0xa8304613, 17)
+ STEP(F, b, c, d, a, SET(7), 0xfd469501, 22)
+ STEP(F, a, b, c, d, SET(8), 0x698098d8, 7)
+ STEP(F, d, a, b, c, SET(9), 0x8b44f7af, 12)
+ STEP(F, c, d, a, b, SET(10), 0xffff5bb1, 17)
+ STEP(F, b, c, d, a, SET(11), 0x895cd7be, 22)
+ STEP(F, a, b, c, d, SET(12), 0x6b901122, 7)
+ STEP(F, d, a, b, c, SET(13), 0xfd987193, 12)
+ STEP(F, c, d, a, b, SET(14), 0xa679438e, 17)
+ STEP(F, b, c, d, a, SET(15), 0x49b40821, 22)
+
+ /* Round 2 */
+ STEP(G, a, b, c, d, GET(1), 0xf61e2562, 5)
+ STEP(G, d, a, b, c, GET(6), 0xc040b340, 9)
+ STEP(G, c, d, a, b, GET(11), 0x265e5a51, 14)
+ STEP(G, b, c, d, a, GET(0), 0xe9b6c7aa, 20)
+ STEP(G, a, b, c, d, GET(5), 0xd62f105d, 5)
+ STEP(G, d, a, b, c, GET(10), 0x02441453, 9)
+ STEP(G, c, d, a, b, GET(15), 0xd8a1e681, 14)
+ STEP(G, b, c, d, a, GET(4), 0xe7d3fbc8, 20)
+ STEP(G, a, b, c, d, GET(9), 0x21e1cde6, 5)
+ STEP(G, d, a, b, c, GET(14), 0xc33707d6, 9)
+ STEP(G, c, d, a, b, GET(3), 0xf4d50d87, 14)
+ STEP(G, b, c, d, a, GET(8), 0x455a14ed, 20)
+ STEP(G, a, b, c, d, GET(13), 0xa9e3e905, 5)
+ STEP(G, d, a, b, c, GET(2), 0xfcefa3f8, 9)
+ STEP(G, c, d, a, b, GET(7), 0x676f02d9, 14)
+ STEP(G, b, c, d, a, GET(12), 0x8d2a4c8a, 20)
+
+ /* Round 3 */
+ STEP(H, a, b, c, d, GET(5), 0xfffa3942, 4)
+ STEP(H2, d, a, b, c, GET(8), 0x8771f681, 11)
+ STEP(H, c, d, a, b, GET(11), 0x6d9d6122, 16)
+ STEP(H2, b, c, d, a, GET(14), 0xfde5380c, 23)
+ STEP(H, a, b, c, d, GET(1), 0xa4beea44, 4)
+ STEP(H2, d, a, b, c, GET(4), 0x4bdecfa9, 11)
+ STEP(H, c, d, a, b, GET(7), 0xf6bb4b60, 16)
+ STEP(H2, b, c, d, a, GET(10), 0xbebfbc70, 23)
+ STEP(H, a, b, c, d, GET(13), 0x289b7ec6, 4)
+ STEP(H2, d, a, b, c, GET(0), 0xeaa127fa, 11)
+ STEP(H, c, d, a, b, GET(3), 0xd4ef3085, 16)
+ STEP(H2, b, c, d, a, GET(6), 0x04881d05, 23)
+ STEP(H, a, b, c, d, GET(9), 0xd9d4d039, 4)
+ STEP(H2, d, a, b, c, GET(12), 0xe6db99e5, 11)
+ STEP(H, c, d, a, b, GET(15), 0x1fa27cf8, 16)
+ STEP(H2, b, c, d, a, GET(2), 0xc4ac5665, 23)
+
+ /* Round 4 */
+ STEP(I, a, b, c, d, GET(0), 0xf4292244, 6)
+ STEP(I, d, a, b, c, GET(7), 0x432aff97, 10)
+ STEP(I, c, d, a, b, GET(14), 0xab9423a7, 15)
+ STEP(I, b, c, d, a, GET(5), 0xfc93a039, 21)
+ STEP(I, a, b, c, d, GET(12), 0x655b59c3, 6)
+ STEP(I, d, a, b, c, GET(3), 0x8f0ccc92, 10)
+ STEP(I, c, d, a, b, GET(10), 0xffeff47d, 15)
+ STEP(I, b, c, d, a, GET(1), 0x85845dd1, 21)
+ STEP(I, a, b, c, d, GET(8), 0x6fa87e4f, 6)
+ STEP(I, d, a, b, c, GET(15), 0xfe2ce6e0, 10)
+ STEP(I, c, d, a, b, GET(6), 0xa3014314, 15)
+ STEP(I, b, c, d, a, GET(13), 0x4e0811a1, 21)
+ STEP(I, a, b, c, d, GET(4), 0xf7537e82, 6)
+ STEP(I, d, a, b, c, GET(11), 0xbd3af235, 10)
+ STEP(I, c, d, a, b, GET(2), 0x2ad7d2bb, 15)
+ STEP(I, b, c, d, a, GET(9), 0xeb86d391, 21)
+
+ a += saved_a;
+ b += saved_b;
+ c += saved_c;
+ d += saved_d;
+
+ ptr += 64;
+ } while (size -= 64);
+
+ ctx->a = a;
+ ctx->b = b;
+ ctx->c = c;
+ ctx->d = d;
+
+ return ptr;
+ }
+
+ void MD5HashGen::init(MD5Context* ctx)
+ {
+ ctx->a = 0x67452301;
+ ctx->b = 0xefcdab89;
+ ctx->c = 0x98badcfe;
+ ctx->d = 0x10325476;
+
+ ctx->lo = 0;
+ ctx->hi = 0;
+ }
+
+ void MD5HashGen::update(MD5Context* ctx, UnownedStringSlice string)
+ {
+ update(ctx, string.begin(), string.getLength());
+ }
+
+ void MD5HashGen::update(MD5Context* ctx, String str)
+ {
+ update(ctx, str.getBuffer(), str.getLength());
+ }
+
+ void MD5HashGen::update(MD5Context* ctx, const slang::Digest& hash)
+ {
+ update(ctx, hash.values, sizeof(hash.values));
+ }
+
+ void MD5HashGen::update(MD5Context* ctx, const void* data, SlangInt size)
+ {
+ MD5_u32plus saved_lo;
+ SlangInt used, available;
+
+ saved_lo = ctx->lo;
+ if ((ctx->lo = (saved_lo + size) & 0x1fffffff) < saved_lo)
+ ctx->hi++;
+ ctx->hi += (MD5_u32plus)size >> 29;
+
+ used = saved_lo & 0x3f;
+
+ if (used) {
+ available = 64 - used;
+
+ if (size < available) {
+ memcpy(&ctx->buffer[used], data, size);
+ return;
+ }
+
+ memcpy(&ctx->buffer[used], data, available);
+ data = (const unsigned char*)data + available;
+ size -= available;
+ body(ctx, ctx->buffer, 64);
+ }
+
+ if (size >= 64) {
+ data = body(ctx, data, size & ~(SlangInt)0x3f);
+ size &= 0x3f;
+ }
+
+ memcpy(ctx->buffer, data, size);
+ }
+
+ #define OUTUINT(dst, src) \
+ (dst)[0] = (uint32_t)src; \
+
+ #define OUTCHAR(dst, src) \
+ (dst)[0] = (unsigned char)(src); \
+ (dst)[1] = (unsigned char)((src) >> 8); \
+ (dst)[2] = (unsigned char)((src) >> 16); \
+ (dst)[3] = (unsigned char)((src) >> 24);
+
+ void MD5HashGen::finalize(MD5Context* ctx, slang::Digest* result)
+ {
+ unsigned long used, available;
+
+ used = ctx->lo & 0x3f;
+
+ ctx->buffer[used++] = 0x80;
+
+ available = 64 - used;
+
+ if (available < 8) {
+ memset(&ctx->buffer[used], 0, available);
+ body(ctx, ctx->buffer, 64);
+ used = 0;
+ available = 64;
+ }
+
+ memset(&ctx->buffer[used], 0, available - 8);
+
+ ctx->lo <<= 3;
+ OUTCHAR(&ctx->buffer[56], ctx->lo)
+ OUTCHAR(&ctx->buffer[60], ctx->hi)
+
+ body(ctx, ctx->buffer, 64);
+
+ OUTUINT(&result->values[0], ctx->a)
+ OUTUINT(&result->values[1], ctx->b)
+ OUTUINT(&result->values[2], ctx->c)
+ OUTUINT(&result->values[3], ctx->d)
+
+ memset(ctx, 0, sizeof(*ctx));
+ }
+}
+
+#endif
diff --git a/source/core/slang-md5.h b/source/core/slang-md5.h
new file mode 100644
index 000000000..8c51a03ec
--- /dev/null
+++ b/source/core/slang-md5.h
@@ -0,0 +1,82 @@
+/*
+ * This is an OpenSSL-compatible implementation of the RSA Data Security, Inc.
+ * MD5 Message-Digest Algorithm (RFC 1321).
+ *
+ * Homepage:
+ * http://openwall.info/wiki/people/solar/software/public-domain-source-code/md5
+ *
+ * Author:
+ * Alexander Peslyak, better known as Solar Designer
+ *
+ * This software was written by Alexander Peslyak in 2001. No copyright is
+ * claimed, and the software is hereby placed in the public domain.
+ * In case this attempt to disclaim copyright and place the software in the
+ * public domain is deemed null and void, then the software is
+ * Copyright (c) 2001 Alexander Peslyak and it is hereby released to the
+ * general public under the following terms:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted.
+ *
+ * There's ABSOLUTELY NO WARRANTY, express or implied.
+ *
+ * See md5.c for more information.
+ */
+
+#ifdef HAVE_OPENSSL
+#include
+#elif !defined(_MD5_H)
+#define _MD5_H
+
+#pragma once
+#include "../../slang.h"
+#include "../core/slang-string.h"
+#include "../core/slang-list.h"
+
+namespace Slang
+{
+ /* Any 32-bit or wider unsigned integer data type will do */
+ typedef uint32_t MD5_u32plus;
+
+ struct MD5Context
+ {
+ MD5_u32plus lo, hi;
+ MD5_u32plus a, b, c, d;
+ unsigned char buffer[64];
+ MD5_u32plus block[16];
+ };
+
+ class MD5HashGen
+ {
+ public:
+ void init(MD5Context* ctx);
+
+ // Helper update function for raw values (e.g. ints, uints)
+ template::value || std::is_arithmetic::value>::type>
+ void update(MD5Context* ctx, const T& val)
+ {
+ update(ctx, &val, sizeof(T));
+ }
+ // Helper update function for Slang::List
+ template
+ void update(MD5Context* ctx, const List& list)
+ {
+ update(ctx, list.getBuffer(), list.getCount());
+ }
+ // Helper update function for UnownedStringSlice
+ void update(MD5Context* ctx, UnownedStringSlice string);
+ // Helper update function for Slang::String
+ void update(MD5Context* ctx, String str);
+ // Helper update function for Checksums
+ void update(MD5Context* ctx, const slang::Digest& checksum);
+
+ void finalize(MD5Context* ctx, slang::Digest* result);
+
+ private:
+ static const void* body(MD5Context* ctx, const void* data, SlangInt size);
+ void update(MD5Context* ctx, const void* data, SlangInt size);
+ };
+}
+
+#endif
diff --git a/source/slang/slang-ast-builder.h b/source/slang/slang-ast-builder.h
index 3023e274d..91fe63c88 100644
--- a/source/slang/slang-ast-builder.h
+++ b/source/slang/slang-ast-builder.h
@@ -155,9 +155,20 @@ public:
/// Create AST types
template
- T* create() { return _initAndAdd(new (m_arena.allocate(sizeof(T))) T); }
+ T* create()
+ {
+ auto alloced = m_arena.allocate(sizeof(T));
+ memset(alloced, 0, sizeof(T));
+ return _initAndAdd(new (alloced) T);
+ }
+
template
- T* create(TArgs... args) { return _initAndAdd(new (m_arena.allocate(sizeof(T))) T(args...)); }
+ T* create(TArgs... args)
+ {
+ auto alloced = m_arena.allocate(sizeof(T));
+ memset(alloced, 0, sizeof(T));
+ return _initAndAdd(new (alloced) T(args...));
+ }
template
SLANG_FORCE_INLINE T* getOrCreate(TArgs ... args)
diff --git a/source/slang/slang-check-expr.cpp b/source/slang/slang-check-expr.cpp
index 2987fecee..f1ccddf15 100644
--- a/source/slang/slang-check-expr.cpp
+++ b/source/slang/slang-check-expr.cpp
@@ -2195,6 +2195,10 @@ namespace Slang
SwizzleExpr* swizExpr = m_astBuilder->create();
swizExpr->loc = memberRefExpr->loc;
swizExpr->base = memberRefExpr->baseExpression;
+ swizExpr->elementIndices[0] = 0;
+ swizExpr->elementIndices[1] = 0;
+ swizExpr->elementIndices[2] = 0;
+ swizExpr->elementIndices[3] = 0;
swizExpr->memberOpLoc = memberRefExpr->memberOperatorLoc;
IntegerLiteralValue limitElement = baseElementCount;
diff --git a/source/slang/slang-compiler.cpp b/source/slang/slang-compiler.cpp
index 06cc9d759..889ecc58a 100644
--- a/source/slang/slang-compiler.cpp
+++ b/source/slang/slang-compiler.cpp
@@ -35,6 +35,7 @@
#include "slang-glsl-extension-tracker.h"
#include "slang-emit-cuda.h"
+#include "slang-serialize-ast.h"
#include "slang-serialize-container.h"
namespace Slang
@@ -222,6 +223,16 @@ namespace Slang
visitor->visitEntryPoint(this, as(specializationInfo));
}
+ void EntryPoint::updateDependencyBasedHash(
+ DigestBuilder& builder,
+ SlangInt entryPointIndex)
+ {
+ // CompositeComponentType will have already hashed the relevant entry point's name
+ // and file path dependencies, so we immediately return.
+ SLANG_UNUSED(builder);
+ SLANG_UNUSED(entryPointIndex);
+ }
+
List const& EntryPoint::getModuleDependencies()
{
if(auto module = getModule())
@@ -290,6 +301,19 @@ namespace Slang
return Super::getInterface(guid);
}
+ void TypeConformance::updateDependencyBasedHash(
+ DigestBuilder& builder,
+ SlangInt entryPointIndex)
+ {
+ SLANG_UNUSED(entryPointIndex);
+
+ //TODO: Implement some kind of hashInto for Val then replace this
+ auto subtypeWitness = m_subtypeWitness->toString();
+
+ builder.addToDigest(subtypeWitness);
+ builder.addToDigest(m_conformanceIdOverride);
+ }
+
List const& TypeConformance::getModuleDependencies()
{
return m_moduleDependency.getModuleList();
diff --git a/source/slang/slang-compiler.h b/source/slang/slang-compiler.h
index 46be07fa3..016c8fefa 100755
--- a/source/slang/slang-compiler.h
+++ b/source/slang/slang-compiler.h
@@ -3,6 +3,7 @@
#include "../core/slang-basic.h"
#include "../core/slang-shared-library.h"
+#include "../core/slang-digest.h"
#include "../compiler-core/slang-downstream-compiler.h"
#include "../compiler-core/slang-downstream-compiler-util.h"
@@ -282,14 +283,23 @@ namespace Slang
const char* newName,
slang::IComponentType** outEntryPoint) SLANG_OVERRIDE;
SLANG_NO_THROW SlangResult SLANG_MCALL link(
- slang::IComponentType** outLinkedComponentType,
- ISlangBlob** outDiagnostics) SLANG_OVERRIDE;
+ slang::IComponentType** outLinkedComponentType,
+ ISlangBlob** outDiagnostics) SLANG_OVERRIDE;
SLANG_NO_THROW SlangResult SLANG_MCALL getEntryPointHostCallable(
int entryPointIndex,
int targetIndex,
ISlangSharedLibrary** outSharedLibrary,
slang::IBlob** outDiagnostics) SLANG_OVERRIDE;
+ /// ComponentType is the only class inheriting from IComponentType that provides a
+ /// meaningful implementation for these two functions. All others should forward these
+ /// and implement updateDependencyBasedHash and updateASTBasedHash instead.
+ SLANG_NO_THROW void SLANG_MCALL computeDependencyBasedHash(
+ SlangInt entryPointIndex,
+ SlangInt targetIndex,
+ slang::Digest* outHash) SLANG_OVERRIDE;
+ SLANG_NO_THROW void SLANG_MCALL computeASTBasedHash(slang::Digest* outHash) SLANG_OVERRIDE;
+
/// Get the linkage (aka "session" in the public API) for this component type.
Linkage* getLinkage() { return m_linkage; }
@@ -298,6 +308,16 @@ namespace Slang
/// The `target` must be a target on the `Linkage` that was used to create this program.
TargetProgram* getTargetProgram(TargetRequest* target);
+ /// Update the hash builder with the dependencies for this component type.
+ virtual void updateDependencyBasedHash(
+ DigestBuilder& hashBuilder,
+ SlangInt entryPointIndex) = 0;
+
+ /// Update the hash builder with the AST contents for this component type.
+ /// The AST is associated with a Module component, so most derived ComponentType classes
+ /// will simply do nothing with this.
+ virtual void updateASTBasedHash(DigestBuilder& hashBuilder) = 0;
+
/// Get the number of entry points linked into this component type.
virtual Index getEntryPointCount() = 0;
@@ -495,6 +515,12 @@ namespace Slang
Linkage* linkage,
List> const& childComponents);
+ virtual void updateDependencyBasedHash(
+ DigestBuilder& hashBuilder,
+ SlangInt entryPointIndex) override;
+
+ virtual void updateASTBasedHash(DigestBuilder& hashBuilder) override;
+
List> const& getChildComponents() { return m_childComponents; };
Index getChildComponentCount() { return m_childComponents.getCount(); }
RefPtr getChildComponent(Index index) { return m_childComponents[index]; }
@@ -571,6 +597,15 @@ namespace Slang
List const& specializationArgs,
DiagnosticSink* sink);
+ virtual void updateDependencyBasedHash(
+ DigestBuilder& hashBuilder,
+ SlangInt entryPointIndex) override;
+
+ virtual void updateASTBasedHash(DigestBuilder& hashBuilder) override
+ {
+ SLANG_UNUSED(hashBuilder);
+ }
+
/// Get the base (unspecialized) component type that is being specialized.
RefPtr getBaseComponentType() { return m_base; }
@@ -754,6 +789,16 @@ namespace Slang
void acceptVisitor(ComponentTypeVisitor* visitor, SpecializationInfo* specializationInfo)
SLANG_OVERRIDE;
+
+ virtual void updateDependencyBasedHash(
+ DigestBuilder& hashBuilder,
+ SlangInt entryPointIndex) override;
+
+ virtual void updateASTBasedHash(DigestBuilder& hashBuilder) override
+ {
+ SLANG_UNUSED(hashBuilder);
+ }
+
private:
RefPtr m_base;
String m_entryPointNameOverride;
@@ -846,6 +891,28 @@ namespace Slang
return Super::getEntryPointHostCallable(entryPointIndex, targetIndex, outSharedLibrary, outDiagnostics);
}
+ SLANG_NO_THROW void SLANG_MCALL computeDependencyBasedHash(
+ SlangInt entryPointIndex,
+ SlangInt targetIndex,
+ slang::Digest* outHash) SLANG_OVERRIDE
+ {
+ return Super::computeDependencyBasedHash(entryPointIndex, targetIndex, outHash);
+ }
+
+ SLANG_NO_THROW void SLANG_MCALL computeASTBasedHash(slang::Digest* outHash)
+ {
+ return Super::computeASTBasedHash(outHash);
+ }
+
+ virtual void updateDependencyBasedHash(
+ DigestBuilder& hashBuilder,
+ SlangInt entryPointIndex) override;
+
+ virtual void updateASTBasedHash(DigestBuilder& hashBuilder) override
+ {
+ SLANG_UNUSED(hashBuilder);
+ }
+
/// Create an entry point that refers to the given function.
static RefPtr create(
Linkage* linkage,
@@ -1051,6 +1118,28 @@ namespace Slang
entryPointIndex, targetIndex, outSharedLibrary, outDiagnostics);
}
+ SLANG_NO_THROW void SLANG_MCALL computeDependencyBasedHash(
+ SlangInt entryPointIndex,
+ SlangInt targetIndex,
+ slang::Digest* outHash) SLANG_OVERRIDE
+ {
+ return Super::computeDependencyBasedHash(entryPointIndex, targetIndex, outHash);
+ }
+
+ SLANG_NO_THROW void SLANG_MCALL computeASTBasedHash(slang::Digest* outHash)
+ {
+ return Super::computeASTBasedHash(outHash);
+ }
+
+ virtual void updateDependencyBasedHash(
+ DigestBuilder& hashBuilder,
+ SlangInt entryPointIndex) override;
+
+ virtual void updateASTBasedHash(DigestBuilder& hashBuilder) override
+ {
+ SLANG_UNUSED(hashBuilder);
+ }
+
List const& getModuleDependencies() SLANG_OVERRIDE;
List const& getFilePathDependencies() SLANG_OVERRIDE;
@@ -1225,6 +1314,25 @@ namespace Slang
//
+ SLANG_NO_THROW void SLANG_MCALL computeDependencyBasedHash(
+ SlangInt entryPointIndex,
+ SlangInt targetIndex,
+ slang::Digest* outHash) SLANG_OVERRIDE
+ {
+ return Super::computeDependencyBasedHash(entryPointIndex, targetIndex, outHash);
+ }
+
+ SLANG_NO_THROW void SLANG_MCALL computeASTBasedHash(slang::Digest* outHash)
+ {
+ return Super::computeASTBasedHash(outHash);
+ }
+
+ virtual void updateDependencyBasedHash(
+ DigestBuilder& hashBuilder,
+ SlangInt entryPointIndex) override;
+
+ virtual void updateASTBasedHash(DigestBuilder& hashBuilder) override;
+
/// Create a module (initially empty).
Module(Linkage* linkage, ASTBuilder* astBuilder = nullptr);
@@ -1657,6 +1765,13 @@ namespace Slang
SLANG_NO_THROW SlangResult SLANG_MCALL createCompileRequest(
SlangCompileRequest** outCompileRequest) override;
+ // Updates the supplied has builder with linkage-related information, which includes preprocessor
+ // defines, the compiler version, and other compiler options. This is then merged with the hash
+ // produced for the program to produce a key that can be used with the shader cache.
+ void updateDependencyBasedHash(
+ DigestBuilder& builder,
+ SlangInt targetIndex);
+
void addTarget(
slang::TargetDesc const& desc);
SlangResult addSearchPath(
diff --git a/source/slang/slang-hash-utils.h b/source/slang/slang-hash-utils.h
new file mode 100644
index 000000000..d0d610cb8
--- /dev/null
+++ b/source/slang/slang-hash-utils.h
@@ -0,0 +1,55 @@
+// slang-hash-utils.h - Utility functions specifically designed to be used with slang::Digest
+#pragma once
+#include "../../slang.h"
+#include "../core/slang-basic.h"
+#include "../core/slang-md5.h"
+#include "../core/slang-digest.h"
+
+namespace Slang
+{
+ // Compute the hash for an UnownedStringSlice
+ inline slang::Digest computeHashForStringSlice(UnownedStringSlice text)
+ {
+ DigestBuilder builder;
+ builder.addToDigest(text);
+
+ slang::Digest textHash;
+ builder.finalize(&textHash);
+
+ return textHash;
+ }
+
+ // Combines the two provided hashes.
+ inline slang::Digest combineHashes(const slang::Digest& hashA, const slang::Digest& hashB)
+ {
+ DigestBuilder builder;
+ builder.addToDigest(hashA);
+ builder.addToDigest(hashB);
+
+ slang::Digest combined;
+ builder.finalize(&combined);
+ return combined;
+ }
+
+ // Returns the stored hash in checksum as a String.
+ inline StringBuilder hashToString(const slang::Digest& hash)
+ {
+ StringBuilder filename;
+
+ for (Index i = 0; i < 4; ++i)
+ {
+ auto hashSegmentString = String(hash.values[i], 16);
+
+ auto leadingZeroCount = 8 - hashSegmentString.getLength();
+ for (Index j = 0; j < leadingZeroCount; ++j)
+ {
+ filename.append("0");
+ }
+ filename.append(hashSegmentString.getBuffer());
+ }
+
+ return filename;
+ }
+
+ // TODO: fromString implementation?
+}
diff --git a/source/slang/slang-serialize-ast.cpp b/source/slang/slang-serialize-ast.cpp
index 31a3f6ac0..04b5f71b1 100644
--- a/source/slang/slang-serialize-ast.cpp
+++ b/source/slang/slang-serialize-ast.cpp
@@ -154,4 +154,29 @@ struct ASTFieldAccess
return SLANG_OK;
}
+/* static */List ASTSerialUtil::serializeAST(ModuleDecl* moduleDecl)
+{
+ //TODO: we should store `classes` in GlobalSession to avoid recomputing them every time.
+ RefPtr classes;
+ SerialClassesUtil::create(classes);
+
+ List contents;
+ OwnedMemoryStream stream(FileAccess::ReadWrite);
+
+ // Only serialize out things *in* this module
+ ModuleSerialFilter filterStorage(moduleDecl);
+
+ SerialFilter* filter = moduleDecl ? &filterStorage : nullptr;
+
+ SerialWriter writer(classes, filter);
+
+ // Lets serialize it all
+ writer.addPointer(moduleDecl);
+ // Let's stick it all in a stream
+ writer.write(&stream);
+
+ stream.swapContents(contents);
+ return contents;
+}
+
} // namespace Slang
diff --git a/source/slang/slang-serialize-ast.h b/source/slang/slang-serialize-ast.h
index 3aa790c86..1052c9a0a 100644
--- a/source/slang/slang-serialize-ast.h
+++ b/source/slang/slang-serialize-ast.h
@@ -48,6 +48,8 @@ struct ASTSerialUtil
/// Tries to serialize out, read back in and test the results are the same.
/// Will write dumped out node to files
static SlangResult testSerialize(NodeBase* node, RootNamePool* rootNamePool, SharedASTBuilder* sharedASTBuilder, SourceManager* sourceManager);
+
+ static List serializeAST(ModuleDecl* moduleDecl);
};
} // namespace Slang
diff --git a/source/slang/slang-serialize-type-info.h b/source/slang/slang-serialize-type-info.h
index c80eb8051..6258a129d 100644
--- a/source/slang/slang-serialize-type-info.h
+++ b/source/slang/slang-serialize-type-info.h
@@ -104,6 +104,12 @@ struct SerialTypeInfo
static void toSerial(SerialWriter* writer, const void* inNative, void* outSerial)
{
SerialElementType* serial = (SerialElementType*)outSerial;
+
+ if (writer->getFlags() & SerialWriter::Flag::ZeroInitialize)
+ {
+ ::memset(outSerial, 0, sizeof(SerialElementType) * N);
+ }
+
const T* native = (const T*)inNative;
for (Index i = 0; i < Index(N); ++i)
{
@@ -278,6 +284,12 @@ struct SerialTypeInfo>
keys.setCount(count);
values.setCount(count);
+ if (writer->getFlags() & SerialWriter::Flag::ZeroInitialize)
+ {
+ ::memset(keys.getBuffer(), 0, count * sizeof(KeySerialType));
+ ::memset(values.getBuffer(), 0, count * sizeof(ValueSerialType));
+ }
+
Index i = 0;
for (const auto& pair : src)
{
diff --git a/source/slang/slang-serialize.cpp b/source/slang/slang-serialize.cpp
index 71ef303b7..9b63d47c5 100644
--- a/source/slang/slang-serialize.cpp
+++ b/source/slang/slang-serialize.cpp
@@ -210,10 +210,11 @@ SerialClasses::SerialClasses():
// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! SerialWriter !!!!!!!!!!!!!!!!!!!!!!!!!!!!
-SerialWriter::SerialWriter(SerialClasses* classes, SerialFilter* filter) :
- m_arena(2048),
- m_classes(classes),
- m_filter(filter)
+SerialWriter::SerialWriter(SerialClasses* classes, SerialFilter* filter, Flags flags)
+ : m_arena(2048)
+ , m_classes(classes)
+ , m_filter(filter)
+ , m_flags(flags)
{
// 0 is always the null pointer
m_entries.add(nullptr);
@@ -236,6 +237,7 @@ SerialIndex SerialWriter::writeObject(const SerialClass* serialCls, const void*
nodeEntry->typeKind = serialCls->typeKind;
nodeEntry->subType = serialCls->subType;
+ nodeEntry->_pad0 = 0;
nodeEntry->info = SerialInfo::makeEntryInfo(serialCls->alignment);
@@ -244,6 +246,12 @@ SerialIndex SerialWriter::writeObject(const SerialClass* serialCls, const void*
// Point to start of payload
uint8_t* serialPayload = (uint8_t*)(nodeEntry + 1);
+
+ if (m_flags & Flag::ZeroInitialize)
+ {
+ ::memset(serialPayload, 0, serialCls->size);
+ }
+
while (serialCls)
{
for (Index i = 0; i < serialCls->fieldsCount; ++i)
@@ -483,6 +491,7 @@ SlangResult SerialWriter::write(Stream* stream)
for (Index i = 1; i < entriesCount; ++i)
{
SerialInfo::Entry* next = entries[i + 1];
+
// Before writing we need to store the next alignment
const size_t nextAlignment = SerialInfo::getAlignment(next->info);
diff --git a/source/slang/slang-serialize.h b/source/slang/slang-serialize.h
index fcd2daa1f..e08d26dd5 100644
--- a/source/slang/slang-serialize.h
+++ b/source/slang/slang-serialize.h
@@ -316,6 +316,18 @@ void SerialReader::getArray(SerialIndex index, List& out)
class SerialWriter : public RefObject
{
public:
+ typedef uint32_t Flags;
+ struct Flag
+ {
+ enum Enum : Flags
+ {
+ /// If set will zero initialize backing memory. This is slower but
+ /// is desirable to make two serializations of the same thing produce the
+ /// identical serialized result.
+ ZeroInitialize = 0x1
+ };
+ };
+
SerialIndex addPointer(const NodeBase* ptr);
SerialIndex addPointer(const RefObject* ptr);
@@ -367,8 +379,11 @@ public:
/// Used for attaching extra objects necessary for serializing
SerialExtraObjects& getExtraObjects() { return m_extraObjects; }
+ /// Get the flag
+ Flags getFlags() const { return m_flags; }
+
/// Ctor
- SerialWriter(SerialClasses* classes, SerialFilter* filter);
+ SerialWriter(SerialClasses* classes, SerialFilter* filter, Flags flags = Flag::ZeroInitialize);
protected:
@@ -388,7 +403,6 @@ protected:
Dictionary m_ptrMap; // Maps a pointer to an entry index
-
// NOTE! Assumes the content stays in scope!
SliceMap m_sliceMap;
SliceMap m_importSymbolMap;
@@ -399,6 +413,8 @@ protected:
MemoryArena m_arena; ///< Holds the payloads
SerialClasses* m_classes;
SerialFilter* m_filter; ///< Filter to control what is serialized
+
+ Flags m_flags; ///< Flags to control behavior
};
// ---------------------------------------------------------------------------
@@ -419,6 +435,11 @@ SerialIndex SerialWriter::addArray(const T* in, Index count)
List work;
work.setCount(count);
+ if (getFlags() & Flag::ZeroInitialize)
+ {
+ ::memset(work.getBuffer(), 0, sizeof(ElementSerialType) * count);
+ }
+
for (Index i = 0; i < count; ++i)
{
ElementTypeInfo::toSerial(this, &in[i], &work[i]);
diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp
index a3fd91ce0..a6dcf8ab2 100644
--- a/source/slang/slang.cpp
+++ b/source/slang/slang.cpp
@@ -44,6 +44,9 @@
#include "slang-check-impl.h"
+#include "../core/slang-md5.h"
+#include "slang-hash-utils.h"
+
#include "../../slang-tag-version.h"
// Used to print exception type names in internal-compiler-error messages
@@ -1318,6 +1321,54 @@ SLANG_NO_THROW SlangResult SLANG_MCALL Linkage::createCompileRequest(
return SLANG_OK;
}
+void Linkage::updateDependencyBasedHash(
+ DigestBuilder& builder,
+ SlangInt targetIndex)
+{
+ // Add the Slang compiler version to the hash
+ auto version = String(getBuildTagString());
+ builder.addToDigest(version);
+
+ // Add the search directory paths to the hash
+ auto searchDirectoryList = getSearchDirectories().searchDirectories;
+ for (auto& searchDir : searchDirectoryList)
+ {
+ auto searchPath = searchDir.path;
+ builder.addToDigest(searchPath);
+ }
+
+ // Add the preprocessor definitions to the hash
+ for (auto& key : preprocessorDefinitions)
+ {
+ builder.addToDigest(key.Key);
+ builder.addToDigest(key.Value);
+ }
+
+ // Add the target specified by targetIndex
+ auto targetReq = targets[targetIndex];
+ builder.addToDigest(targetReq->getTarget());
+ builder.addToDigest(targetReq->getTargetFlags());
+ builder.addToDigest(targetReq->getFloatingPointMode());
+ builder.addToDigest(targetReq->getLineDirectiveMode());
+ builder.addToDigest(targetReq->shouldDumpIntermediates());
+ builder.addToDigest(targetReq->getForceGLSLScalarBufferLayout());
+ builder.addToDigest(targetReq->shouldTrackLiveness());
+
+ auto targetProfile = targetReq->getTargetProfile();
+ builder.addToDigest(targetProfile.getStage());
+ builder.addToDigest(targetProfile.getVersion());
+ builder.addToDigest(targetProfile.getFamily());
+
+ auto targetProfileName = String(targetProfile.getName());
+ builder.addToDigest(targetProfileName);
+
+ auto cookedCapabilities = targetReq->getTargetCaps().getExpandedAtoms();
+ for (auto& capability : cookedCapabilities)
+ {
+ builder.addToDigest(capability);
+ }
+}
+
SlangResult Linkage::addSearchPath(
char const* path)
{
@@ -1618,10 +1669,20 @@ void TranslationUnitRequest::_addSourceFile(SourceFile* sourceFile)
// an associated path and/or wasn't from a file.
auto pathInfo = sourceFile->getPathInfo();
- if (pathInfo.hasFileFoundPath())
+ if (pathInfo.hasFoundPath())
{
getModule()->addFilePathDependency(pathInfo.foundPath);
}
+ else
+ {
+ // No path exists for this source, so we generate a new string to use as a
+ // fake path in the list of file path dependencies. This is needed to account
+ // for non-file-based dependencies later when shader files are being hashed for
+ // the shader cache.
+
+ slang::Digest sourceHash = computeHashForStringSlice(sourceFile->getContent());
+ getModule()->addFilePathDependency(hashToString(sourceHash));
+ }
}
List const& TranslationUnitRequest::getSourceFiles()
@@ -3149,8 +3210,6 @@ void FilePathDependencyList::addDependency(Module* module)
}
}
-
-
//
// Module
//
@@ -3178,6 +3237,22 @@ ISlangUnknown* Module::getInterface(const Guid& guid)
return Super::getInterface(guid);
}
+void Module::updateDependencyBasedHash(
+ DigestBuilder& builder,
+ SlangInt entryPointIndex)
+{
+ // CompositeComponentType will have already hashed this Module's file
+ // dependencies.
+ SLANG_UNUSED(builder);
+ SLANG_UNUSED(entryPointIndex);
+}
+
+void Module::updateASTBasedHash(DigestBuilder& builder)
+{
+ auto serializedAST = ASTSerialUtil::serializeAST(getModuleDecl());
+ builder.addToDigest(serializedAST);
+}
+
void Module::addModuleDependency(Module* module)
{
m_moduleDependencyList.addDependency(module);
@@ -3376,6 +3451,52 @@ SLANG_NO_THROW SlangResult SLANG_MCALL ComponentType::getEntryPointCode(
return artifact->loadBlob(ArtifactKeep::Yes, outCode);
}
+SLANG_NO_THROW void SLANG_MCALL ComponentType::computeDependencyBasedHash(
+ SlangInt entryPointIndex,
+ SlangInt targetIndex,
+ slang::Digest* outHash)
+{
+ DigestBuilder builder;
+
+ // A note on enums that may be hashed in as part of the following two function calls:
+ //
+ // While enums are not guaranteed to be encoded the same way across all versions of
+ // the compiler, part of hashing the linkage is hashing in the compiler version.
+ // Consequently, any encoding differences as a result of different compiler versions
+ // will already be reflected in the resulting hash.
+ getLinkage()->updateDependencyBasedHash(builder, targetIndex);
+ updateDependencyBasedHash(builder, entryPointIndex);
+
+ // Add file path dependencies to the hash - all child components
+ // will have file path dependencies that are a subset of this list.
+ auto fileDeps = getFilePathDependencies();
+ for (auto& file : fileDeps)
+ {
+ builder.addToDigest(file);
+ }
+
+ // Add the name and name override for the specified entry point
+ // to the hash.
+ auto entryPointName = getEntryPoint(entryPointIndex)->getName()->text;
+ builder.addToDigest(entryPointName);
+ auto entryPointNameOverride = getEntryPointNameOverride(entryPointIndex);
+ builder.addToDigest(entryPointNameOverride);
+
+ slang::Digest hash;
+ builder.finalize(&hash);
+ *outHash = hash;
+}
+
+SLANG_NO_THROW void SLANG_MCALL ComponentType::computeASTBasedHash(slang::Digest* outHash)
+{
+ DigestBuilder builder;
+ updateASTBasedHash(builder);
+
+ slang::Digest hash;
+ builder.finalize(&hash);
+ *outHash = hash;
+}
+
SLANG_NO_THROW SlangResult SLANG_MCALL ComponentType::getEntryPointHostCallable(
int entryPointIndex,
int targetIndex,
@@ -3708,6 +3829,28 @@ CompositeComponentType::CompositeComponentType(
}
}
+void CompositeComponentType::updateDependencyBasedHash(
+ DigestBuilder& builder,
+ SlangInt entryPointIndex)
+{
+ auto componentCount = getChildComponentCount();
+
+ for (Index i = 0; i < componentCount; ++i)
+ {
+ getChildComponent(i)->updateDependencyBasedHash(builder, entryPointIndex);
+ }
+}
+
+void CompositeComponentType::updateASTBasedHash(DigestBuilder& builder)
+{
+ auto componentCount = getChildComponentCount();
+
+ for (Index i = 0; i < componentCount; ++i)
+ {
+ getChildComponent(i)->updateASTBasedHash(builder);
+ }
+}
+
Index CompositeComponentType::getEntryPointCount()
{
return m_entryPoints.getCount();
@@ -4187,6 +4330,22 @@ SpecializedComponentType::SpecializedComponentType(
collector.visitSpecialized(this);
}
+void SpecializedComponentType::updateDependencyBasedHash(
+ DigestBuilder& builder,
+ SlangInt entryPointIndex)
+{
+ auto specializationArgCount = getSpecializationArgCount();
+ for (Index i = 0; i < specializationArgCount; ++i)
+ {
+ auto specializationArg = getSpecializationArg(i);
+ auto argString = specializationArg.val->toString();
+ builder.addToDigest(argString);
+ }
+
+ slang::Digest baseHash;
+ getBaseComponentType()->updateDependencyBasedHash(builder, entryPointIndex);
+}
+
void SpecializedComponentType::acceptVisitor(ComponentTypeVisitor* visitor, SpecializationInfo* specializationInfo)
{
SLANG_ASSERT(specializationInfo == nullptr);
@@ -4231,6 +4390,16 @@ void RenamedEntryPointComponentType::acceptVisitor(
this, as(specializationInfo));
}
+void RenamedEntryPointComponentType::updateDependencyBasedHash(
+ DigestBuilder& builder,
+ SlangInt entryPointIndex)
+{
+ // CompositeComponentType will have already hashed the name override and file
+ // dependencies for this entry point.
+ SLANG_UNUSED(entryPointIndex);
+ SLANG_UNUSED(builder);
+}
+
void ComponentTypeVisitor::visitChildren(CompositeComponentType* composite, CompositeComponentType::CompositeSpecializationInfo* specializationInfo)
{
auto childCount = composite->getChildComponentCount();
diff --git a/tools/gfx-unit-test/gfx-test-util.cpp b/tools/gfx-unit-test/gfx-test-util.cpp
index 9a6a7c4f6..90aec2647 100644
--- a/tools/gfx-unit-test/gfx-test-util.cpp
+++ b/tools/gfx-unit-test/gfx-test-util.cpp
@@ -54,6 +54,13 @@ namespace gfx_test
diagnosticsBlob.writeRef());
diagnoseIfNeeded(diagnosticsBlob);
SLANG_RETURN_ON_FAIL(result);
+
+ ComPtr linkedProgram;
+ result = composedProgram->link(linkedProgram.writeRef(), diagnosticsBlob.writeRef());
+ diagnoseIfNeeded(diagnosticsBlob);
+ SLANG_RETURN_ON_FAIL(result);
+
+ composedProgram = linkedProgram;
slangReflection = composedProgram->getLayout();
gfx::IShaderProgram::Desc programDesc = {};
@@ -169,7 +176,7 @@ namespace gfx_test
compareComputeResultFuzzy(result, expectedResult, expectedBufferSize);
}
- Slang::ComPtr createTestingDevice(UnitTestContext* context, Slang::RenderApiFlag::Enum api)
+ Slang::ComPtr createTestingDevice(UnitTestContext* context, Slang::RenderApiFlag::Enum api, ISlangMutableFileSystem* fileSystem)
{
Slang::ComPtr device;
gfx::IDevice::Desc deviceDesc = {};
@@ -208,6 +215,11 @@ namespace gfx_test
void* extDescPtr = &extDesc;
deviceDesc.extendedDescs = &extDescPtr;
+ if (fileSystem)
+ {
+ deviceDesc.shaderCacheFileSystem = fileSystem;
+ }
+
auto createDeviceResult = gfxCreateDevice(&deviceDesc, device.writeRef());
if (SLANG_FAILED(createDeviceResult))
{
diff --git a/tools/gfx-unit-test/gfx-test-util.h b/tools/gfx-unit-test/gfx-test-util.h
index 2977ac599..e3b8f493b 100644
--- a/tools/gfx-unit-test/gfx-test-util.h
+++ b/tools/gfx-unit-test/gfx-test-util.h
@@ -69,20 +69,20 @@ namespace gfx_test
return compareComputeResult(device, buffer, 0, expectedBuffer.getBuffer(), bufferSize);
}
- Slang::ComPtr createTestingDevice(UnitTestContext* context, Slang::RenderApiFlag::Enum api);
+ Slang::ComPtr createTestingDevice(UnitTestContext* context, Slang::RenderApiFlag::Enum api, ISlangMutableFileSystem* fileSystem = nullptr);
void initializeRenderDoc();
void renderDocBeginFrame();
void renderDocEndFrame();
template
- void runTestImpl(const ImplFunc& f, UnitTestContext* context, Slang::RenderApiFlag::Enum api)
+ void runTestImpl(const ImplFunc& f, UnitTestContext* context, Slang::RenderApiFlag::Enum api, ISlangMutableFileSystem* fileSystem = nullptr)
{
if ((api & context->enabledApis) == 0)
{
SLANG_IGNORE_TEST
}
- auto device = createTestingDevice(context, api);
+ auto device = createTestingDevice(context, api, fileSystem);
if (!device)
{
SLANG_IGNORE_TEST
diff --git a/tools/gfx-unit-test/imported.slang b/tools/gfx-unit-test/imported.slang
new file mode 100644
index 000000000..a44782390
--- /dev/null
+++ b/tools/gfx-unit-test/imported.slang
@@ -0,0 +1,8 @@
+struct TestFunction
+ {
+ void simpleElementAdd(RWStructuredBuffer buffer, uint index)
+ {
+ var input = buffer[index];
+ buffer[index] = input + 1.0f;
+ }
+ };
\ No newline at end of file
diff --git a/tools/gfx-unit-test/importing-shader-cache-shader.slang b/tools/gfx-unit-test/importing-shader-cache-shader.slang
new file mode 100644
index 000000000..bede42492
--- /dev/null
+++ b/tools/gfx-unit-test/importing-shader-cache-shader.slang
@@ -0,0 +1,15 @@
+#include "imported.slang"
+
+ uniform RWStructuredBuffer buffer;
+
+ [shader("compute")]
+ [numthreads(4, 1, 1)]
+ void computeMain(
+ uint3 sv_dispatchThreadID : SV_DispatchThreadID)
+ {
+ TestFunction test;
+ for (uint i = 0; i < 4; ++i)
+ {
+ test.simpleElementAdd(buffer, i);
+ }
+ }
\ No newline at end of file
diff --git a/tools/gfx-unit-test/multiple-entry-point-shader-cache-shader.slang b/tools/gfx-unit-test/multiple-entry-point-shader-cache-shader.slang
new file mode 100644
index 000000000..9287b62ea
--- /dev/null
+++ b/tools/gfx-unit-test/multiple-entry-point-shader-cache-shader.slang
@@ -0,0 +1,28 @@
+uniform RWStructuredBuffer buffer;
+
+[shader("compute")]
+[numthreads(4, 1, 1)]
+void computeA(
+uint3 sv_dispatchThreadID : SV_DispatchThreadID)
+{
+ var input = buffer[sv_dispatchThreadID.x];
+ buffer[sv_dispatchThreadID.x] = input + 1.0f;
+}
+
+[shader("compute")]
+[numthreads(4, 1, 1)]
+void computeB(
+uint3 sv_dispatchThreadID : SV_DispatchThreadID)
+{
+ var input = buffer[sv_dispatchThreadID.x];
+ buffer[sv_dispatchThreadID.x] = input + 2.0f;
+}
+
+[shader("compute")]
+[numthreads(4, 1, 1)]
+void computeC(
+uint3 sv_dispatchThreadID : SV_DispatchThreadID)
+{
+ var input = buffer[sv_dispatchThreadID.x];
+ buffer[sv_dispatchThreadID.x] = input + 3.0f;
+}
diff --git a/tools/gfx-unit-test/shader-cache-shader-A.slang b/tools/gfx-unit-test/shader-cache-shader-A.slang
new file mode 100644
index 000000000..c2dbe56f0
--- /dev/null
+++ b/tools/gfx-unit-test/shader-cache-shader-A.slang
@@ -0,0 +1,10 @@
+uniform RWStructuredBuffer buffer;
+
+ [shader("compute")]
+ [numthreads(4, 1, 1)]
+ void computeMain(
+ uint3 sv_dispatchThreadID : SV_DispatchThreadID)
+ {
+ var input = buffer[sv_dispatchThreadID.x];
+ buffer[sv_dispatchThreadID.x] = input + 3.0f;
+ }
\ No newline at end of file
diff --git a/tools/gfx-unit-test/shader-cache-shader-B.slang b/tools/gfx-unit-test/shader-cache-shader-B.slang
new file mode 100644
index 000000000..341e32893
--- /dev/null
+++ b/tools/gfx-unit-test/shader-cache-shader-B.slang
@@ -0,0 +1,10 @@
+uniform RWStructuredBuffer buffer;
+
+ [shader("compute")]
+ [numthreads(4, 1, 1)]
+ void computeMain(
+ uint3 sv_dispatchThreadID : SV_DispatchThreadID)
+ {
+ var input = buffer[sv_dispatchThreadID.x];
+ buffer[sv_dispatchThreadID.x] = input + 1.0f;
+ }
\ No newline at end of file
diff --git a/tools/gfx-unit-test/shader-cache-shader-C.slang b/tools/gfx-unit-test/shader-cache-shader-C.slang
new file mode 100644
index 000000000..826475761
--- /dev/null
+++ b/tools/gfx-unit-test/shader-cache-shader-C.slang
@@ -0,0 +1,10 @@
+uniform RWStructuredBuffer buffer;
+
+ [shader("compute")]
+ [numthreads(4, 1, 1)]
+ void computeMain(
+ uint3 sv_dispatchThreadID : SV_DispatchThreadID)
+ {
+ var input = buffer[sv_dispatchThreadID.x];
+ buffer[sv_dispatchThreadID.x] = input + 2.0f;
+ }
\ No newline at end of file
diff --git a/tools/gfx-unit-test/shader-cache-shader.slang b/tools/gfx-unit-test/shader-cache-shader.slang
new file mode 100644
index 000000000..c2dbe56f0
--- /dev/null
+++ b/tools/gfx-unit-test/shader-cache-shader.slang
@@ -0,0 +1,10 @@
+uniform RWStructuredBuffer buffer;
+
+ [shader("compute")]
+ [numthreads(4, 1, 1)]
+ void computeMain(
+ uint3 sv_dispatchThreadID : SV_DispatchThreadID)
+ {
+ var input = buffer[sv_dispatchThreadID.x];
+ buffer[sv_dispatchThreadID.x] = input + 3.0f;
+ }
\ No newline at end of file
diff --git a/tools/gfx-unit-test/shader-cache-tests.cpp b/tools/gfx-unit-test/shader-cache-tests.cpp
new file mode 100644
index 000000000..184b71c06
--- /dev/null
+++ b/tools/gfx-unit-test/shader-cache-tests.cpp
@@ -0,0 +1,624 @@
+#include "tools/unit-test/slang-unit-test.h"
+
+#include "slang-gfx.h"
+#include "gfx-test-util.h"
+#include "tools/gfx-util/shader-cursor.h"
+#include "source/core/slang-basic.h"
+
+#include "source/core/slang-memory-file-system.h"
+#include "source/core/slang-file-system.h"
+
+using namespace gfx;
+
+namespace gfx_test
+{
+
+ struct BaseShaderCacheTest
+ {
+ UnitTestContext* context;
+ Slang::RenderApiFlag::Enum api;
+
+ ComPtr device;
+ ComPtr pipelineState;
+ ComPtr bufferView;
+
+ // diskFileSystem is used to save the test shaders to disk as necessary as
+ // loadComputeProgram() always loads from the OS file system. cacheFileSystem
+ // is used to hold the shader cache in memory to avoid needing to manually erase all the
+ // shader cache entry files between consecutive runs of the test.
+ ComPtr diskFileSystem;
+ ComPtr cacheFileSystem;
+
+ // Simple compute shaders we can pipe to our individual shader files for cache testing
+ Slang::String contentsA = Slang::String(
+ R"(uniform RWStructuredBuffer buffer;
+
+ [shader("compute")]
+ [numthreads(4, 1, 1)]
+ void computeMain(
+ uint3 sv_dispatchThreadID : SV_DispatchThreadID)
+ {
+ var input = buffer[sv_dispatchThreadID.x];
+ buffer[sv_dispatchThreadID.x] = input + 1.0f;
+ })");
+
+ Slang::String contentsB = Slang::String(
+ R"(uniform RWStructuredBuffer buffer;
+
+ [shader("compute")]
+ [numthreads(4, 1, 1)]
+ void computeMain(
+ uint3 sv_dispatchThreadID : SV_DispatchThreadID)
+ {
+ var input = buffer[sv_dispatchThreadID.x];
+ buffer[sv_dispatchThreadID.x] = input + 2.0f;
+ })");
+
+ Slang::String contentsC = Slang::String(
+ R"(uniform RWStructuredBuffer buffer;
+
+ [shader("compute")]
+ [numthreads(4, 1, 1)]
+ void computeMain(
+ uint3 sv_dispatchThreadID : SV_DispatchThreadID)
+ {
+ var input = buffer[sv_dispatchThreadID.x];
+ buffer[sv_dispatchThreadID.x] = input + 3.0f;
+ })");
+
+ void createRequiredResources()
+ {
+ const int numberCount = 4;
+ float initialData[] = { 0.0f, 1.0f, 2.0f, 3.0f };
+ IBufferResource::Desc bufferDesc = {};
+ bufferDesc.sizeInBytes = numberCount * sizeof(float);
+ bufferDesc.format = gfx::Format::Unknown;
+ bufferDesc.elementSize = sizeof(float);
+ bufferDesc.allowedStates = ResourceStateSet(
+ ResourceState::ShaderResource,
+ ResourceState::UnorderedAccess,
+ ResourceState::CopyDestination,
+ ResourceState::CopySource);
+ bufferDesc.defaultState = ResourceState::UnorderedAccess;
+ bufferDesc.memoryType = MemoryType::DeviceLocal;
+
+ ComPtr numbersBuffer;
+ GFX_CHECK_CALL_ABORT(device->createBufferResource(
+ bufferDesc,
+ (void*)initialData,
+ numbersBuffer.writeRef()));
+
+ IResourceView::Desc viewDesc = {};
+ viewDesc.type = IResourceView::Type::UnorderedAccess;
+ viewDesc.format = Format::Unknown;
+ GFX_CHECK_CALL_ABORT(
+ device->createBufferView(numbersBuffer, nullptr, viewDesc, bufferView.writeRef()));
+ }
+
+ void freeOldResources()
+ {
+ bufferView = nullptr;
+ pipelineState = nullptr;
+ device = nullptr;
+ }
+
+ // TODO: This should be removed at some point. Currently exists as a workaround for module loading
+ // seemingly not accounting for updated shader code under the same module name with the same entry point.
+ void generateNewDevice()
+ {
+ freeOldResources();
+ device = createTestingDevice(context, api, cacheFileSystem);
+ }
+
+ void init(ComPtr device, UnitTestContext* context)
+ {
+ this->device = device;
+ this->context = context;
+ switch (device->getDeviceInfo().deviceType)
+ {
+ case DeviceType::DirectX11:
+ api = Slang::RenderApiFlag::D3D11;
+ break;
+ case DeviceType::DirectX12:
+ api = Slang::RenderApiFlag::D3D12;
+ break;
+ case DeviceType::Vulkan:
+ api = Slang::RenderApiFlag::Vulkan;
+ break;
+ case DeviceType::CPU:
+ api = Slang::RenderApiFlag::CPU;
+ break;
+ case DeviceType::CUDA:
+ api = Slang::RenderApiFlag::CUDA;
+ break;
+ case DeviceType::OpenGl:
+ api = Slang::RenderApiFlag::OpenGl;
+ break;
+ default:
+ SLANG_IGNORE_TEST
+ }
+
+ cacheFileSystem = new Slang::MemoryFileSystem();
+ diskFileSystem = Slang::OSFileSystem::getMutableSingleton();
+ diskFileSystem = new Slang::RelativeFileSystem(diskFileSystem, "tools/gfx-unit-test");
+ }
+
+ void submitGPUWork()
+ {
+ Slang::ComPtr transientHeap;
+ ITransientResourceHeap::Desc transientHeapDesc = {};
+ transientHeapDesc.constantBufferSize = 4096;
+ GFX_CHECK_CALL_ABORT(
+ device->createTransientResourceHeap(transientHeapDesc, transientHeap.writeRef()));
+
+ ICommandQueue::Desc queueDesc = { ICommandQueue::QueueType::Graphics };
+ auto queue = device->createCommandQueue(queueDesc);
+
+ auto commandBuffer = transientHeap->createCommandBuffer();
+ auto encoder = commandBuffer->encodeComputeCommands();
+
+ auto rootObject = encoder->bindPipeline(pipelineState);
+
+ ShaderCursor rootCursor(rootObject);
+ // Bind buffer view to the entry point.
+ rootCursor.getPath("buffer").setResource(bufferView);
+
+ encoder->dispatchCompute(1, 1, 1);
+ encoder->endEncoding();
+ commandBuffer->close();
+ queue->executeCommandBuffer(commandBuffer);
+ queue->waitOnHost();
+ }
+ };
+
+ // One shader file on disk, all modifications are done to the same file
+ struct SingleEntryShaderCache : BaseShaderCacheTest
+ {
+ void generateNewPipelineState(Slang::String shaderContents)
+ {
+ diskFileSystem->saveFile("shader-cache-shader.slang", shaderContents.getBuffer(), shaderContents.getLength());
+
+ ComPtr shaderProgram;
+ slang::ProgramLayout* slangReflection;
+ GFX_CHECK_CALL_ABORT(loadComputeProgram(device, shaderProgram, "shader-cache-shader", "computeMain", slangReflection));
+
+ ComputePipelineStateDesc pipelineDesc = {};
+ pipelineDesc.program = shaderProgram.get();
+ GFX_CHECK_CALL_ABORT(
+ device->createComputePipelineState(pipelineDesc, pipelineState.writeRef()));
+ }
+
+ void run()
+ {
+ ComPtr shaderCacheStats;
+
+ // Due to needing a workaround to prevent loading old, outdated modules, we need to
+ // recreate the device between each segment of the test. However, we need to maintain the
+ // same cache filesystem for the duration of the test, so the device is immediately recreated
+ // to ensure we can pass the filesystem all the way through.
+ //
+ // TODO: Remove the repeated generateNewDevice() and createRequiredResources() calls once
+ // a solution exists that allows source code changes under the same module name to be picked
+ // up on load.
+ generateNewDevice();
+ createRequiredResources();
+ generateNewPipelineState(contentsA);
+ submitGPUWork();
+
+ device->queryInterface(SLANG_UUID_IShaderCacheStatistics, (void**)shaderCacheStats.writeRef());
+ SLANG_CHECK(shaderCacheStats->getCacheMissCount() == 1);
+ SLANG_CHECK(shaderCacheStats->getCacheHitCount() == 0);
+ SLANG_CHECK(shaderCacheStats->getCacheEntryDirtyCount() == 0);
+
+ generateNewDevice();
+ createRequiredResources();
+ generateNewPipelineState(contentsA);
+ submitGPUWork();
+
+ device->queryInterface(SLANG_UUID_IShaderCacheStatistics, (void**)shaderCacheStats.writeRef());
+ SLANG_CHECK(shaderCacheStats->getCacheMissCount() == 0);
+ SLANG_CHECK(shaderCacheStats->getCacheHitCount() == 1);
+ SLANG_CHECK(shaderCacheStats->getCacheEntryDirtyCount() == 0);
+
+ generateNewDevice();
+ createRequiredResources();
+ generateNewPipelineState(contentsC);
+ submitGPUWork();
+
+ device->queryInterface(SLANG_UUID_IShaderCacheStatistics, (void**)shaderCacheStats.writeRef());
+ SLANG_CHECK(shaderCacheStats->getCacheMissCount() == 0);
+ SLANG_CHECK(shaderCacheStats->getCacheHitCount() == 0);
+ SLANG_CHECK(shaderCacheStats->getCacheEntryDirtyCount() == 1);
+ }
+ };
+
+ // Several shader files on disk, modifications may be done to any file
+ struct MultipleEntryShaderCache : BaseShaderCacheTest
+ {
+ void modifyShaderA(Slang::String shaderContents)
+ {
+ diskFileSystem->saveFile("shader-cache-shader-A.slang", shaderContents.getBuffer(), shaderContents.getLength());
+ }
+
+ void modifyShaderB(Slang::String shaderContents)
+ {
+ diskFileSystem->saveFile("shader-cache-shader-B.slang", shaderContents.getBuffer(), shaderContents.getLength());
+ }
+
+ void modifyShaderC(Slang::String shaderContents)
+ {
+ diskFileSystem->saveFile("shader-cache-shader-C.slang", shaderContents.getBuffer(), shaderContents.getLength());
+ }
+
+ void generateNewPipelineState(GfxIndex shaderIndex)
+ {
+ ComPtr shaderProgram;
+ slang::ProgramLayout* slangReflection;
+ char* shaderFilename;
+ switch (shaderIndex)
+ {
+ case 0:
+ shaderFilename = "shader-cache-shader-A";
+ break;
+ case 1:
+ shaderFilename = "shader-cache-shader-B";
+ break;
+ case 2:
+ shaderFilename = "shader-cache-shader-C";
+ break;
+ default:
+ // Should never reach this point since we wrote the test
+ SLANG_IGNORE_TEST;
+ }
+ GFX_CHECK_CALL_ABORT(loadComputeProgram(device, shaderProgram, shaderFilename, "computeMain", slangReflection));
+
+ ComputePipelineStateDesc pipelineDesc = {};
+ pipelineDesc.program = shaderProgram.get();
+ GFX_CHECK_CALL_ABORT(
+ device->createComputePipelineState(pipelineDesc, pipelineState.writeRef()));
+ }
+
+ void checkAllCacheEntries()
+ {
+ generateNewPipelineState(0);
+ submitGPUWork();
+ generateNewPipelineState(1);
+ submitGPUWork();
+ generateNewPipelineState(2);
+ submitGPUWork();
+ }
+
+ void run()
+ {
+ ComPtr shaderCacheStats;
+
+ // Due to needing a workaround to prevent loading old, outdated modules, we need to
+ // recreate the device between each segment of the test. However, we need to maintain the
+ // same cache filesystem for the duration of the test, so the device is immediately recreated
+ // to ensure we can pass the filesystem all the way through.
+ //
+ // TODO: Remove the repeated generateNewDevice() and createRequiredResources() calls once
+ // a solution exists that allows source code changes under the same module name to be picked
+ // up on load.
+ generateNewDevice();
+ createRequiredResources();
+ modifyShaderA(contentsA);
+ modifyShaderB(contentsB);
+ modifyShaderC(contentsC);
+ checkAllCacheEntries();
+
+ device->queryInterface(SLANG_UUID_IShaderCacheStatistics, (void**)shaderCacheStats.writeRef());
+ SLANG_CHECK(shaderCacheStats->getCacheMissCount() == 3);
+ SLANG_CHECK(shaderCacheStats->getCacheHitCount() == 0);
+ SLANG_CHECK(shaderCacheStats->getCacheEntryDirtyCount() == 0);
+
+ generateNewDevice();
+ createRequiredResources();
+ checkAllCacheEntries();
+
+ device->queryInterface(SLANG_UUID_IShaderCacheStatistics, (void**)shaderCacheStats.writeRef());
+ SLANG_CHECK(shaderCacheStats->getCacheMissCount() == 0);
+ SLANG_CHECK(shaderCacheStats->getCacheHitCount() == 3);
+ SLANG_CHECK(shaderCacheStats->getCacheEntryDirtyCount() == 0);
+
+ generateNewDevice();
+ createRequiredResources();
+ modifyShaderA(contentsB);
+ checkAllCacheEntries();
+
+ device->queryInterface(SLANG_UUID_IShaderCacheStatistics, (void**)shaderCacheStats.writeRef());
+ SLANG_CHECK(shaderCacheStats->getCacheMissCount() == 0);
+ SLANG_CHECK(shaderCacheStats->getCacheHitCount() == 2);
+ SLANG_CHECK(shaderCacheStats->getCacheEntryDirtyCount() == 1);
+
+ generateNewDevice();
+ createRequiredResources();
+ modifyShaderA(contentsC);
+ modifyShaderB(contentsA);
+ modifyShaderC(contentsB);
+ checkAllCacheEntries();
+
+ device->queryInterface(SLANG_UUID_IShaderCacheStatistics, (void**)shaderCacheStats.writeRef());
+ SLANG_CHECK(shaderCacheStats->getCacheMissCount() == 0);
+ SLANG_CHECK(shaderCacheStats->getCacheHitCount() == 0);
+ SLANG_CHECK(shaderCacheStats->getCacheEntryDirtyCount() == 3);
+ }
+ };
+
+ // One shader file on disk containing several entry points, no modifications are made to the file
+ struct MultipleEntryPointShader : BaseShaderCacheTest
+ {
+ void generateNewPipelineState(GfxIndex shaderIndex)
+ {
+ ComPtr shaderProgram;
+ slang::ProgramLayout* slangReflection;
+ char* entryPointName;
+ switch (shaderIndex)
+ {
+ case 0:
+ entryPointName = "computeA";
+ break;
+ case 1:
+ entryPointName = "computeB";
+ break;
+ case 2:
+ entryPointName = "computeC";
+ break;
+ default:
+ // Should never reach this point since we wrote the test
+ SLANG_IGNORE_TEST;
+ }
+ GFX_CHECK_CALL_ABORT(loadComputeProgram(device, shaderProgram, "multiple-entry-point-shader-cache-shader", entryPointName, slangReflection));
+
+ ComputePipelineStateDesc pipelineDesc = {};
+ pipelineDesc.program = shaderProgram.get();
+ GFX_CHECK_CALL_ABORT(
+ device->createComputePipelineState(pipelineDesc, pipelineState.writeRef()));
+ }
+
+ void run()
+ {
+ ComPtr shaderCacheStats;
+
+ // Due to needing a workaround to prevent loading old, outdated modules, we need to
+ // recreate the device between each segment of the test. However, we need to maintain the
+ // same cache filesystem for the duration of the test, so the device is immediately recreated
+ // to ensure we can pass the filesystem all the way through.
+ //
+ // TODO: Remove the repeated generateNewDevice() and createRequiredResources() calls once
+ // a solution exists that allows source code changes under the same module name to be picked
+ // up on load.
+ generateNewDevice();
+ createRequiredResources();
+ generateNewPipelineState(0);
+ submitGPUWork();
+
+ device->queryInterface(SLANG_UUID_IShaderCacheStatistics, (void**)shaderCacheStats.writeRef());
+ SLANG_CHECK(shaderCacheStats->getCacheMissCount() == 1);
+ SLANG_CHECK(shaderCacheStats->getCacheHitCount() == 0);
+ SLANG_CHECK(shaderCacheStats->getCacheEntryDirtyCount() == 0);
+
+ generateNewDevice();
+ createRequiredResources();
+ generateNewPipelineState(1);
+ submitGPUWork();
+ generateNewPipelineState(0);
+ submitGPUWork();
+
+ device->queryInterface(SLANG_UUID_IShaderCacheStatistics, (void**)shaderCacheStats.writeRef());
+ SLANG_CHECK(shaderCacheStats->getCacheMissCount() == 1);
+ SLANG_CHECK(shaderCacheStats->getCacheHitCount() == 1);
+ SLANG_CHECK(shaderCacheStats->getCacheEntryDirtyCount() == 0);
+
+ generateNewDevice();
+ createRequiredResources();
+ generateNewPipelineState(2);
+ submitGPUWork();
+ generateNewPipelineState(1);
+ submitGPUWork();
+ generateNewPipelineState(0);
+ submitGPUWork();
+
+ device->queryInterface(SLANG_UUID_IShaderCacheStatistics, (void**)shaderCacheStats.writeRef());
+ SLANG_CHECK(shaderCacheStats->getCacheMissCount() == 1);
+ SLANG_CHECK(shaderCacheStats->getCacheHitCount() == 2);
+ SLANG_CHECK(shaderCacheStats->getCacheEntryDirtyCount() == 0);
+ }
+ };
+
+ // One shader file contains an import/include, direct code modifications are made to the imported file
+ // This test specifically checks four cases:
+ // 1. import w/o changes in the imported file
+ // 2. import w/ changes in the imported file
+ // 3. #include w/o changes in the included file (the included file is the same as the imported file in the prior step)
+ // 4. #include w/ changes in the included file
+ struct ShaderFileImportsShaderCache : BaseShaderCacheTest
+ {
+ Slang::String importedContentsA = Slang::String(
+ R"(struct TestFunction
+ {
+ void simpleElementAdd(RWStructuredBuffer buffer, uint index)
+ {
+ var input = buffer[index];
+ buffer[index] = input + 1.0f;
+ }
+ };)");
+
+ Slang::String importedContentsB = Slang::String(
+ R"(struct TestFunction
+ {
+ void simpleElementAdd(RWStructuredBuffer buffer, uint index)
+ {
+ var input = buffer[index];
+ buffer[index] = input + 2.0f;
+ }
+ };)");
+
+ Slang::String importFile = Slang::String(
+ R"(import imported;
+
+ uniform RWStructuredBuffer buffer;
+
+ [shader("compute")]
+ [numthreads(4, 1, 1)]
+ void computeMain(
+ uint3 sv_dispatchThreadID : SV_DispatchThreadID)
+ {
+ TestFunction test;
+ for (uint i = 0; i < 4; ++i)
+ {
+ test.simpleElementAdd(buffer, i);
+ }
+ })");
+
+ Slang::String includeFile = Slang::String(
+ R"(#include "imported.slang"
+
+ uniform RWStructuredBuffer buffer;
+
+ [shader("compute")]
+ [numthreads(4, 1, 1)]
+ void computeMain(
+ uint3 sv_dispatchThreadID : SV_DispatchThreadID)
+ {
+ TestFunction test;
+ for (uint i = 0; i < 4; ++i)
+ {
+ test.simpleElementAdd(buffer, i);
+ }
+ })");
+
+ void initializeFiles()
+ {
+ diskFileSystem->saveFile("imported.slang", importedContentsA.getBuffer(), importedContentsA.getLength());
+ diskFileSystem->saveFile("importing-shader-cache-shader.slang", importFile.getBuffer(), importFile.getLength());
+ }
+
+ void modifyImportedFile(Slang::String importedContents)
+ {
+ diskFileSystem->saveFile("imported.slang", importedContents.getBuffer(), importedContents.getLength());
+ }
+
+ void changeImportToInclude()
+ {
+ diskFileSystem->saveFile("importing-shader-cache-shader.slang", includeFile.getBuffer(), includeFile.getLength());
+ }
+
+ void generateNewPipelineState()
+ {
+ ComPtr shaderProgram;
+ slang::ProgramLayout* slangReflection;
+ GFX_CHECK_CALL_ABORT(loadComputeProgram(device, shaderProgram, "importing-shader-cache-shader", "computeMain", slangReflection));
+
+ ComputePipelineStateDesc pipelineDesc = {};
+ pipelineDesc.program = shaderProgram.get();
+ GFX_CHECK_CALL_ABORT(
+ device->createComputePipelineState(pipelineDesc, pipelineState.writeRef()));
+ }
+
+ void run()
+ {
+ ComPtr shaderCacheStats;
+
+ // Due to needing a workaround to prevent loading old, outdated modules, we need to
+ // recreate the device between each segment of the test. However, we need to maintain the
+ // same cache filesystem for the duration of the test, so the device is immediately recreated
+ // to ensure we can pass the filesystem all the way through.
+ //
+ // TODO: Remove the repeated generateNewDevice() and createRequiredResources() calls once
+ // a solution exists that allows source code changes under the same module name to be picked
+ // up on load.
+ generateNewDevice();
+ createRequiredResources();
+ initializeFiles();
+ generateNewPipelineState();
+ submitGPUWork();
+
+ device->queryInterface(SLANG_UUID_IShaderCacheStatistics, (void**)shaderCacheStats.writeRef());
+ SLANG_CHECK(shaderCacheStats->getCacheMissCount() == 1);
+ SLANG_CHECK(shaderCacheStats->getCacheHitCount() == 0);
+ SLANG_CHECK(shaderCacheStats->getCacheEntryDirtyCount() == 0);
+
+ generateNewDevice();
+ createRequiredResources();
+ modifyImportedFile(importedContentsB);
+ generateNewPipelineState();
+ submitGPUWork();
+
+ device->queryInterface(SLANG_UUID_IShaderCacheStatistics, (void**)shaderCacheStats.writeRef());
+ SLANG_CHECK(shaderCacheStats->getCacheMissCount() == 0);
+ SLANG_CHECK(shaderCacheStats->getCacheHitCount() == 0);
+ SLANG_CHECK(shaderCacheStats->getCacheEntryDirtyCount() == 1);
+
+ generateNewDevice();
+ createRequiredResources();
+ changeImportToInclude();
+ generateNewPipelineState();
+ submitGPUWork();
+
+ device->queryInterface(SLANG_UUID_IShaderCacheStatistics, (void**)shaderCacheStats.writeRef());
+ SLANG_CHECK(shaderCacheStats->getCacheMissCount() == 0);
+ SLANG_CHECK(shaderCacheStats->getCacheHitCount() == 0);
+ SLANG_CHECK(shaderCacheStats->getCacheEntryDirtyCount() == 1);
+
+ generateNewDevice();
+ createRequiredResources();
+ modifyImportedFile(importedContentsA);
+ generateNewPipelineState();
+ submitGPUWork();
+
+ device->queryInterface(SLANG_UUID_IShaderCacheStatistics, (void**)shaderCacheStats.writeRef());
+ SLANG_CHECK(shaderCacheStats->getCacheMissCount() == 0);
+ SLANG_CHECK(shaderCacheStats->getCacheHitCount() == 0);
+ SLANG_CHECK(shaderCacheStats->getCacheEntryDirtyCount() == 1);
+ }
+ };
+
+ template
+ void shaderCacheTestImpl(ComPtr device, UnitTestContext* context)
+ {
+ T test;
+ test.init(device, context);
+ test.run();
+ }
+
+ SLANG_UNIT_TEST(singleEntryShaderCacheD3D12)
+ {
+ runTestImpl(shaderCacheTestImpl, unitTestContext, Slang::RenderApiFlag::D3D12);
+ }
+
+ SLANG_UNIT_TEST(singleEntryShaderCacheVulkan)
+ {
+ runTestImpl(shaderCacheTestImpl, unitTestContext, Slang::RenderApiFlag::Vulkan);
+ }
+
+ SLANG_UNIT_TEST(multipleEntryShaderCacheD3D12)
+ {
+ runTestImpl(shaderCacheTestImpl, unitTestContext, Slang::RenderApiFlag::D3D12);
+ }
+
+ SLANG_UNIT_TEST(multipleEntryShaderCacheVulkan)
+ {
+ runTestImpl(shaderCacheTestImpl, unitTestContext, Slang::RenderApiFlag::Vulkan);
+ }
+
+ SLANG_UNIT_TEST(multipleEntryPointShaderCacheD3D12)
+ {
+ runTestImpl(shaderCacheTestImpl, unitTestContext, Slang::RenderApiFlag::D3D12);
+ }
+
+ SLANG_UNIT_TEST(multipleEntryPointShaderCacheVulkan)
+ {
+ runTestImpl(shaderCacheTestImpl, unitTestContext, Slang::RenderApiFlag::Vulkan);
+ }
+
+ SLANG_UNIT_TEST(shaderFileImportsShaderCacheD3D12)
+ {
+ runTestImpl(shaderCacheTestImpl, unitTestContext, Slang::RenderApiFlag::D3D12);
+ }
+
+ SLANG_UNIT_TEST(shaderFileImportsShaderCacheVulkan)
+ {
+ runTestImpl(shaderCacheTestImpl, unitTestContext, Slang::RenderApiFlag::Vulkan);
+ }
+}
diff --git a/tools/gfx/cuda/cuda-device.cpp b/tools/gfx/cuda/cuda-device.cpp
index be5dbbc96..32454109b 100644
--- a/tools/gfx/cuda/cuda-device.cpp
+++ b/tools/gfx/cuda/cuda-device.cpp
@@ -913,7 +913,7 @@ SLANG_NO_THROW Result SLANG_MCALL DeviceImpl::createProgram(
ComPtr kernelCode;
ComPtr diagnostics;
- auto compileResult = desc.slangGlobalScope->getEntryPointCode(
+ auto compileResult = getEntryPointCodeFromShaderCache(desc.slangGlobalScope,
(SlangInt)0, 0, kernelCode.writeRef(), diagnostics.writeRef());
if (diagnostics)
{
diff --git a/tools/gfx/d3d11/d3d11-device.cpp b/tools/gfx/d3d11/d3d11-device.cpp
index aa665ebd4..969eb7d1b 100644
--- a/tools/gfx/d3d11/d3d11-device.cpp
+++ b/tools/gfx/d3d11/d3d11-device.cpp
@@ -1239,7 +1239,7 @@ Result DeviceImpl::createProgram(
ComPtr kernelCode;
ComPtr diagnostics;
- auto compileResult = slangGlobalScope->getEntryPointCode(
+ auto compileResult = getEntryPointCodeFromShaderCache(slangGlobalScope,
(SlangInt)i, 0, kernelCode.writeRef(), diagnostics.writeRef());
if (diagnostics)
diff --git a/tools/gfx/d3d12/d3d12-pipeline-state.cpp b/tools/gfx/d3d12/d3d12-pipeline-state.cpp
index ec073bf44..adfdcd518 100644
--- a/tools/gfx/d3d12/d3d12-pipeline-state.cpp
+++ b/tools/gfx/d3d12/d3d12-pipeline-state.cpp
@@ -50,7 +50,7 @@ Result PipelineStateImpl::ensureAPIPipelineStateCreated()
auto programImpl = static_cast(m_program.Ptr());
if (programImpl->m_shaders.getCount() == 0)
{
- SLANG_RETURN_ON_FAIL(programImpl->compileShaders());
+ SLANG_RETURN_ON_FAIL(programImpl->compileShaders(m_device));
}
if (desc.type == PipelineType::Graphics)
{
@@ -356,7 +356,7 @@ Result RayTracingPipelineStateImpl::ensureAPIPipelineStateCreated()
SlangInt entryPointIndex)
{
ComPtr codeBlob;
- auto compileResult = component->getEntryPointCode(
+ auto compileResult = m_device->getEntryPointCodeFromShaderCache(component,
entryPointIndex, 0, codeBlob.writeRef(), diagnostics.writeRef());
if (diagnostics.get())
{
diff --git a/tools/gfx/gfx.slang b/tools/gfx/gfx.slang
index 17a38e28d..c1296b472 100644
--- a/tools/gfx/gfx.slang
+++ b/tools/gfx/gfx.slang
@@ -1733,6 +1733,8 @@ struct DeviceDesc
void *apiCommandDispatcher = nullptr;
// The slot (typically UAV) used to identify NVAPI intrinsics. If >=0 NVAPI is required.
GfxIndex nvapiExtnSlot = -1;
+ // The root directory for the shader cache.
+ NativeString shaderCachePath;
// The file system for loading cached shader kernels. The layer does not maintain a strong reference to the object,
// instead the user is responsible for holding the object alive during the lifetime of an `IDevice`.
void *shaderCacheFileSystem = nullptr;
diff --git a/tools/gfx/open-gl/render-gl.cpp b/tools/gfx/open-gl/render-gl.cpp
index 7daf577ef..fdec875f0 100644
--- a/tools/gfx/open-gl/render-gl.cpp
+++ b/tools/gfx/open-gl/render-gl.cpp
@@ -2791,7 +2791,7 @@ Result GLDevice::createProgram(
{
ComPtr kernelCode;
ComPtr diagnostics;
- auto compileResult = desc.slangGlobalScope->getEntryPointCode(
+ auto compileResult = getEntryPointCodeFromShaderCache(desc.slangGlobalScope,
i, 0, kernelCode.writeRef(), diagnostics.writeRef());
if (diagnostics)
{
diff --git a/tools/gfx/renderer-shared.cpp b/tools/gfx/renderer-shared.cpp
index 43629a0f2..8ed776776 100644
--- a/tools/gfx/renderer-shared.cpp
+++ b/tools/gfx/renderer-shared.cpp
@@ -3,6 +3,11 @@
#include "core/slang-io.h"
#include "core/slang-token-reader.h"
+#include "../../source/core/slang-file-system.h"
+
+#include "../../slang.h"
+#include "../../source/slang/slang-hash-utils.h"
+
using namespace Slang;
namespace gfx
@@ -23,6 +28,7 @@ const Slang::Guid GfxGUID::IID_IResource = SLANG_UUID_IResource;
const Slang::Guid GfxGUID::IID_IBufferResource = SLANG_UUID_IBufferResource;
const Slang::Guid GfxGUID::IID_ITextureResource = SLANG_UUID_ITextureResource;
const Slang::Guid GfxGUID::IID_IDevice = SLANG_UUID_IDevice;
+const Slang::Guid GfxGUID::IID_IShaderCacheStatistics = SLANG_UUID_IShaderCacheStatistics;
const Slang::Guid GfxGUID::IID_IShaderObject = SLANG_UUID_IShaderObject;
const Slang::Guid GfxGUID::IID_IRenderPassLayout = SLANG_UUID_IRenderPassLayout;
@@ -325,6 +331,129 @@ void PipelineStateBase::initializeBase(const PipelineStateDesc& inDesc)
}
}
+void updateCacheEntry(ISlangMutableFileSystem* fileSystem, slang::IBlob* compiledCode, String shaderFilename, slang::Digest ASTHash)
+{
+ auto hashSize = sizeof(slang::Digest);
+
+ auto bufferSize = hashSize + compiledCode->getBufferSize();
+ List contents;
+ contents.setCount(bufferSize);
+ uint8_t* buffer = contents.begin();
+ memcpy(buffer, &ASTHash, hashSize);
+ memcpy(buffer + hashSize, (void*)compiledCode->getBufferPointer(), compiledCode->getBufferSize());
+ fileSystem->saveFile(shaderFilename.getBuffer(), buffer, bufferSize);
+}
+
+Result RendererBase::getEntryPointCodeFromShaderCache(
+ slang::IComponentType* program,
+ SlangInt entryPointIndex,
+ SlangInt targetIndex,
+ slang::IBlob** outCode,
+ slang::IBlob** outDiagnostics)
+{
+ // TODO: Need a way in filesystem to query both file size and file creation time, if cache size exceeds
+ // specified maximum size (in bytes or files) then delete oldest files - cache eviction policy
+
+ // Immediately call getEntryPointCode if no shader cache was provided on initialization
+ if (!shaderCacheFileSystem)
+ {
+ return program->getEntryPointCode(entryPointIndex, targetIndex, outCode, outDiagnostics);
+ }
+
+ // Produce a string which we can use to query the shader cache by combining two separate hashes which
+ // together comprise all the compilation arguments for this program.
+ ComPtr session;
+ getSlangSession(session.writeRef());
+
+ slang::Digest shaderKeyHash;
+ program->computeDependencyBasedHash(entryPointIndex, targetIndex, &shaderKeyHash);
+
+ StringBuilder shaderKey = hashToString(shaderKeyHash);
+
+ // Produce a hash using the AST for this program - This is needed to check whether a cache entry is effectively dirty,
+ // or to save along with the compiled code into an entry so the entry can be checked if fetched later on.
+ slang::Digest ASTHash;
+ program->computeASTBasedHash(&ASTHash);
+
+ ComPtr codeBlob;
+
+ // Query shaderCacheFileSystem for an entry whose key matches shaderFilename
+ // - If we find it, then copy the file contents into memory and return in outCode
+ auto result = shaderCacheFileSystem->loadFile(shaderKey.getBuffer(), codeBlob.writeRef());
+
+ if (SLANG_FAILED(result))
+ {
+ // If we didn't find it, call program->getEntryPointCode() to get and return the code. We also
+ // make sure to save a new entry in the shader cache.
+ if (SLANG_SUCCEEDED(program->getEntryPointCode(entryPointIndex, targetIndex, codeBlob.writeRef(), outDiagnostics)))
+ {
+ if (mutableShaderCacheFileSystem)
+ {
+ updateCacheEntry(mutableShaderCacheFileSystem, codeBlob, shaderKey, ASTHash);
+ }
+
+ shaderCacheMissCount++;
+ }
+ else
+ {
+ // If getEntryPointCode() failed to fetch the code, we return SLANG_FAIL along with the diagnostics output
+ // in outDiagnostics.
+ return SLANG_FAIL;
+ }
+ }
+ else
+ {
+ // If the entry exists, we need to check that the entry isn't effectively dirty. Since we stored
+ // the AST hash with the compiled code, we can determine this by comparing the stored hash with the
+ // AST hash generated earlier.
+ auto entryContents = codeBlob->getBufferPointer();
+ auto hashSize = sizeof(slang::Digest);
+ if (memcmp(ASTHash.values, entryContents, hashSize) != 0)
+ {
+ // The AST hash stored in the entry does not match the AST hash generated earlier, indicating
+ // that the shader code has changed and the entry needs to be updated.
+ if (SLANG_SUCCEEDED(program->getEntryPointCode(entryPointIndex, targetIndex, codeBlob.writeRef(), outDiagnostics)))
+ {
+ if (mutableShaderCacheFileSystem)
+ {
+ updateCacheEntry(mutableShaderCacheFileSystem, codeBlob, shaderKey, ASTHash);
+ }
+
+ shaderCacheEntryDirtyCount++;
+ }
+ else
+ {
+ // If getEntryPointCode() failed to fetch the code, we return SLANG_FAIL along with the diagnostics output
+ // in outDiagnostics.
+ return SLANG_FAIL;
+ }
+ }
+ else
+ {
+ auto compiledCode = RawBlob::create((uint8_t*)codeBlob->getBufferPointer() + hashSize, codeBlob->getBufferSize() - hashSize);
+ codeBlob = compiledCode;
+
+ shaderCacheHitCount++;
+ }
+ }
+
+ *outCode = codeBlob.detach();
+ return SLANG_OK;
+}
+
+SlangResult RendererBase::queryInterface(SlangUUID const& uuid, void** outObject)
+{
+ if (uuid == GfxGUID::IID_IShaderCacheStatistics)
+ {
+ *outObject = static_cast(this);
+ addRef();
+ return SLANG_OK;
+ }
+
+ *outObject = getInterface(uuid);
+ return SLANG_OK;
+}
+
IDevice* gfx::RendererBase::getInterface(const Guid& guid)
{
return (guid == GfxGUID::IID_ISlangUnknown || guid == GfxGUID::IID_IDevice)
@@ -334,6 +463,28 @@ IDevice* gfx::RendererBase::getInterface(const Guid& guid)
SLANG_NO_THROW Result SLANG_MCALL RendererBase::initialize(const Desc& desc)
{
+ // If a shader cache file system was provided, use the provided system.
+ if (desc.shaderCacheFileSystem)
+ {
+ shaderCacheFileSystem = desc.shaderCacheFileSystem;
+ }
+ if (desc.shaderCachePath)
+ {
+ // Only a path was provided, create a RelativeFileSystem using the path
+ if (!shaderCacheFileSystem)
+ {
+ shaderCacheFileSystem = OSFileSystem::getMutableSingleton();
+ }
+ shaderCacheFileSystem = new RelativeFileSystem(shaderCacheFileSystem, desc.shaderCachePath);
+ }
+
+ // If we initialized a file system for the shader cache, check if it's mutable. If so, store a pointer
+ // to the mutable version in order to save new entries later.
+ if (shaderCacheFileSystem)
+ {
+ shaderCacheFileSystem->queryInterface(ISlangMutableFileSystem::getTypeGuid(), (void**)mutableShaderCacheFileSystem.writeRef());
+ }
+
if (desc.apiCommandDispatcher)
{
desc.apiCommandDispatcher->queryInterface(
@@ -664,7 +815,28 @@ Result RendererBase::getShaderObjectLayout(
return SLANG_OK;
}
+GfxCount RendererBase::getCacheMissCount()
+{
+ return shaderCacheMissCount;
+}
+GfxCount RendererBase::getCacheHitCount()
+{
+ return shaderCacheHitCount;
+}
+
+GfxCount RendererBase::getCacheEntryDirtyCount()
+{
+ return shaderCacheEntryDirtyCount;
+}
+
+Result RendererBase::resetCacheStatistics()
+{
+ shaderCacheMissCount = 0;
+ shaderCacheHitCount = 0;
+ shaderCacheEntryDirtyCount = 0;
+ return SLANG_OK;
+}
ShaderComponentID ShaderCache::getComponentId(slang::TypeReflection* type)
{
@@ -908,7 +1080,7 @@ void ShaderProgramBase::init(const IShaderProgram::Desc& inDesc)
}
}
-Result ShaderProgramBase::compileShaders()
+Result ShaderProgramBase::compileShaders(RendererBase* device)
{
// For a fully specialized program, read and store its kernel code in `shaderProgram`.
auto compileShader = [&](slang::EntryPointReflection* entryPointInfo,
@@ -918,7 +1090,7 @@ Result ShaderProgramBase::compileShaders()
auto stage = entryPointInfo->getStage();
ComPtr kernelCode;
ComPtr diagnostics;
- auto compileResult = entryPointComponent->getEntryPointCode(
+ auto compileResult = device->getEntryPointCodeFromShaderCache(entryPointComponent,
entryPointIndex, 0, kernelCode.writeRef(), diagnostics.writeRef());
if (diagnostics)
{
diff --git a/tools/gfx/renderer-shared.h b/tools/gfx/renderer-shared.h
index d0e4b52fb..a753fe017 100644
--- a/tools/gfx/renderer-shared.h
+++ b/tools/gfx/renderer-shared.h
@@ -26,6 +26,7 @@ struct GfxGUID
static const Slang::Guid IID_ITextureResource;
static const Slang::Guid IID_IInputLayout;
static const Slang::Guid IID_IDevice;
+ static const Slang::Guid IID_IShaderCacheStatistics;
static const Slang::Guid IID_IShaderObjectLayout;
static const Slang::Guid IID_IShaderObject;
static const Slang::Guid IID_IRenderPassLayout;
@@ -857,7 +858,7 @@ public:
return false;
}
- Slang::Result compileShaders();
+ Slang::Result compileShaders(RendererBase* device);
virtual Slang::Result createShaderModule(
slang::EntryPointReflection* entryPointInfo, Slang::ComPtr kernelCode);
@@ -1211,11 +1212,12 @@ public:
// Renderer implementation shared by all platforms.
// Responsible for shader compilation, specialization and caching.
-class RendererBase : public IDevice, public Slang::ComObject
+class RendererBase : public IDevice, public IShaderCacheStatistics, public Slang::ComObject
{
friend class ShaderObjectBase;
public:
- SLANG_COM_OBJECT_IUNKNOWN_ALL
+ SLANG_COM_OBJECT_IUNKNOWN_ADD_REF
+ SLANG_COM_OBJECT_IUNKNOWN_RELEASE
virtual SLANG_NO_THROW Result SLANG_MCALL getNativeDeviceHandles(InteropHandles* outHandles) SLANG_OVERRIDE;
virtual SLANG_NO_THROW Result SLANG_MCALL getFeatures(
@@ -1224,6 +1226,8 @@ public:
virtual SLANG_NO_THROW Result SLANG_MCALL
getFormatSupportedResourceStates(Format format, ResourceStateSet* outStates) override;
virtual SLANG_NO_THROW Result SLANG_MCALL getSlangSession(slang::ISession** outSlangSession) SLANG_OVERRIDE;
+ virtual SLANG_NO_THROW SlangResult SLANG_MCALL
+ queryInterface(SlangUUID const& uuid, void** outObject) SLANG_OVERRIDE;
IDevice* getInterface(const Slang::Guid& guid);
virtual SLANG_NO_THROW Result SLANG_MCALL createTextureFromNativeHandle(
@@ -1309,6 +1313,13 @@ public:
// Provides a default implementation that returns SLANG_E_NOT_AVAILABLE.
virtual SLANG_NO_THROW Result SLANG_MCALL getTextureRowAlignment(size_t* outAlignment) override;
+ Result getEntryPointCodeFromShaderCache(
+ slang::IComponentType* program,
+ SlangInt entryPointIndex,
+ SlangInt targetIndex,
+ slang::IBlob** outCode,
+ slang::IBlob** outDiagnostics = nullptr);
+
Result getShaderObjectLayout(
slang::TypeReflection* type,
ShaderObjectContainerType container,
@@ -1346,10 +1357,24 @@ protected:
protected:
Slang::List m_features;
+public:
+ virtual SLANG_NO_THROW GfxCount SLANG_MCALL getCacheMissCount() override;
+ virtual SLANG_NO_THROW GfxCount SLANG_MCALL getCacheHitCount() override;
+ virtual SLANG_NO_THROW GfxCount SLANG_MCALL getCacheEntryDirtyCount() override;
+ virtual SLANG_NO_THROW Result SLANG_MCALL resetCacheStatistics() override;
+
+protected:
+ GfxCount shaderCacheMissCount = 0;
+ GfxCount shaderCacheHitCount = 0;
+ GfxCount shaderCacheEntryDirtyCount = 0;
+
public:
SlangContext slangContext;
ShaderCache shaderCache;
+ ISlangFileSystem* shaderCacheFileSystem = nullptr;
+ ComPtr mutableShaderCacheFileSystem = nullptr;
+
Slang::Dictionary> m_shaderObjectLayoutCache;
Slang::ComPtr m_pipelineCreationAPIDispatcher;
};
diff --git a/tools/gfx/vulkan/vk-pipeline-state.cpp b/tools/gfx/vulkan/vk-pipeline-state.cpp
index 710cbdaef..06bd13197 100644
--- a/tools/gfx/vulkan/vk-pipeline-state.cpp
+++ b/tools/gfx/vulkan/vk-pipeline-state.cpp
@@ -251,7 +251,7 @@ Result PipelineStateImpl::createVKGraphicsPipelineState()
auto programImpl = static_cast(m_program.Ptr());
if (programImpl->m_stageCreateInfos.getCount() == 0)
{
- SLANG_RETURN_ON_FAIL(programImpl->compileShaders());
+ SLANG_RETURN_ON_FAIL(programImpl->compileShaders(m_device));
}
pipelineInfo.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO;
@@ -281,7 +281,7 @@ Result PipelineStateImpl::createVKComputePipelineState()
auto programImpl = static_cast(m_program.Ptr());
if (programImpl->m_stageCreateInfos.getCount() == 0)
{
- SLANG_RETURN_ON_FAIL(programImpl->compileShaders());
+ SLANG_RETURN_ON_FAIL(programImpl->compileShaders(m_device));
}
VkPipelineCache pipelineCache = VK_NULL_HANDLE;
@@ -340,7 +340,7 @@ Result RayTracingPipelineStateImpl::createVKRayTracingPipelineState()
auto programImpl = static_cast(m_program.Ptr());
if (programImpl->m_stageCreateInfos.getCount() == 0)
{
- SLANG_RETURN_ON_FAIL(programImpl->compileShaders());
+ SLANG_RETURN_ON_FAIL(programImpl->compileShaders(m_device));
}
VkRayTracingPipelineCreateInfoKHR raytracingPipelineInfo = {
diff --git a/tools/slang-unit-test/unit-test-checksum.cpp b/tools/slang-unit-test/unit-test-checksum.cpp
new file mode 100644
index 000000000..8980bd31f
--- /dev/null
+++ b/tools/slang-unit-test/unit-test-checksum.cpp
@@ -0,0 +1,32 @@
+// unit-test-checksum.cpp
+
+#include "tools/unit-test/slang-unit-test.h"
+
+#include "../../source/slang/slang-hash-utils.h"
+
+using namespace Slang;
+
+SLANG_UNIT_TEST(checksum)
+{
+ {
+ slang::Digest testA;
+ testA.values[0] = 1;
+ testA.values[1] = 2;
+ testA.values[2] = 3;
+ testA.values[3] = 4;
+
+ String testAString = hashToString(testA);
+ SLANG_CHECK(testAString.equals(String("00000001000000020000000300000004")));
+ }
+
+ {
+ slang::Digest testC;
+ testC.values[0] = 0x11111111;
+ testC.values[1] = 0x22222222;
+ testC.values[2] = 0x33333333;
+ testC.values[3] = 0x44444444;
+
+ String testCString = hashToString(testC);
+ SLANG_CHECK(testCString.equals(String("11111111222222223333333344444444")));
+ }
+}
diff --git a/tools/slang-unit-test/unit-test-md5.cpp b/tools/slang-unit-test/unit-test-md5.cpp
new file mode 100644
index 000000000..95235e0ed
--- /dev/null
+++ b/tools/slang-unit-test/unit-test-md5.cpp
@@ -0,0 +1,97 @@
+// unit-test-md5.cpp
+#include "tools/unit-test/slang-unit-test.h"
+
+#include "../../source/core/slang-md5.h"
+#include "../../source/core/slang-string.h"
+#include "../../source/slang/slang-hash-utils.h"
+
+using namespace Slang;
+
+SLANG_UNIT_TEST(md5hash)
+{
+ {
+ // Raw numerical values, etc.
+ MD5Context testCtx;
+ MD5HashGen testHashGen;
+ testHashGen.init(&testCtx);
+
+ int64_t valueA = -1;
+ uint64_t valueB = 1;
+ testHashGen.update(&testCtx, valueA);
+ testHashGen.update(&testCtx, valueB);
+
+ slang::Digest testA;
+ testHashGen.finalize(&testCtx, &testA);
+
+ String testAString = hashToString(testA);
+ SLANG_CHECK(testAString.equals(String("E271A15BD2BD98081390630579266F74")));
+ }
+
+ {
+ // List
+ MD5Context testCtx;
+ MD5HashGen testHashGen;
+ testHashGen.init(&testCtx);
+
+ List listA;
+ listA.add(1);
+ listA.add(2);
+ listA.add(3);
+ listA.add(4);
+ testHashGen.update(&testCtx, listA);
+
+ slang::Digest testB;
+ testHashGen.finalize(&testCtx, &testB);
+
+ String testBString = hashToString(testB);
+ SLANG_CHECK(testBString.equals(String("8AD852437539AA78D60CF70BA5CA7BF2")));
+ }
+
+ {
+ // UnownedStringSlice
+ MD5Context testCtx;
+ MD5HashGen testHashGen;
+ testHashGen.init(&testCtx);
+
+ UnownedStringSlice stringSlice = UnownedStringSlice("String Slice Test");
+ testHashGen.update(&testCtx, stringSlice);
+
+ slang::Digest testC;
+ testHashGen.finalize(&testCtx, &testC);
+
+ String testCString = hashToString(testC);
+ SLANG_CHECK(testCString.equals(String("8EC56C5DDFA424183957CFD01633605B")));
+ }
+
+ {
+ // String
+ MD5Context testCtx;
+ MD5HashGen testHashGen;
+ testHashGen.init(&testCtx);
+
+ String str = String("String Test");
+ testHashGen.update(&testCtx, str);
+
+ slang::Digest testD;
+ testHashGen.finalize(&testCtx, &testD);
+
+ String testDString = hashToString(testD);
+ SLANG_CHECK(testDString.equals(String("CC795ADF40C7702106A5F01C24CEB0CE")));
+ }
+
+ {
+ // Hash
+ MD5Context testCtx;
+ MD5HashGen testHashGen;
+ testHashGen.init(&testCtx);
+
+ slang::Digest Hash;
+ testHashGen.update(&testCtx, Hash);
+
+ slang::Digest testE;
+ testHashGen.finalize(&testCtx, &testE);
+
+ String testEString = hashToString(testE);
+ SLANG_CHECK(testEString.equals(String("3613E74ABFF94BE42E75D279A5184823")));
+ }
+}
--
cgit v1.2.3