Shader caching (#2432)

* Changed all getEntryPointCode calls to use RendererBase::getEntryPointCodeFromShaderCache

* Hashing hooked up, tests pass but need to add more to fully test functionality

* checkpoint

* Checkpoint: File system creation seems functional, saving is broken

* checkpoint: Fixed filename generation from MD5 hash, shader blob might be going missing ahead of pipeline state creation

* Fixed a lot of bugs related to hash code generation, shader cache is likely working but needs further testing

* Added workaround for module loading by re-creating the test device, shader cache test functional

* Vulkan shader caching bug fixed, checkpoint commit before more refinement

* pre-ToT merge checkpoint

* checkpoint commit, improving cache keys

* Significantly expanded items included in the dependency hash for Module; Added dependency hash functions to SpecializedComponentType and RenamedEntryPointComponentType

* Temporarily disable shader cache test

* Mid cleanup changes, solution successfully builds

* Added several helper update functions to slang-md5 to help simplify usage; Added a function under ISession to compute a hash for all linkage-related items; Function renames and cleaned up some comments

* Ran premake.bat; Renamed getASTBasedHashCode to computeASTBasedHash

* Added slang unit tests for Checksum and MD5; Extended gfx shader cache test to test with multiple shader files and one shader file with multiple entry points

* Solution builds and shader cache tests pass, but at least a couple other tests now failing

* ran premake.bat

* More cleanup changes

* Added shaderCachePath field to IDevice desc in gfx.slang, gfx-smoke.slang should be functional

* ran premake

* cleanup changes; Adding test printf to getEntryPointCodeFromShaderCache to see if output can be seen in CI

* Removed debugging printfs; Added handling for getEntryPointCode() failing

* Cleanup changes; Jonathan's fixes to SerialWriter to zero initialize otherwise uninitialized memory; Change to SwizzleExpr creation to zero initialize elementCount

* Changed enable_if_t to enable_if

* Fixed enable_if

* Added test for import vs include and changes to included and imported files; Fixed build errors in CUDA; Renamed shader cache statistics fields

* cleanup changes

* Readd removed file

* Restructured computeDependencyBasedHash calls, added computeDependencyBasedHashImpl to all classes dervied from ComponentType

* Applied same restructuring to the AST hash functions

* Cleanup changes; Moved HashBuilder out to slang-digest.h and added some helper functions to streamline the process of adding items to a hash

* Cleanup; Fixed incorrect expected results for shader import and include test

3 files changed, 425 insertions, 0 deletions

diff --git a/source/core/slang-digest.h b/source/core/slang-digest.h
new file mode 100644
index 000000000..d1bc80b40
--- /dev/null
+++ b/source/core/slang-digest.h
@@ -0,0 +1,31 @@
+#pragma once
+#include "slang-md5.h"
+#include "../../slang.h"
+
+namespace Slang
+{
+    // Wrapper struct that holds objects necessary for hashing.
+    struct DigestBuilder
+    {
+    public:
+        DigestBuilder()
+        {
+            hashGen.init(&context);
+        }
+
+        template <typename T>
+        void addToDigest(T item)
+        {
+            hashGen.update(&context, item);
+        }
+
+        void finalize(slang::Digest* outHash)
+        {
+            hashGen.finalize(&context, outHash);
+        }
+
+    private:
+        MD5HashGen hashGen;
+        MD5Context context;
+    };
+}
diff --git a/source/core/slang-md5.cpp b/source/core/slang-md5.cpp
new file mode 100644
index 000000000..95b4816bf
--- /dev/null
+++ b/source/core/slang-md5.cpp
@@ -0,0 +1,312 @@
+/*
+ * This is an OpenSSL-compatible implementation of the RSA Data Security, Inc.
+ * MD5 Message-Digest Algorithm (RFC 1321).
+ *
+ * Homepage:
+ * http://openwall.info/wiki/people/solar/software/public-domain-source-code/md5
+ *
+ * Author:
+ * Alexander Peslyak, better known as Solar Designer <solar at openwall.com>
+ *
+ * This software was written by Alexander Peslyak in 2001.  No copyright is
+ * claimed, and the software is hereby placed in the public domain.
+ * In case this attempt to disclaim copyright and place the software in the
+ * public domain is deemed null and void, then the software is
+ * Copyright (c) 2001 Alexander Peslyak and it is hereby released to the
+ * general public under the following terms:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted.
+ *
+ * There's ABSOLUTELY NO WARRANTY, express or implied.
+ *
+ * (This is a heavily cut-down "BSD license".)
+ *
+ * This differs from Colin Plumb's older public domain implementation in that
+ * no exactly 32-bit integer data type is required (any 32-bit or wider
+ * unsigned integer data type will do), there's no compile-time endianness
+ * configuration, and the function prototypes match OpenSSL's.  No code from
+ * Colin Plumb's implementation has been reused; this comment merely compares
+ * the properties of the two independent implementations.
+ *
+ * The primary goals of this implementation are portability and ease of use.
+ * It is meant to be fast, but not as fast as possible.  Some known
+ * optimizations are not included to reduce source code size and avoid
+ * compile-time configuration.
+ */
+ 
+#ifndef HAVE_OPENSSL
+ 
+#include <string.h>
+ 
+#include "slang-md5.h"
+
+namespace Slang
+{
+    /*
+     * The basic MD5 functions.
+     *
+     * F and G are optimized compared to their RFC 1321 definitions for
+     * architectures that lack an AND-NOT instruction, just like in Colin Plumb's
+     * implementation.
+     */
+    #define F(x, y, z)			((z) ^ ((x) & ((y) ^ (z))))
+    #define G(x, y, z)			((y) ^ ((z) & ((x) ^ (y))))
+    #define H(x, y, z)			(((x) ^ (y)) ^ (z))
+    #define H2(x, y, z)			((x) ^ ((y) ^ (z)))
+    #define I(x, y, z)			((y) ^ ((x) | ~(z)))
+ 
+    /*
+     * The MD5 transformation for all four rounds.
+     */
+    #define STEP(f, a, b, c, d, x, t, s) \
+	    (a) += f((b), (c), (d)) + (x) + (t); \
+	    (a) = (((a) << (s)) | (((a) & 0xffffffff) >> (32 - (s)))); \
+	    (a) += (b);
+ 
+    /*
+     * SET reads 4 input bytes in little-endian byte order and stores them in a
+     * properly aligned word in host byte order.
+     *
+     * The check for little-endian architectures that tolerate unaligned memory
+     * accesses is just an optimization.  Nothing will break if it fails to detect
+     * a suitable architecture.
+     *
+     * Unfortunately, this optimization may be a C strict aliasing rules violation
+     * if the caller's data buffer has effective type that cannot be aliased by
+     * MD5_u32plus.  In practice, this problem may occur if these MD5 routines are
+     * inlined into a calling function, or with future and dangerously advanced
+     * link-time optimizations.  For the time being, keeping these MD5 routines in
+     * their own translation unit avoids the problem.
+     */
+    #if defined(__i386__) || defined(__x86_64__) || defined(__vax__)
+    #define SET(n) \
+	    (*(MD5_u32plus *)&ptr[(n) * 4])
+    #define GET(n) \
+	    SET(n)
+    #else
+    #define SET(n) \
+	    (ctx->block[(n)] = \
+	    (MD5_u32plus)ptr[(n) * 4] | \
+	    ((MD5_u32plus)ptr[(n) * 4 + 1] << 8) | \
+	    ((MD5_u32plus)ptr[(n) * 4 + 2] << 16) | \
+	    ((MD5_u32plus)ptr[(n) * 4 + 3] << 24))
+    #define GET(n) \
+	    (ctx->block[(n)])
+    #endif
+ 
+    /*
+     * This processes one or more 64-byte data blocks, but does NOT update the bit
+     * counters.  There are no alignment requirements.
+     */
+    /*static*/const void* MD5HashGen::body(MD5Context* ctx, const void* data, SlangInt size)
+    {
+	    const unsigned char* ptr;
+	    MD5_u32plus a, b, c, d;
+	    MD5_u32plus saved_a, saved_b, saved_c, saved_d;
+ 
+	    ptr = (const unsigned char*)data;
+ 
+	    a = ctx->a;
+	    b = ctx->b;
+	    c = ctx->c;
+	    d = ctx->d;
+ 
+	    do {
+		    saved_a = a;
+		    saved_b = b;
+		    saved_c = c;
+		    saved_d = d;
+ 
+    /* Round 1 */
+		    STEP(F, a, b, c, d, SET(0), 0xd76aa478, 7)
+		    STEP(F, d, a, b, c, SET(1), 0xe8c7b756, 12)
+		    STEP(F, c, d, a, b, SET(2), 0x242070db, 17)
+		    STEP(F, b, c, d, a, SET(3), 0xc1bdceee, 22)
+		    STEP(F, a, b, c, d, SET(4), 0xf57c0faf, 7)
+		    STEP(F, d, a, b, c, SET(5), 0x4787c62a, 12)
+		    STEP(F, c, d, a, b, SET(6), 0xa8304613, 17)
+		    STEP(F, b, c, d, a, SET(7), 0xfd469501, 22)
+		    STEP(F, a, b, c, d, SET(8), 0x698098d8, 7)
+		    STEP(F, d, a, b, c, SET(9), 0x8b44f7af, 12)
+		    STEP(F, c, d, a, b, SET(10), 0xffff5bb1, 17)
+		    STEP(F, b, c, d, a, SET(11), 0x895cd7be, 22)
+		    STEP(F, a, b, c, d, SET(12), 0x6b901122, 7)
+		    STEP(F, d, a, b, c, SET(13), 0xfd987193, 12)
+		    STEP(F, c, d, a, b, SET(14), 0xa679438e, 17)
+		    STEP(F, b, c, d, a, SET(15), 0x49b40821, 22)
+ 
+    /* Round 2 */
+		    STEP(G, a, b, c, d, GET(1), 0xf61e2562, 5)
+		    STEP(G, d, a, b, c, GET(6), 0xc040b340, 9)
+		    STEP(G, c, d, a, b, GET(11), 0x265e5a51, 14)
+		    STEP(G, b, c, d, a, GET(0), 0xe9b6c7aa, 20)
+		    STEP(G, a, b, c, d, GET(5), 0xd62f105d, 5)
+		    STEP(G, d, a, b, c, GET(10), 0x02441453, 9)
+		    STEP(G, c, d, a, b, GET(15), 0xd8a1e681, 14)
+		    STEP(G, b, c, d, a, GET(4), 0xe7d3fbc8, 20)
+		    STEP(G, a, b, c, d, GET(9), 0x21e1cde6, 5)
+		    STEP(G, d, a, b, c, GET(14), 0xc33707d6, 9)
+		    STEP(G, c, d, a, b, GET(3), 0xf4d50d87, 14)
+		    STEP(G, b, c, d, a, GET(8), 0x455a14ed, 20)
+		    STEP(G, a, b, c, d, GET(13), 0xa9e3e905, 5)
+		    STEP(G, d, a, b, c, GET(2), 0xfcefa3f8, 9)
+		    STEP(G, c, d, a, b, GET(7), 0x676f02d9, 14)
+		    STEP(G, b, c, d, a, GET(12), 0x8d2a4c8a, 20)
+ 
+    /* Round 3 */
+		    STEP(H, a, b, c, d, GET(5), 0xfffa3942, 4)
+		    STEP(H2, d, a, b, c, GET(8), 0x8771f681, 11)
+		    STEP(H, c, d, a, b, GET(11), 0x6d9d6122, 16)
+		    STEP(H2, b, c, d, a, GET(14), 0xfde5380c, 23)
+		    STEP(H, a, b, c, d, GET(1), 0xa4beea44, 4)
+		    STEP(H2, d, a, b, c, GET(4), 0x4bdecfa9, 11)
+		    STEP(H, c, d, a, b, GET(7), 0xf6bb4b60, 16)
+		    STEP(H2, b, c, d, a, GET(10), 0xbebfbc70, 23)
+		    STEP(H, a, b, c, d, GET(13), 0x289b7ec6, 4)
+		    STEP(H2, d, a, b, c, GET(0), 0xeaa127fa, 11)
+		    STEP(H, c, d, a, b, GET(3), 0xd4ef3085, 16)
+		    STEP(H2, b, c, d, a, GET(6), 0x04881d05, 23)
+		    STEP(H, a, b, c, d, GET(9), 0xd9d4d039, 4)
+		    STEP(H2, d, a, b, c, GET(12), 0xe6db99e5, 11)
+		    STEP(H, c, d, a, b, GET(15), 0x1fa27cf8, 16)
+		    STEP(H2, b, c, d, a, GET(2), 0xc4ac5665, 23)
+ 
+    /* Round 4 */
+		    STEP(I, a, b, c, d, GET(0), 0xf4292244, 6)
+		    STEP(I, d, a, b, c, GET(7), 0x432aff97, 10)
+		    STEP(I, c, d, a, b, GET(14), 0xab9423a7, 15)
+		    STEP(I, b, c, d, a, GET(5), 0xfc93a039, 21)
+		    STEP(I, a, b, c, d, GET(12), 0x655b59c3, 6)
+		    STEP(I, d, a, b, c, GET(3), 0x8f0ccc92, 10)
+		    STEP(I, c, d, a, b, GET(10), 0xffeff47d, 15)
+		    STEP(I, b, c, d, a, GET(1), 0x85845dd1, 21)
+		    STEP(I, a, b, c, d, GET(8), 0x6fa87e4f, 6)
+		    STEP(I, d, a, b, c, GET(15), 0xfe2ce6e0, 10)
+		    STEP(I, c, d, a, b, GET(6), 0xa3014314, 15)
+		    STEP(I, b, c, d, a, GET(13), 0x4e0811a1, 21)
+		    STEP(I, a, b, c, d, GET(4), 0xf7537e82, 6)
+		    STEP(I, d, a, b, c, GET(11), 0xbd3af235, 10)
+		    STEP(I, c, d, a, b, GET(2), 0x2ad7d2bb, 15)
+		    STEP(I, b, c, d, a, GET(9), 0xeb86d391, 21)
+ 
+		    a += saved_a;
+		    b += saved_b;
+		    c += saved_c;
+		    d += saved_d;
+ 
+		    ptr += 64;
+	    } while (size -= 64);
+ 
+	    ctx->a = a;
+	    ctx->b = b;
+	    ctx->c = c;
+	    ctx->d = d;
+ 
+	    return ptr;
+    }
+ 
+    void MD5HashGen::init(MD5Context* ctx)
+    {
+	    ctx->a = 0x67452301;
+	    ctx->b = 0xefcdab89;
+	    ctx->c = 0x98badcfe;
+	    ctx->d = 0x10325476;
+ 
+	    ctx->lo = 0;
+	    ctx->hi = 0;
+    }
+
+    void MD5HashGen::update(MD5Context* ctx, UnownedStringSlice string)
+    {
+        update(ctx, string.begin(), string.getLength());
+    }
+
+    void MD5HashGen::update(MD5Context* ctx, String str)
+    {
+        update(ctx, str.getBuffer(), str.getLength());
+    }
+
+    void MD5HashGen::update(MD5Context* ctx, const slang::Digest& hash)
+    {
+        update(ctx, hash.values, sizeof(hash.values));
+    }
+
+    void MD5HashGen::update(MD5Context* ctx, const void* data, SlangInt size)
+    {
+	    MD5_u32plus saved_lo;
+	    SlangInt used, available;
+ 
+	    saved_lo = ctx->lo;
+	    if ((ctx->lo = (saved_lo + size) & 0x1fffffff) < saved_lo)
+		    ctx->hi++;
+	    ctx->hi += (MD5_u32plus)size >> 29;
+ 
+	    used = saved_lo & 0x3f;
+ 
+	    if (used) {
+		    available = 64 - used;
+ 
+		    if (size < available) {
+			    memcpy(&ctx->buffer[used], data, size);
+			    return;
+		    }
+ 
+		    memcpy(&ctx->buffer[used], data, available);
+		    data = (const unsigned char*)data + available;
+		    size -= available;
+		    body(ctx, ctx->buffer, 64);
+	    }
+ 
+	    if (size >= 64) {
+		    data = body(ctx, data, size & ~(SlangInt)0x3f);
+		    size &= 0x3f;
+	    }
+ 
+	    memcpy(ctx->buffer, data, size);
+    }
+ 
+    #define OUTUINT(dst, src) \
+	    (dst)[0] = (uint32_t)src; \
+
+    #define OUTCHAR(dst, src) \
+	    (dst)[0] = (unsigned char)(src); \
+	    (dst)[1] = (unsigned char)((src) >> 8); \
+	    (dst)[2] = (unsigned char)((src) >> 16); \
+	    (dst)[3] = (unsigned char)((src) >> 24);
+ 
+    void MD5HashGen::finalize(MD5Context* ctx, slang::Digest* result)
+    {
+	    unsigned long used, available;
+ 
+	    used = ctx->lo & 0x3f;
+ 
+	    ctx->buffer[used++] = 0x80;
+ 
+	    available = 64 - used;
+ 
+	    if (available < 8) {
+		    memset(&ctx->buffer[used], 0, available);
+		    body(ctx, ctx->buffer, 64);
+		    used = 0;
+		    available = 64;
+	    }
+ 
+	    memset(&ctx->buffer[used], 0, available - 8);
+ 
+	    ctx->lo <<= 3;
+	    OUTCHAR(&ctx->buffer[56], ctx->lo)
+	    OUTCHAR(&ctx->buffer[60], ctx->hi)
+ 
+	    body(ctx, ctx->buffer, 64);
+
+	    OUTUINT(&result->values[0], ctx->a)
+	    OUTUINT(&result->values[1], ctx->b)
+	    OUTUINT(&result->values[2], ctx->c)
+	    OUTUINT(&result->values[3], ctx->d)
+ 
+	    memset(ctx, 0, sizeof(*ctx));
+    }
+}
+ 
+#endif
diff --git a/source/core/slang-md5.h b/source/core/slang-md5.h
new file mode 100644
index 000000000..8c51a03ec
--- /dev/null
+++ b/source/core/slang-md5.h
@@ -0,0 +1,82 @@
+/*
+ * This is an OpenSSL-compatible implementation of the RSA Data Security, Inc.
+ * MD5 Message-Digest Algorithm (RFC 1321).
+ *
+ * Homepage:
+ * http://openwall.info/wiki/people/solar/software/public-domain-source-code/md5
+ *
+ * Author:
+ * Alexander Peslyak, better known as Solar Designer <solar at openwall.com>
+ *
+ * This software was written by Alexander Peslyak in 2001.  No copyright is
+ * claimed, and the software is hereby placed in the public domain.
+ * In case this attempt to disclaim copyright and place the software in the
+ * public domain is deemed null and void, then the software is
+ * Copyright (c) 2001 Alexander Peslyak and it is hereby released to the
+ * general public under the following terms:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted.
+ *
+ * There's ABSOLUTELY NO WARRANTY, express or implied.
+ *
+ * See md5.c for more information.
+ */
+ 
+#ifdef HAVE_OPENSSL
+#include <openssl/md5.h>
+#elif !defined(_MD5_H)
+#define _MD5_H
+
+#pragma once
+#include "../../slang.h"
+#include "../core/slang-string.h"
+#include "../core/slang-list.h"
+
+namespace Slang
+{
+    /* Any 32-bit or wider unsigned integer data type will do */
+    typedef uint32_t MD5_u32plus;
+
+    struct MD5Context
+    {
+        MD5_u32plus lo, hi;
+        MD5_u32plus a, b, c, d;
+        unsigned char buffer[64];
+        MD5_u32plus block[16];
+    };
+
+    class MD5HashGen
+    {
+    public:
+        void init(MD5Context* ctx);
+
+        // Helper update function for raw values (e.g. ints, uints)
+        template<typename T,
+            typename = typename std::enable_if<std::is_enum<T>::value || std::is_arithmetic<T>::value>::type>
+        void update(MD5Context* ctx, const T& val)
+        {
+            update(ctx, &val, sizeof(T));
+        }
+        // Helper update function for Slang::List
+        template<typename T>
+        void update(MD5Context* ctx, const List<T>& list)
+        {
+            update(ctx, list.getBuffer(), list.getCount());
+        }
+        // Helper update function for UnownedStringSlice
+        void update(MD5Context* ctx, UnownedStringSlice string);
+        // Helper update function for Slang::String
+        void update(MD5Context* ctx, String str);
+        // Helper update function for Checksums
+        void update(MD5Context* ctx, const slang::Digest& checksum);
+
+        void finalize(MD5Context* ctx, slang::Digest* result);
+
+    private:
+        static const void* body(MD5Context* ctx, const void* data, SlangInt size);
+        void update(MD5Context* ctx, const void* data, SlangInt size);
+    };
+}
+ 
+#endif