FileStream-based implementation for updating cache index file (#2485)

* Draft FileStream-based implementation for updating cache file

* File streams fully integrated into shader cache code paths; Tests will not run unless file system is on disk as file streams do not play nicely with in-memory

* Brought old code back as fallback path, but tests need to ensure previous is freed first

* Testing structure updated, beginning cleanup work

* All tests working

* Cleanup changes

* Removed an extra tab at the end of a line

* Cleanup change

* Undo externals change

* Removed redundant logic for OS vs memory file system handling of the shader cache; Removed extra helper function left over from old cache implementation

* Reverted performance change to generate contents hashes when modules are being loaded as this code path is not always followed; Contents hashing now uses a combination of hashing and checking the last modified time for all file dependencies, only reading in and hashing the contents of all files if the last modified hash does not match

* Added handling to Module::updateContentsBasedHash for file dependencies which are not from a physical source file on disk; Added test for above

Co-authored-by: Lucy Chen <lucchen@nvidia.com>
Co-authored-by: Yong He <yonghe@outlook.com>

4 files changed, 53 insertions, 23 deletions

diff --git a/source/slang/slang-compiler.h b/source/slang/slang-compiler.h
index 14ac63531..07e611355 100755
--- a/source/slang/slang-compiler.h
+++ b/source/slang/slang-compiler.h
@@ -1366,14 +1366,6 @@ namespace Slang
             ///
         void setIRModule(IRModule* irModule) { m_irModule = irModule; }
 
-        DigestBuilder& getContentsDigestBuilder() { return contentsBuilder; }
-
-            /// Set the contents digest for this module.
-        void setContentsDigest(slang::Digest digest) { contentsDigest = digest; }
-
-            /// Get the contents digest for this module.
-        slang::Digest getContentsDigest() { return contentsDigest; }
-
         Index getEntryPointCount() SLANG_OVERRIDE { return 0; }
         RefPtr<EntryPoint> getEntryPoint(Index index) SLANG_OVERRIDE { SLANG_UNUSED(index); return nullptr; }
         String getEntryPointMangledName(Index index) SLANG_OVERRIDE { SLANG_UNUSED(index); return String(); }
@@ -1483,7 +1475,7 @@ namespace Slang
         StringSlicePool m_mangledExportPool;
         List<NodeBase*> m_mangledExportSymbols;
 
-        DigestBuilder contentsBuilder;
+        slang::Digest lastModifiedDigest;
         slang::Digest contentsDigest;
     };
     typedef Module LoadedModule;
diff --git a/source/slang/slang-preprocessor.cpp b/source/slang/slang-preprocessor.cpp
index c977e44ab..fca8f5029 100644
--- a/source/slang/slang-preprocessor.cpp
+++ b/source/slang/slang-preprocessor.cpp
@@ -32,10 +32,9 @@ void PreprocessorHandler::handleEndOfTranslationUnit(Preprocessor* preprocessor)
     SLANG_UNUSED(preprocessor);
 }
 
-void PreprocessorHandler::handleFileDependency(String const& path, ISlangBlob* contents)
+void PreprocessorHandler::handleFileDependency(String const& path)
 {
     SLANG_UNUSED(path);
-    SLANG_UNUSED(contents);
 }
 
 // In order to simplify the naming scheme, we will nest the implementaiton of the
@@ -2973,7 +2972,7 @@ static SlangResult readFile(
     //
     if( auto handler = context->m_preprocessor->handler )
     {
-        handler->handleFileDependency(path, *outBlob);
+        handler->handleFileDependency(path);
     }
 
     return SLANG_OK;
diff --git a/source/slang/slang-preprocessor.h b/source/slang/slang-preprocessor.h
index 90e03e964..4d7721d31 100644
--- a/source/slang/slang-preprocessor.h
+++ b/source/slang/slang-preprocessor.h
@@ -28,7 +28,7 @@ using preprocessor::Preprocessor;
 struct PreprocessorHandler
 {
     virtual void handleEndOfTranslationUnit(Preprocessor* preprocessor);
-    virtual void handleFileDependency(String const& path, ISlangBlob* contents = nullptr);
+    virtual void handleFileDependency(String const& path);
 };
 
     /// Description of a preprocessor options/dependencies
diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp
index 361f1b193..5add881d6 100644
--- a/source/slang/slang.cpp
+++ b/source/slang/slang.cpp
@@ -49,6 +49,8 @@
 
 #include "../../slang-tag-version.h"
 
+#include <sys/stat.h>
+
 // Used to print exception type names in internal-compiler-error messages
 #include <typeinfo>
 
@@ -1898,13 +1900,9 @@ protected:
     // by applications to decide when they need to "hot reload"
     // their shader code.
     //
-    void handleFileDependency(String const& path, ISlangBlob* sourceBlob) SLANG_OVERRIDE
+    void handleFileDependency(String const& path) SLANG_OVERRIDE
     {
         m_module->addFilePathDependency(path);
-        if (sourceBlob)
-        {
-            m_module->getContentsDigestBuilder().addToDigest(sourceBlob);
-        }
     }
 
     // The second task that this handler deals with is detecting
@@ -3046,10 +3044,6 @@ RefPtr<Module> Linkage::loadModule(
         return nullptr;
     }
 
-    auto builder = module->getContentsDigestBuilder();
-    builder.addToDigest(sourceBlob);
-    module->setContentsDigest(builder.finalize());
-
     return module;
 }
 
@@ -3269,7 +3263,52 @@ void Module::updateDependencyBasedHash(
 
 void Module::updateContentsBasedHash(DigestBuilder& builder)
 {
-    builder.addToDigest(getContentsDigest());
+    auto filePathDependencies = getFilePathDependencies();
+
+    DigestBuilder lastModifiedBuilder;
+    auto statFailed = false;
+    for (auto file : filePathDependencies)
+    {
+        struct stat fileStatus;
+        auto res = stat(file.getBuffer(), &fileStatus);
+        if (res != 0)
+        {
+            statFailed = true;
+            break;
+        }
+        lastModifiedBuilder.addToDigest(fileStatus.st_mtime);
+    }
+
+    slang::Digest temp = lastModifiedBuilder.finalize();
+    if (statFailed || temp != lastModifiedDigest)
+    {
+        // Either a stat() call failed, or changes were made to at least one of the file dependencies,
+        // so we will need to re-generate the contents digest and save the new digest.
+        DigestBuilder contentsBuilder;
+        for (auto file : filePathDependencies)
+        {
+            List<uint8_t> fileContents;
+            if (SLANG_FAILED(File::readAllBytes(file, fileContents)))
+            {
+                // Failure to read the file means this is a digest for the contents of a source
+                // file which does not live on disk.
+                contentsBuilder.addToDigest(DigestUtil::fromString(file.getUnownedSlice()));
+            }
+            else
+            {
+                contentsBuilder.addToDigest(fileContents);
+            }
+        }
+        contentsDigest = contentsBuilder.finalize();
+        if (!statFailed)
+        {
+            // If no stat() calls failed, then we have a valid last modified digest and should
+            // update the one we have saved.
+            lastModifiedDigest = temp;
+        }
+    }
+
+    builder.addToDigest(contentsDigest);
 }
 
 void Module::addModuleDependency(Module* module)