5 files changed, 416 insertions, 7 deletions

diff --git a/source/core/slang-crypto.cpp b/source/core/slang-crypto.cpp
index ece7b01e9..dfe246c7c 100644
--- a/source/core/slang-crypto.cpp
+++ b/source/core/slang-crypto.cpp
@@ -82,15 +82,19 @@ void MD5::update(const void* data, SlangInt size)
 
     saved_lo = m_lo;
     if ((m_lo = (saved_lo + size) & 0x1fffffff) < saved_lo)
+    {
         m_hi++;
+    }
     m_hi += (uint32_t)size >> 29;
 
     used = saved_lo & 0x3f;
 
-    if (used) {
+    if (used)
+    {
         available = 64 - used;
 
-        if (size < available) {
+        if (size < available)
+        {
             ::memcpy(&m_buffer[used], data, size);
             return;
         }
@@ -101,7 +105,8 @@ void MD5::update(const void* data, SlangInt size)
         processBlock(m_buffer, 64);
     }
 
-    if (size >= 64) {
+    if (size >= 64)
+    {
         data = processBlock(data, size & ~(SlangInt)0x3f);
         size &= 0x3f;
     }
@@ -119,7 +124,8 @@ MD5::Digest MD5::finalize()
 
     available = 64 - used;
 
-    if (available < 8) {
+    if (available < 8)
+    {
         ::memset(&m_buffer[used], 0, available);
         processBlock(m_buffer, 64);
         used = 0;
diff --git a/source/core/slang-io.cpp b/source/core/slang-io.cpp
index d8ef48ee3..f2be44f4f 100644
--- a/source/core/slang-io.cpp
+++ b/source/core/slang-io.cpp
@@ -682,7 +682,7 @@ namespace Slang
         WIN32_FIND_DATAW fileData;
 
         HANDLE findHandle = FindFirstFileW(searchPath.toWString(), &fileData);
-        if (!findHandle)
+        if (findHandle == INVALID_HANDLE_VALUE)
         {
             return SLANG_E_NOT_FOUND;
         }
diff --git a/source/core/slang-io.h b/source/core/slang-io.h
index fc5cbfa9d..e766359ff 100644
--- a/source/core/slang-io.h
+++ b/source/core/slang-io.h
@@ -266,9 +266,9 @@ namespace Slang
 
     private:
         LockFile(const LockFile&) = delete;
-        LockFile(LockFile&) = delete;
+        LockFile(LockFile&&) = delete;
         LockFile& operator=(const LockFile&) = delete;
-        LockFile& operator=(const LockFile&&) = delete;
+        LockFile& operator=(LockFile&&) = delete;
 
 #if SLANG_WINDOWS_FAMILY
         void* m_fileHandle;
@@ -278,6 +278,29 @@ namespace Slang
         bool m_isOpen;
     };
 
+    class LockFileGuard
+    {
+    public:
+        LockFileGuard(LockFile& lockFile, LockFile::LockType lockType = LockFile::LockType::Exclusive)
+            : m_lockFile(lockFile)
+        {
+            m_lockFile.lock(lockType);
+        }
+
+        ~LockFileGuard()
+        {
+            m_lockFile.unlock();
+        }
+
+    private:
+        LockFileGuard(const LockFileGuard&) = delete;
+        LockFileGuard(LockFileGuard&&) = delete;
+        LockFileGuard& operator=(const LockFileGuard&) = delete;
+        LockFileGuard& operator=(LockFileGuard&&) = delete;
+
+        LockFile& m_lockFile;
+    };
+
 }
 
 #endif
diff --git a/source/core/slang-persistent-cache.cpp b/source/core/slang-persistent-cache.cpp
new file mode 100644
index 000000000..2b4113e16
--- /dev/null
+++ b/source/core/slang-persistent-cache.cpp
@@ -0,0 +1,289 @@
+#include "slang-persistent-cache.h"
+
+#include "../core/slang-io.h"
+#include "../core/slang-stream.h"
+#include "../core/slang-string-util.h"
+#include "../core/slang-blob.h"
+
+namespace Slang
+{
+
+PersistentCache::PersistentCache(const Desc& desc)
+{
+    m_cacheDirectory = Path::simplify(desc.directory);
+    Path::createDirectory(m_cacheDirectory);
+
+    m_lockFileName = Path::simplify(m_cacheDirectory + "/lock");
+    m_indexFileName = Path::simplify(m_cacheDirectory + "/index");
+
+    m_lockFile.open(m_lockFileName);
+
+    m_maxEntryCount = desc.maxEntryCount;
+
+    resetStats();
+
+    initialize();
+}
+
+PersistentCache::~PersistentCache()
+{
+}
+
+SlangResult PersistentCache::clear()
+{
+    if (!m_lockFile.isOpen())
+    {
+        return SLANG_E_CANNOT_OPEN;
+    }
+
+    // Acquire the exclusive lock.
+    std::lock_guard<std::mutex> mutexLock(m_mutex);
+    LockFileGuard fileLock(m_lockFile);
+
+    struct Visitor : Path::Visitor
+    {
+        const String& directory;
+        const String& lockFileName;
+
+        Visitor(const String& directory, const String& lockFileName)
+            : directory(directory)
+            , lockFileName(lockFileName)
+        {}
+
+        void accept(Path::Type type, const UnownedStringSlice& fileName) SLANG_OVERRIDE
+        {
+            String fullPath = Path::simplify(directory + "/" + fileName);;
+            if (type == Path::Type::File && lockFileName != fullPath)
+            {
+                Path::remove(fullPath);
+            }
+        }
+    };
+
+    Visitor visitor(m_cacheDirectory, m_lockFileName);
+    Path::find(m_cacheDirectory, nullptr, &visitor);
+
+    m_stats.entryCount = 0;
+
+    return SLANG_OK;
+}
+
+void PersistentCache::resetStats()
+{
+    m_stats.entryCount = 0;
+    m_stats.hitCount = 0;
+    m_stats.missCount = 0;
+}
+
+SlangResult PersistentCache::readEntry(const Key& key, ISlangBlob** outData)
+{
+    // Be pessimistic and assume we have a cache miss.
+    ++m_stats.missCount;
+
+    if (!m_lockFile.isOpen())
+    {
+        return SLANG_E_CANNOT_OPEN;
+    }
+
+    // Acquire the exclusive lock.
+    std::lock_guard<std::mutex> mutexLock(m_mutex);
+    LockFileGuard fileLock(m_lockFile);
+
+    // Return if index does not exist.
+    if (!File::exists(m_indexFileName))
+    {
+        return SLANG_E_NOT_FOUND;
+    }
+
+    // Read the cache index.
+    CacheIndex cacheIndex;
+    SLANG_RETURN_ON_FAIL(readIndex(m_indexFileName, cacheIndex));
+
+    // Increase the age of all entries in the cache.
+    for (auto& entry : cacheIndex)
+    {
+        ++entry.age;
+    }
+
+    // Find the entry.
+    Index entryIndex = cacheIndex.findFirstIndex([&key] (const CacheEntry& entry) { return entry.key == key; });
+    if (entryIndex == -1)
+    {
+        return SLANG_E_NOT_FOUND;
+    }
+
+    // Read the entry.
+    String entryFileName = getEntryFileName(key);
+    ScopedAllocation data;
+    SlangResult result = File::readAllBytes(entryFileName, data);
+    if (result == SLANG_OK)
+    {
+        --m_stats.missCount;
+        ++m_stats.hitCount;
+        cacheIndex[entryIndex].age = 0;
+        auto blob = RawBlob::moveCreate(data);
+        *outData = blob.detach();
+    }
+    else
+    {
+        cacheIndex.removeAt(entryIndex);
+    }
+
+    // Write the cache index.
+    SLANG_RETURN_ON_FAIL(writeIndex(m_indexFileName, cacheIndex));
+    m_stats.entryCount = (Count)cacheIndex.getCount();
+
+    return result;
+}
+
+SlangResult PersistentCache::writeEntry(const Key& key, ISlangBlob* data)
+{
+    SLANG_ASSERT(data);
+
+    if (!m_lockFile.isOpen())
+    {
+        return SLANG_E_CANNOT_OPEN;
+    }
+
+    // Acquire the exclusive lock.
+    std::lock_guard<std::mutex> mutexLock(m_mutex);
+    LockFileGuard fileLock(m_lockFile);
+
+    // Read the cache index.
+    // We ignore any errors when reading the index and just write a new one.
+    CacheIndex cacheIndex;
+    readIndex(m_indexFileName, cacheIndex);
+
+    // Increase the age of all entries in the cache and get the index of
+    // the oldest entry.
+    Index oldestEntryIndex = -1;
+    uint32_t oldestEntryAge = 0;
+    for (Index entryIndex = 0; entryIndex < cacheIndex.getCount(); ++entryIndex)
+    {
+        auto& entry = cacheIndex[entryIndex];
+        ++entry.age;
+        if (entry.age > oldestEntryAge)
+        {
+            oldestEntryIndex = entryIndex;
+            oldestEntryAge = entry.age;
+        }
+    }
+
+    // Write the cache entry.
+    String entryFileName = getEntryFileName(key);
+    SLANG_RETURN_ON_FAIL(File::writeAllBytes(entryFileName, data->getBufferPointer(), data->getBufferSize()));
+
+    // Update the index.
+    if (m_maxEntryCount > 0 && cacheIndex.getCount() >= m_maxEntryCount)
+    {
+        // Replace oldest entry.
+        SLANG_ASSERT(oldestEntryIndex >= 0);
+        File::remove(getEntryFileName(cacheIndex[oldestEntryIndex].key));
+        cacheIndex[oldestEntryIndex] = CacheEntry{ key, 0 };
+    }
+    else
+    {
+        // Add new entry.
+        cacheIndex.add(CacheEntry{ key, 0 });
+    }
+
+    // Write the cache index.
+    SlangResult result = writeIndex(m_indexFileName, cacheIndex);
+    if (result == SLANG_OK)
+    {
+        m_stats.entryCount = (Count)cacheIndex.getCount();
+    }
+    else
+    {
+        // If writing the index failed, remove the entry file to avoid growing the cache.
+        Path::remove(entryFileName);
+    }
+
+    return result;
+}
+
+SlangResult PersistentCache::initialize()
+{
+    if (!m_lockFile.isOpen())
+    {
+        return SLANG_E_CANNOT_OPEN;
+    }
+
+    // Acquire the exclusive lock.
+    std::lock_guard<std::mutex> mutexLock(m_mutex);
+    LockFileGuard fileLock(m_lockFile);
+
+    CacheIndex cacheIndex;
+    if (SLANG_SUCCEEDED(readIndex(m_indexFileName, cacheIndex)))
+    {
+        m_stats.entryCount = (Count)cacheIndex.getCount();
+    }
+
+    return SLANG_OK;
+}
+
+String PersistentCache::getEntryFileName(const Key& key)
+{
+    StringBuilder str;
+    str << m_cacheDirectory << "/" << key.toString();
+    return str;
+}
+
+struct CacheIndexHeader
+{
+    char magic[4];
+    uint32_t version;
+    uint32_t count;
+    uint32_t reserved;
+};
+
+static const char* kMagic = "SLS$";
+static const uint32_t kVersion = 1;
+
+SlangResult PersistentCache::readIndex(const String& fileName, CacheIndex& outIndex)
+{
+    FileStream fs;
+    SLANG_RETURN_ON_FAIL(fs.init(fileName, FileMode::Open));
+
+    // Get file size.
+    SLANG_RETURN_ON_FAIL(fs.seek(SeekOrigin::End, 0));
+    size_t fileSize = (size_t)fs.getPosition();
+    SLANG_RETURN_ON_FAIL(fs.seek(SeekOrigin::Start, 0));
+
+    CacheIndexHeader header;
+    SLANG_RETURN_ON_FAIL(fs.readExactly(&header, sizeof(header)));
+    if (::memcmp(header.magic, kMagic, 4) != 0 || header.version != kVersion)
+    {
+        return SLANG_E_INTERNAL_FAIL;
+    }
+
+    // Return if payload does not have the right size.
+    if (header.count * sizeof(CacheEntry) != fileSize - sizeof(header))
+    {
+        return SLANG_E_INTERNAL_FAIL;
+    }
+
+    outIndex.setCount(header.count);
+    SLANG_RETURN_ON_FAIL(fs.readExactly(outIndex.getBuffer(), header.count * sizeof(CacheEntry)));
+    
+    return SLANG_OK;
+}
+
+SlangResult PersistentCache::writeIndex(const String& fileName, const CacheIndex& index)
+{
+    FileStream fs;
+    SLANG_RETURN_ON_FAIL(fs.init(fileName, FileMode::Create));
+
+    CacheIndexHeader header;
+    ::memcpy(header.magic, kMagic, 4);
+    header.version = kVersion;
+    header.count = (uint32_t)index.getCount();
+    header.reserved = 0;
+    SLANG_RETURN_ON_FAIL(fs.write(&header, sizeof(header)));
+
+    SLANG_RETURN_ON_FAIL(fs.write(index.getBuffer(), index.getCount() * sizeof(CacheEntry)));
+
+    return SLANG_OK;
+}
+
+}
diff --git a/source/core/slang-persistent-cache.h b/source/core/slang-persistent-cache.h
new file mode 100644
index 000000000..db5ef0a2e
--- /dev/null
+++ b/source/core/slang-persistent-cache.h
@@ -0,0 +1,91 @@
+#pragma once
+#include "../../slang.h"
+#include "../core/slang-crypto.h"
+#include "../core/slang-io.h"
+#include "../core/slang-string.h"
+
+#include <mutex>
+
+namespace Slang
+{
+
+/// Implements a simple persistent cache on the filesystem for storing key/value pairs.
+/// Keys are SHA1 hashes and values are arbitrary blobs of data.
+/// The cache is save for concurrent access from multiple threads/processes by using
+/// a lock file within the cache directory. Furthermore, the cache implements a LRU
+/// eviction policy.
+class PersistentCache : public RefObject
+{
+public:
+    struct Desc
+    {   
+        // The root directory for the cache.
+        const char* directory = nullptr;
+        // The maximum number of entries stored in the cache. By default, there is no limit.
+        Count maxEntryCount = 0;
+    };
+
+    struct Stats
+    {
+        // Number of cache hits since last resetting the stats.
+        Count hitCount;
+        // Number of cache misses since last resetting the stats.
+        Count missCount;
+        // Current number of entries in the cache.
+        Count entryCount;
+    };
+
+    using Key = SHA1::Digest;
+
+    PersistentCache(const Desc& desc);
+    ~PersistentCache();
+
+    /// Clear the contents of the cache by removing the cache index and all entry files.
+    SlangResult clear();
+
+    const Stats& getStats() const { return m_stats; }
+    void resetStats();
+
+    /// Read an entry from the cache.
+    /// Returns SLANG_OK if successful, SLANG_E_NOT_FOUND if the entry is not in the cache.
+    SlangResult readEntry(const Key& key, ISlangBlob** outData);
+    
+    /// Write an entry to the cache.
+    /// Returns SLANG_OK if successful.
+    SlangResult writeEntry(const Key& key, ISlangBlob* data);
+
+private:
+    struct CacheEntry
+    {
+        Key key;
+        uint32_t age;
+    };
+
+    using CacheIndex = List<CacheEntry>;
+
+    SlangResult initialize();
+
+    String getEntryFileName(const Key& key);
+
+    SlangResult readIndex(const String& fileName, CacheIndex& outIndex);
+    SlangResult writeIndex(const String& fileName, const CacheIndex& index);
+
+    String m_cacheDirectory;
+    String m_lockFileName;
+    String m_indexFileName;
+
+    // For exclusive locking we need both a mutex (acquired first)
+    // followed by a a file lock. The mutex is needed because on Linux
+    // the file lock is only locking between processes, not threads.
+    std::mutex m_mutex;
+    Slang::LockFile m_lockFile;
+
+    Count m_maxEntryCount;
+
+    Stats m_stats;
+
+    // Used for unit tests.
+    friend struct PersistentCacheTest;
+};
+
+}