Version information on repro binary format. (#1090)

* * Added semantic versioning and hashing test on repro data.
* Added RiffSemanticVersion type

* Fix linux build warning.

4 files changed, 166 insertions, 29 deletions

diff --git a/source/core/slang-riff.cpp b/source/core/slang-riff.cpp
index ce8b7129b..0b0519b48 100644
--- a/source/core/slang-riff.cpp
+++ b/source/core/slang-riff.cpp
@@ -42,20 +42,28 @@ namespace Slang
 }
 
 
-/* static */SlangResult RiffUtil::writeData(uint32_t riffType, const void* data, size_t size, Stream* out)
+/* static */SlangResult RiffUtil::writeData(const RiffChunk* header, size_t headerSize, const void* payload, size_t payloadSize, Stream* out)
 {
-    SLANG_ASSERT(uint64_t(size) <= uint64_t(0xfffffffff));
+    SLANG_ASSERT(uint64_t(payloadSize) <= uint64_t(0xfffffffff));
+    SLANG_ASSERT(headerSize >= sizeof(RiffChunk));
+    SLANG_ASSERT((headerSize & 3) == 0);
 
     // TODO(JS): Could handle endianness here
+
     RiffChunk chunk;
-    chunk.m_type = riffType;
-    chunk.m_size = uint32_t(size);
+    chunk.m_type = header->m_type;
+    chunk.m_size = uint32_t(headerSize - sizeof(RiffChunk) + payloadSize);
 
     try
     {
-        out->write(&chunk, sizeof(chunk));
-        out->write(data, size);
-        size_t remaining = size & 3;
+        // The chunk
+        out->write(&chunk, sizeof(RiffChunk));
+        // The rest of the header
+        out->write(header + 1, headerSize - sizeof(RiffChunk));
+
+        out->write(payload, payloadSize);
+        size_t remaining = payloadSize & 3;
+
         if (remaining)
         {
             uint8_t end[4] = { 0, 0, 0, 0};
@@ -70,19 +78,30 @@ namespace Slang
     return SLANG_OK;
 }
 
-
-/* static */SlangResult RiffUtil::readData(Stream* stream, RiffChunk& outChunk, List<uint8_t>& data)
+/* static */SlangResult RiffUtil::readData(Stream* stream, RiffChunk* outHeader, size_t headerSize, List<uint8_t>& data)
 {
-    SLANG_RETURN_ON_FAIL(readChunk(stream, outChunk));
+    RiffChunk chunk;
+    SLANG_RETURN_ON_FAIL(readChunk(stream, chunk));
+    if (chunk.m_size < headerSize)
+    {
+        return SLANG_FAIL;
+    }
 
-    data.setCount(outChunk.m_size);
+    *outHeader = chunk;
 
     try
     {
-        stream->read(data.getBuffer(), outChunk.m_size);
+        // Read the header
+        stream->read(outHeader + 1, headerSize - sizeof(RiffChunk));
+
+        const size_t payloadSize = chunk.m_size - (headerSize - sizeof(RiffChunk));
+
+        data.setCount(payloadSize);
+
+        stream->read(data.getBuffer(), payloadSize);
 
         // Skip to the alignment
-        uint32_t remaining = outChunk.m_size & 3;
+        uint32_t remaining = payloadSize & 3;
         if (remaining)
         {
             stream->seek(SeekOrigin::Current, 4 - remaining);
diff --git a/source/core/slang-riff.h b/source/core/slang-riff.h
index f6854310b..f681d6d68 100644
--- a/source/core/slang-riff.h
+++ b/source/core/slang-riff.h
@@ -17,6 +17,59 @@ struct RiffFourCC
      static const uint32_t kRiff = SLANG_FOUR_CC('R', 'I', 'F', 'F');
 };
 
+// Follows semantic version rules
+// https://semver.org/
+// 
+// major.minor.patch
+// Patch versions indicate a change. 
+// Minor means a change that is backwards compatible with previous minor versions. A step in minor and/or major zeros patch.
+// Major means a non compatible change. A step in major, zeros minor and patch. 
+struct RiffSemanticVersion
+{
+    typedef RiffSemanticVersion ThisType;
+    typedef uint32_t RawType;
+
+        /// ==
+    bool operator==(const ThisType& rhs) const { return m_raw == rhs.m_raw; }
+    bool operator!=(const ThisType& rhs) const { return !(*this == rhs); }
+    
+        /// A patch change indices a different version but does not change the compatibility of the format
+    int getPatch() const { return m_raw & 0xff; }
+        /// A minor change implies a format change that is backwards compatible  
+    int getMinor() const { return (m_raw >> 8) & 0xff; }
+        /// A major change is binary incompatible by default
+    int getMajor() const { return (m_raw >> 16); }
+
+    static RawType makeRaw(int major, int minor, int patch)
+    {
+        SLANG_ASSERT((major | minor | patch) >= 0);
+        SLANG_ASSERT(major < 0x10000 && minor < 0x100 && patch < 0x100);
+        return (RawType(major) << 16) | (RawType(minor) << 8) | RawType(patch);
+    }
+
+    static RiffSemanticVersion makeFromRaw(RawType raw)
+    {
+        ThisType version;
+        version.m_raw = raw;
+        return version;
+    }
+
+    static RiffSemanticVersion make(int major, int minor, int patch) { return makeFromRaw(makeRaw(major, minor, patch)); }
+
+        /// True if the read version is compatible with the current version, based on semantic rules.
+    static bool areCompatible(const ThisType& currentVersion, const ThisType& readVersion)
+    {
+        const RawType currentRaw = currentVersion.m_raw;
+        const RawType readRaw = readVersion.m_raw;
+
+        // Must have same major version.
+        // For minor version, the read version must be less than or equal. 
+        return ((currentRaw & 0xffff0000) == (readRaw & 0xffff0000)) && ((currentRaw & 0xff00) >= (readRaw & 0xff00));
+    }
+
+    RawType m_raw;
+};
+
 struct RiffChunk
 {
     uint32_t m_type;            ///< The FourCC code that identifies this chunk
@@ -31,9 +84,8 @@ struct RiffUtil
 
     static SlangResult readChunk(Stream* stream, RiffChunk& outChunk);
 
-
-    static SlangResult writeData(uint32_t riffType, const void* data, size_t size, Stream* out);
-    static SlangResult readData(Stream* stream, RiffChunk& outChunk, List<uint8_t>& data);
+    static SlangResult writeData(const RiffChunk* header, size_t headerSize, const void* payload, size_t payloadSize, Stream* out);
+    static SlangResult readData(Stream* stream, RiffChunk* outHeader, size_t headerSize, List<uint8_t>& data);
 };
 
 }
diff --git a/source/slang/slang-state-serialize.cpp b/source/slang/slang-state-serialize.cpp
index cbcce6c3a..13dd8b132 100644
--- a/source/slang/slang-state-serialize.cpp
+++ b/source/slang/slang-state-serialize.cpp
@@ -9,6 +9,53 @@
 
 namespace Slang {
 
+/* static */const RiffSemanticVersion StateSerializeUtil::g_semanticVersion =
+    RiffSemanticVersion::make(StateSerializeUtil::kMajorVersion, StateSerializeUtil::kMinorVersion, StateSerializeUtil::kPatchVersion);
+
+// A function to calculate the hash related in list in part to how the types used are sized. Can catch crude breaking binary differences.
+static uint32_t _calcTypeHash()
+{
+    typedef StateSerializeUtil Util;
+
+    const size_t sizes[] =
+    {
+        sizeof(Util::FileState),
+        sizeof(Util::PathInfoState),
+            sizeof(Util::PathInfoState::CompressedResult),
+            sizeof(SlangPathType),
+        sizeof(Util::PathAndPathInfo),
+        sizeof(Util::TargetRequestState),
+            sizeof(Profile),
+            sizeof(CodeGenTarget),
+            sizeof(SlangTargetFlags),
+            sizeof(FloatingPointMode),
+        sizeof(Util::StringPair),
+        sizeof(Util::SourceFileState),
+            sizeof(PathInfo::Type),
+        sizeof(Util::TranslationUnitRequestState),
+            sizeof(SourceLanguage),
+        sizeof(Util::EntryPointState),
+            sizeof(Profile),
+        sizeof(Util::RequestState),
+            sizeof(SlangCompileFlags),
+            sizeof(bool),                       //< Unfortunately bools size can change across compilers/versions
+            sizeof(LineDirectiveMode),
+            sizeof(DebugInfoLevel),
+            sizeof(OptimizationLevel),
+            sizeof(ContainerFormat),
+            sizeof(PassThroughMode),
+            sizeof(SlangMatrixLayoutMode),
+    };
+
+    return uint32_t(GetHashCode((const char*)&sizes, sizeof(sizes)));
+}
+
+static uint32_t _getTypeHash()
+{
+    static uint32_t s_hash = _calcTypeHash();
+    return s_hash;
+}
+
 
 namespace { // anonymous
 
@@ -866,7 +913,13 @@ static void _loadDefines(const Relative32Array<StateSerializeUtil::StringPair>&
     RelativeContainer container;
     Safe32Ptr<RequestState> requestState;
     SLANG_RETURN_ON_FAIL(store(request, container, requestState));
-    return RiffUtil::writeData(kSlangStateFourCC, container.getData(), container.getDataCount(), stream);
+
+    Header header;
+    header.m_chunk.m_type = kSlangStateFourCC;
+    header.m_semanticVersion = g_semanticVersion;
+    header.m_typeHash = _getTypeHash();
+
+    return RiffUtil::writeData(&header.m_chunk, sizeof(header),container.getData(), container.getDataCount(), stream);
 }
 
 /* static */SlangResult StateSerializeUtil::saveState(EndToEndCompileRequest* request, const String& filename)
@@ -892,10 +945,20 @@ static void _loadDefines(const Relative32Array<StateSerializeUtil::StringPair>&
 
 /* static */ SlangResult StateSerializeUtil::loadState(Stream* stream, List<uint8_t>& buffer)
 {
-    RiffChunk chunk;
-    SLANG_RETURN_ON_FAIL(RiffUtil::readData(stream, chunk, buffer));
+    Header header;
+
+    SLANG_RETURN_ON_FAIL(RiffUtil::readData(stream, &header.m_chunk, sizeof(header), buffer));
+    if (header.m_chunk.m_type != kSlangStateFourCC)
+    {
+        return SLANG_FAIL;
+    }
+
+    if (!RiffSemanticVersion::areCompatible(g_semanticVersion, header.m_semanticVersion))
+    {
+        return SLANG_FAIL;
+    }
 
-    if (chunk.m_type != kSlangStateFourCC)
+    if (header.m_typeHash != _getTypeHash())
     {
         return SLANG_FAIL;
     }
diff --git a/source/slang/slang-state-serialize.h b/source/slang/slang-state-serialize.h
index dbd1e3992..42edd4080 100644
--- a/source/slang/slang-state-serialize.h
+++ b/source/slang/slang-state-serialize.h
@@ -16,12 +16,21 @@ namespace Slang {
 
 struct StateSerializeUtil
 {
+    enum
+    {
+        kMajorVersion = 0,
+        kMinorVersion = 0,
+        kPatchVersion = 0,
+    };
+
     static const uint32_t kSlangStateFourCC = SLANG_FOUR_CC('S', 'L', 'S', 'T');             ///< Holds all the slang specific chunks
-    
+    static const RiffSemanticVersion g_semanticVersion;
+
     struct Header
     {
-        RiffChunk m_chunk;
-        uint32_t m_compressionType;         ///< Holds the compression type used (if used at all)
+        RiffChunk m_chunk;                              ///< The chunk 
+        RiffSemanticVersion m_semanticVersion;          ///< The semantic version
+        uint32_t m_typeHash;                            ///< A hash based on the binary representation. If doesn't match then not binary compatible (extra check over semantic versioning)
     };
 
     struct FileState
@@ -73,12 +82,6 @@ struct StateSerializeUtil
         Relative32Array<OutputState> outputStates;
     };
 
-    struct FileReference
-    {
-        Relative32Ptr<RelativeString> name;
-        Relative32Ptr<FileState> file;
-    };
-
     struct StringPair
     {
         Relative32Ptr<RelativeString> first;