Fixes related to AST serialization change (#6953)

* Fixes related to AST serialization change

There are two fixes included here.

The smaller fix is in `slang-ast-decl.h`, where the `CallableDecl::primaryDecl` and `::nextDecl` fields need to be serialized to make sure that we can properly deserialize a module that contains any function redeclarations.

The larger fix is that the `Encoder` and `Decoder` types used to serialize out the AST nodes in a JSON-like hierarchy were being very strict about matching of integer types, which causes problems in any case where serialization code might use a type that is 32-bit on some targets and 64-bit on others, if serialized modules are ever created on one of those targets and consumed on the other (which happens for the core module for some of our targets).

The fix that this change implements is to make the serialization logic there more forgiving, and thus more robust:

* In the writing/encoding direction, the logic now looks at the actual value being encoded to decide whether to write it as a 32- or 64-bit value.

* In the reading/decoding direction, the logic handles the presence of *any* of the FOURCCs that are used to encode integers, for whatever type is being read. As a small bit of safety, there is a dynamic check made for cases where a value would be read with a different sign than it was written with.

The actual logic in `slang-serialize-ast.cpp` is largely unchanged (it continues to use pointer-sized integers in certain cases), but it should not cause problems because it bottlenecks through the `Encoder`/`Decoder` methods that were changed. The only fix made in the AST serialization itself is to account for all of the FOURCCs that can represent integers when peeking at the input to decide whether a `DeclBase` is represented as an indirection to a `Decl`, or is serialized inline (as a `DeclGroup`).

* format code

---------

Co-authored-by: slangbot <186143334+slangbot@users.noreply.github.com>

3 files changed, 121 insertions, 18 deletions

diff --git a/source/slang/slang-ast-decl.h b/source/slang/slang-ast-decl.h
index 5b0c883b0..3201a3e1d 100644
--- a/source/slang/slang-ast-decl.h
+++ b/source/slang/slang-ast-decl.h
@@ -394,11 +394,11 @@ class CallableDecl : public ContainerDecl
 
     // The "primary" declaration of the function, which will
     // be used whenever we need to unique things.
-    CallableDecl* primaryDecl = nullptr;
+    FIDDLE() CallableDecl* primaryDecl = nullptr;
 
     // The next declaration of the "same" function (that is,
     // with the same `primaryDecl`).
-    CallableDecl* nextDecl = nullptr;
+    FIDDLE() CallableDecl* nextDecl = nullptr;
 };
 
 // Base class for callable things that may also have a body that is evaluated to produce their
diff --git a/source/slang/slang-serialize-ast.cpp b/source/slang/slang-serialize-ast.cpp
index aad3bcc57..b91fac89a 100644
--- a/source/slang/slang-serialize-ast.cpp
+++ b/source/slang/slang-serialize-ast.cpp
@@ -1135,14 +1135,27 @@ private:
     template<typename T>
     void decodePtr(T*& node, Decoder& decoder, DeclBase*)
     {
-        if (decoder.getTag() == SerialBinary::kInt64FourCC)
-        {
-            DeclID id = decodeDeclID(decoder);
-            node = static_cast<T*>(getDeclByID(id));
-        }
-        else
+        // This case is a bit of a hack. We need
+        // to identify whether we are looking at
+        // an indirection to a `Decl` (which would
+        // be serialized as an integer `DeclID`),
+        // or something else derived from `DeclBase`.
+        //
+        switch (decoder.getTag())
         {
+        default:
             decodeASTNode(node, decoder);
+            break;
+
+        case SerialBinary::kInt32FourCC:
+        case SerialBinary::kInt64FourCC:
+        case SerialBinary::kUInt32FourCC:
+        case SerialBinary::kUInt64FourCC:
+            {
+                DeclID id = decodeDeclID(decoder);
+                node = static_cast<T*>(getDeclByID(id));
+            }
+            break;
         }
     }
 
diff --git a/source/slang/slang-serialize.h b/source/slang/slang-serialize.h
index 20916f735..4de63471e 100644
--- a/source/slang/slang-serialize.h
+++ b/source/slang/slang-serialize.h
@@ -90,13 +90,38 @@ public:
         encodeData(value ? SerialBinary::kTrueFourCC : SerialBinary::kFalseFourCC, nullptr, 0);
     }
 
-    void encode(Int32 value) { encodeData(SerialBinary::kInt32FourCC, &value, sizeof(value)); }
+    void encodeInt(Int64 value)
+    {
+        if (Int32(value) == value)
+        {
+            auto v = Int32(value);
+            encodeData(SerialBinary::kInt32FourCC, &v, sizeof(v));
+        }
+        else
+        {
+            encodeData(SerialBinary::kInt64FourCC, &value, sizeof(value));
+        }
+    }
 
-    void encode(UInt32 value) { encodeData(SerialBinary::kUInt32FourCC, &value, sizeof(value)); }
 
-    void encode(Int64 value) { encodeData(SerialBinary::kInt64FourCC, &value, sizeof(value)); }
+    void encodeUInt(UInt64 value)
+    {
+        if (UInt32(value) == value)
+        {
+            auto v = UInt32(value);
+            encodeData(SerialBinary::kUInt32FourCC, &v, sizeof(v));
+        }
+        else
+        {
+            encodeData(SerialBinary::kUInt64FourCC, &value, sizeof(value));
+        }
+    }
+
+    void encode(Int32 value) { encodeInt(value); }
+    void encode(Int64 value) { encodeInt(value); }
 
-    void encode(UInt64 value) { encodeData(SerialBinary::kUInt64FourCC, &value, sizeof(value)); }
+    void encode(UInt32 value) { encodeUInt(value); }
+    void encode(UInt64 value) { encodeUInt(value); }
 
     void encode(float value) { encodeData(SerialBinary::kFloat32FourCC, &value, sizeof(value)); }
 
@@ -265,18 +290,83 @@ public:
         return value;
     }
 
-    Int64 decodeInt64() { return _decodeSimpleValue<Int64>(SerialBinary::kInt64FourCC); }
+    Int64 decodeInt()
+    {
+        switch (getTag())
+        {
+        case SerialBinary::kInt64FourCC:
+            return _decodeSimpleValue<Int64>(getTag());
+        case SerialBinary::kInt32FourCC:
+            return _decodeSimpleValue<Int32>(getTag());
 
-    UInt64 decodeUInt64() { return _decodeSimpleValue<UInt64>(SerialBinary::kUInt64FourCC); }
+        case SerialBinary::kUInt32FourCC:
+            return _decodeSimpleValue<UInt32>(getTag());
 
-    Int32 decodeInt32() { return _decodeSimpleValue<Int32>(SerialBinary::kInt32FourCC); }
+        case SerialBinary::kUInt64FourCC:
+            {
+                auto uintValue = _decodeSimpleValue<UInt64>(getTag());
+                if (Int64(uintValue) < 0)
+                {
+                    SLANG_UNEXPECTED("signed/unsigned mismatch in RIFF");
+                }
+                return Int64(uintValue);
+            }
 
-    UInt32 decodeUInt32() { return _decodeSimpleValue<UInt32>(SerialBinary::kUInt32FourCC); }
+        default:
+            SLANG_UNEXPECTED("invalid format in RIFF");
+            UNREACHABLE_RETURN(0);
+        }
+    }
+
+    UInt64 decodeUInt()
+    {
+        switch (getTag())
+        {
+        case SerialBinary::kUInt64FourCC:
+            return _decodeSimpleValue<UInt64>(getTag());
+        case SerialBinary::kUInt32FourCC:
+            return _decodeSimpleValue<UInt32>(getTag());
+
+        case SerialBinary::kInt32FourCC:
+        case SerialBinary::kInt64FourCC:
+            {
+                auto intValue = decodeInt();
+                if (intValue < 0)
+                {
+                    SLANG_UNEXPECTED("signed/unsigned mismatch in RIFF");
+                }
+                return UInt64(intValue);
+            }
+
+        default:
+            SLANG_UNEXPECTED("invalid format in RIFF");
+            UNREACHABLE_RETURN(0);
+        }
+    }
+
+    double decodeFloat()
+    {
+        switch (getTag())
+        {
+        case SerialBinary::kFloat32FourCC:
+            return _decodeSimpleValue<float>(getTag());
+        case SerialBinary::kFloat64FourCC:
+            return _decodeSimpleValue<double>(getTag());
+
+        default:
+            SLANG_UNEXPECTED("invalid format in RIFF");
+            UNREACHABLE_RETURN(0);
+        }
+    }
 
-    float decodeFloat32() { return _decodeSimpleValue<float>(SerialBinary::kFloat32FourCC); }
+    Int32 decodeInt32() { return Int32(decodeInt()); }
+    Int64 decodeInt64() { return decodeInt(); }
 
-    double decodeFloat64() { return _decodeSimpleValue<double>(SerialBinary::kFloat64FourCC); }
+    UInt32 decodeUInt32() { return UInt32(decodeUInt()); }
+    UInt64 decodeUInt64() { return decodeUInt(); }
 
+    float decodeFloat32() { return float(decodeFloat()); }
+    double decodeFloat64() { return decodeFloat(); }
 
     FourCC getTag() { return _chunk ? _chunk->m_fourCC : 0; }