Add a memory-mappable binary serialization format (#7222)

The files `slang-fossil.{h,cpp}` define a new serialization format that is designed to support data being memory-mapped in and then traversed as-is.
The `docs/design/serialization.md` document was updated with details on this new format.

The `slang-serialize-fossil.{h,cpp}` files define implementations of the recently introduced `ISerializerImpl` interface for reading/writing this new binary format.
The overall structure of these implementations is heavily based on the existing RIFF implementation from `slang-serialize-riff.{h,cpp}`.

Switching the AST serialization over to use this format required almost no changes to `slang-serialize-ast.cpp`.
The new format is more space-efficient than the RIFF-based format in memory (by factor of over 2x), but is actually *worse* than the RIFF-based format in terms of how it affects the size of `slang.dll`, because the new format is seemingly less amenable to LZ4 compression.

A few pieces of utility code were added or moved as part of this work:

* The `core/slang-internally-linked-list.*` implementation is just a type that was used as part of `core/slang-riff.*`, but that wasn't really RIFF-specific.

* The `core/slang-blob-builder.*` files implement a low-level utility for building a binary format in memory out of "chunks". The overall structure of this type is based on the RIFF-specific builder implementation, but has been generalized so that it should apply to other kinds of binary serialization.

* The `core/slang-relative-ptr.h` file implements a simple relative pointer type, which is currently only used by the `slang-fossil.h` format.

If there are concerns about adopting the new format immediately for the AST, this change could be modified to introduce all the new code, but leave the AST serialization using the previous RIFF-based format.

1 files changed, 187 insertions, 0 deletions

diff --git a/source/slang/slang-fossil.cpp b/source/slang/slang-fossil.cpp
new file mode 100644
index 000000000..e3b2e2c9d
--- /dev/null
+++ b/source/slang/slang-fossil.cpp
@@ -0,0 +1,187 @@
+// slang-fossil.cpp
+#include "slang-fossil.h"
+
+namespace Slang
+{
+namespace Fossil
+{
+
+const char Fossil::Header::kMagic[16] = {
+    '\xAB', // byte 0
+    'f',    // byte 1
+    'o',    // byte 2
+    's',    // byte 3
+    's',    // byte 4
+    'i',    // byte 5
+    'l',    // byte 6
+    ' ',    // byte 7
+    '1',    // byte 8
+    '0',    // byte 9
+    '0',    // byte 10
+    '\xBB', // byte 11
+    '\r',   // byte 12
+    '\n',   // byte 13
+    '\x1A', // byte 14
+    '\n'    // byte 15
+};
+
+FossilizedValRef getRootValue(ISlangBlob* blob)
+{
+    return getRootValue(blob->getBufferPointer(), blob->getBufferSize());
+}
+
+FossilizedValRef getRootValue(void const* data, Size size)
+{
+    if (!data)
+    {
+        SLANG_UNEXPECTED("bad format for fossil");
+    }
+
+    // There must be enough data to at least hold the header.
+    //
+    // (In practice there would need to be more data than
+    // just the header, but checking this invariant is a start).
+    //
+    if (size < sizeof(Fossil::Header))
+    {
+        SLANG_UNEXPECTED("bad format for fossil");
+    }
+
+    // Once we've checked that there's enough data, we can read
+    // the contents of the header.
+    //
+    auto header = reinterpret_cast<Fossil::Header const*>(data);
+
+    // The "magic" bytes at the start of the header must be
+    // what we expect (which is the contents of `Fossil::Header::kMagic`).
+    //
+    if (memcmp(header->magic, Fossil::Header::kMagic, sizeof(Fossil::Header::kMagic)) != 0)
+    {
+        SLANG_UNEXPECTED("bad format for fossil");
+    }
+
+    auto reportedSize = header->totalSizeIncludingHeader;
+    if (reportedSize > size)
+    {
+        SLANG_UNEXPECTED("bad format for fossil");
+    }
+
+    auto rootValueVariant = header->rootValue.get();
+    if (!rootValueVariant)
+    {
+        SLANG_UNEXPECTED("bad format for fossil");
+    }
+
+    return FossilizedValRef(
+        rootValueVariant->getContentData(),
+        rootValueVariant->getContentLayout());
+}
+
+} // namespace Fossil
+
+Size FossilizedStringObj::getSize() const
+{
+    auto sizePtr = (FossilUInt*)this - 1;
+    return Size(*sizePtr);
+}
+
+UnownedTerminatedStringSlice FossilizedStringObj::getValue() const
+{
+    auto size = getSize();
+    return UnownedTerminatedStringSlice((char*)this, size);
+}
+
+Count FossilizedContainerObj::getElementCount() const
+{
+    auto countPtr = (FossilUInt*)this - 1;
+    return Size(*countPtr);
+}
+
+FossilizedValLayout* FossilizedVariantObj::getContentLayout() const
+{
+    auto layoutPtrPtr = (FossilizedPtr<FossilizedValLayout>*)this - 1;
+    return (*layoutPtrPtr).get();
+}
+
+FossilizedValRef getPtrTarget(FossilizedPtrValRef ptrRef)
+{
+    auto ptrLayout = ptrRef.getLayout();
+    auto ptrPtr = ptrRef.getData();
+    return FossilizedValRef(ptrPtr->getTargetData(), ptrLayout->elementLayout);
+}
+
+bool hasValue(FossilizedOptionalObjRef optionalRef)
+{
+    return optionalRef.getData() != nullptr;
+}
+
+FossilizedValRef getValue(FossilizedOptionalObjRef optionalRef)
+{
+    auto optionalLayout = optionalRef.getLayout();
+    auto valuePtr = optionalRef.getData();
+    return FossilizedValRef(valuePtr, optionalLayout->elementLayout);
+}
+
+Count getElementCount(FossilizedContainerObjRef containerRef)
+{
+    if (!containerRef)
+        return 0;
+
+    auto containerPtr = containerRef.getData();
+    return containerPtr->getElementCount();
+}
+
+FossilizedValRef getElement(FossilizedContainerObjRef containerRef, Index index)
+{
+    SLANG_ASSERT(index >= 0);
+    SLANG_ASSERT(index < getElementCount(containerRef));
+
+    auto containerLayout = containerRef.getLayout();
+    auto elementLayout = containerLayout->elementLayout.get();
+    auto elementStride = containerLayout->elementStride;
+
+    auto elementsPtr = (Byte*)containerRef.getData();
+    auto elementPtr = (FossilizedVal*)(elementsPtr + elementStride * index);
+    return FossilizedValRef(elementPtr, elementLayout);
+}
+
+Count getFieldCount(FossilizedRecordValRef recordRef)
+{
+    auto recordLayout = recordRef.getLayout();
+    return recordLayout->fieldCount;
+}
+
+FossilizedRecordElementLayout* FossilizedRecordLayout::getField(Index index) const
+{
+    SLANG_ASSERT(index >= 0);
+    SLANG_ASSERT(index < fieldCount);
+
+    auto fieldsPtr = (FossilizedRecordElementLayout*)(this + 1);
+    return fieldsPtr + index;
+}
+
+
+FossilizedValRef getField(FossilizedRecordValRef recordRef, Index index)
+{
+    SLANG_ASSERT(index >= 0);
+    SLANG_ASSERT(index < getFieldCount(recordRef));
+
+    auto recordLayout = recordRef.getLayout();
+    auto field = recordLayout->getField(index);
+
+    auto fieldsPtr = (Byte*)recordRef.getData();
+    auto fieldPtr = (FossilizedVal*)(fieldsPtr + field->offset);
+    return FossilizedValRef(fieldPtr, field->layout);
+}
+
+FossilizedValRef getVariantContent(FossilizedVariantObjRef variantRef)
+{
+    return getVariantContent(variantRef.getData());
+}
+
+FossilizedValRef getVariantContent(FossilizedVariantObj* variantPtr)
+{
+    return FossilizedValRef(variantPtr->getContentData(), variantPtr->getContentLayout());
+}
+
+} // namespace Slang