diff options
| author | Theresa Foley <10618364+tangent-vector@users.noreply.github.com> | 2025-05-30 10:00:38 -0700 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2025-05-30 17:00:38 +0000 |
| commit | ec7ab914f79978b8980c7797e20d3399604b1f86 (patch) | |
| tree | 2e6b01dc99fc0998e4f17a9aeaf22ef3d48817e0 /source | |
| parent | 14409bf1015af47691f09d2be6afb18cfb999aea (diff) | |
Add a memory-mappable binary serialization format (#7222)
The files `slang-fossil.{h,cpp}` define a new serialization format that is designed to support data being memory-mapped in and then traversed as-is.
The `docs/design/serialization.md` document was updated with details on this new format.
The `slang-serialize-fossil.{h,cpp}` files define implementations of the recently introduced `ISerializerImpl` interface for reading/writing this new binary format.
The overall structure of these implementations is heavily based on the existing RIFF implementation from `slang-serialize-riff.{h,cpp}`.
Switching the AST serialization over to use this format required almost no changes to `slang-serialize-ast.cpp`.
The new format is more space-efficient than the RIFF-based format in memory (by factor of over 2x), but is actually *worse* than the RIFF-based format in terms of how it affects the size of `slang.dll`, because the new format is seemingly less amenable to LZ4 compression.
A few pieces of utility code were added or moved as part of this work:
* The `core/slang-internally-linked-list.*` implementation is just a type that was used as part of `core/slang-riff.*`, but that wasn't really RIFF-specific.
* The `core/slang-blob-builder.*` files implement a low-level utility for building a binary format in memory out of "chunks". The overall structure of this type is based on the RIFF-specific builder implementation, but has been generalized so that it should apply to other kinds of binary serialization.
* The `core/slang-relative-ptr.h` file implements a simple relative pointer type, which is currently only used by the `slang-fossil.h` format.
If there are concerns about adopting the new format immediately for the AST, this change could be modified to introduce all the new code, but leave the AST serialization using the previous RIFF-based format.
Diffstat (limited to 'source')
| -rw-r--r-- | source/core/slang-blob-builder.cpp | 473 | ||||
| -rw-r--r-- | source/core/slang-blob-builder.h | 352 | ||||
| -rw-r--r-- | source/core/slang-internally-linked-list.cpp | 2 | ||||
| -rw-r--r-- | source/core/slang-internally-linked-list.h | 126 | ||||
| -rw-r--r-- | source/core/slang-memory-arena.h | 11 | ||||
| -rw-r--r-- | source/core/slang-relative-ptr.cpp | 2 | ||||
| -rw-r--r-- | source/core/slang-relative-ptr.h | 97 | ||||
| -rw-r--r-- | source/core/slang-riff.h | 68 | ||||
| -rw-r--r-- | source/slang/slang-fossil.cpp | 187 | ||||
| -rw-r--r-- | source/slang/slang-fossil.h | 488 | ||||
| -rw-r--r-- | source/slang/slang-serialize-ast.cpp | 48 | ||||
| -rw-r--r-- | source/slang/slang-serialize-fossil.cpp | 1636 | ||||
| -rw-r--r-- | source/slang/slang-serialize-fossil.h | 741 | ||||
| -rw-r--r-- | source/slang/slang-serialize-riff.cpp | 40 | ||||
| -rw-r--r-- | source/slang/slang-serialize-riff.h | 12 | ||||
| -rw-r--r-- | source/slang/slang-serialize.h | 43 |
16 files changed, 4231 insertions, 95 deletions
diff --git a/source/core/slang-blob-builder.cpp b/source/core/slang-blob-builder.cpp new file mode 100644 index 000000000..10dd4003e --- /dev/null +++ b/source/core/slang-blob-builder.cpp @@ -0,0 +1,473 @@ +// slang-blob-builder.cpp +#include "slang-blob-builder.h" + +namespace Slang +{ + +// +// BlobBuilder +// + +BlobBuilder::BlobBuilder() + : _arena(4096) +{ +} + +void BlobBuilder::writeTo(Stream* stream) +{ + // First we scan through all the chunks to set their + // absolute offsets, which enables us to compute the + // correct values for relative pointers when we write + // them out. + // + SLANG_MAYBE_UNUSED + Size sizeComputed = _calcSizeAndSetCachedChunkOffsets(); + + // Now we can scan through the chunks again and write + // the bytes of each of their shards. + // + SLANG_MAYBE_UNUSED + Size sizeWritten = _writeChunksTo(stream); + + SLANG_ASSERT(sizeComputed == sizeWritten); +} + +Size BlobBuilder::_calcSizeAndSetCachedChunkOffsets() +{ + Size totalSize = 0; + for (auto chunk : _chunks) + { + auto chunkPrefixSize = chunk->getPrefixSize(); + auto chunkContentSize = chunk->getContentSize(); + auto chunkAlignment = chunk->getAlignment(); + + // We add the size of the chunk prefix (if any) *before* + // aligning the current offset. Doing it this way + // means that sometimes the prefix can fit "for free" + // in space that would otherwise be padding. + // + // For example, if the current `totalSize` is 4, the + // `chunkAlignment` is 16, and the `chunkPrefixSize` is 8, + // then the sequence will be: + // + // * Add `totalSize += chunkPrefixSize`, resulting in a `totalSize` + // of 12. + // + // * Round the `totalSize` up to the `chunkAlignment`, to compute + // a `chunkOffset` of 16. + // + // The result is that the chunk's content starts at offset 16, + // and the prefix can occupy the 8 bytes before that (so the + // prefix is at offset 8). + // + // In the best case this approach can save a few bytes here or + // there when `chunkAlignment` is larger than the `chunkPrefixSize`, + // and in the worst case it does no harm. + + totalSize += chunkPrefixSize; + auto chunkOffset = roundUpToAlignment(totalSize, chunkAlignment); + + chunk->_setCachedOffset(chunkOffset); + + totalSize = chunkOffset + chunkContentSize; + } + return totalSize; +} + +Size BlobBuilder::_writeChunksTo(Stream* stream) +{ + Size totalSize = 0; + for (auto chunk : _chunks) + { + auto chunkPrefixSize = chunk->getPrefixSize(); + auto chunkContentSize = chunk->getContentSize(); + auto chunkOffset = chunk->_getCachedOffset(); + + SLANG_ASSERT( + chunkOffset == roundUpToAlignment(totalSize + chunkPrefixSize, chunk->getAlignment())); + + auto paddingSize = chunkOffset - totalSize; + + // The "padding" before the chunk's content also + // includes the space reserved for the chunk's prefix. + // + // We can thus subtract the prefix size from the number + // of pad bytes to write. + + SLANG_ASSERT(paddingSize >= chunkPrefixSize); + paddingSize -= chunkPrefixSize; + + while (paddingSize--) + { + Byte padding = 0; + stream->write(&padding, sizeof(padding)); + } + + // The `ChunkBuilder::_writeTo()` call will write the + // prefix *and* the chunk content, so it is appropriate + // to call it here even when the total number of bytes + // written to the stream is not equal to the `chunkOffset` + // (because in that case the total bytes written so far + // should be `chunkOffset - chunkPrefixSize`). + // + chunk->_writeTo(stream); + + totalSize = chunkOffset + chunkContentSize; + } + + return totalSize; +} + +void BlobBuilder::writeToBlob(ISlangBlob** outBlob) +{ + OwnedMemoryStream stream(FileAccess::Write); + writeTo(&stream); + + List<uint8_t> data; + stream.swapContents(data); + + *outBlob = ListBlob::moveCreate(data).detach(); +} + +ChunkBuilder* BlobBuilder::createUnparentedChunk() +{ + auto chunk = new (_arena) ChunkBuilder(this); + return chunk; +} + +void BlobBuilder::addChunk(ChunkBuilder* chunk) +{ + // TODO(tfoley): it would be good to have a way to assert + // that the chunk has not already been added. + + _chunks.add(chunk); +} + +ChunkBuilder* BlobBuilder::addChunk() +{ + auto chunk = new (_arena) ChunkBuilder(this); + _chunks.add(chunk); + return chunk; +} + +ChunkBuilder* BlobBuilder::addChunkAfter(ChunkBuilder* existingChunk) +{ + auto newChunk = new (_arena) ChunkBuilder(this); + _chunks.insertAfter(existingChunk, newChunk); + return newChunk; +} + +// +// ChunkBuilder +// + +void ChunkBuilder::setAlignmentToAtLeast(Size alignment) +{ + SLANG_ASSERT(alignment > 0); + SLANG_ASSERT(isPowerOfTwo(alignment)); + + _contentAlignment = std::max(_contentAlignment, alignment); +} + +void ChunkBuilder::writePaddingToAlignTo(Size alignment) +{ + setAlignmentToAtLeast(alignment); + + auto alignedSize = roundUpToAlignment(_contentSize, alignment); + + auto requiredPaddingSize = alignedSize - _contentSize; + + while (requiredPaddingSize) + { + Byte padByte = 0; + writeData(&padByte, sizeof(padByte)); + requiredPaddingSize -= sizeof(padByte); + } +} + +ChunkBuilder::ChunkBuilder(BlobBuilder* parentBlob) + : _parentBlob(parentBlob) +{ +} + +Size ChunkBuilder::getContentSize() const +{ + return _contentSize; +} + +Size ChunkBuilder::getPrefixSize() const +{ + if (!_prefixShard) + return 0; + + return _prefixShard->getSize(); +} + +Size ChunkBuilder::getAlignment() const +{ + return _contentAlignment; +} + + +void ChunkBuilder::writeData(void const* data, Size size) +{ + // Adding no data should be a no-op. + // + if (size == 0) + return; + + + // The most interesting implementation detail here + // is that we will try to detect cases where we + // can re-use an existing `ShardBuilder` by adding the data + // to the end of that shard's allocation. + // + // This is only possible because of the way that + // we are using a single `MemoryArena` to allocate + // everything, which makes it possible that the + // next address the arena would return for an allocation + // of `size` bytes is the same as the ending address + // of the payload for the last shard of this chunk. + // + auto& arena = getParentBlob()->_getArena(); + + // We start by checking if this chunk already has + // a last shard that we could consider appending to. + // + auto lastShard = _childShards.getLast(); + if (lastShard && lastShard->_kind == ShardBuilder::Kind::Data) + { + // If there is a last shard, then we can compute + // the end address of its payload, and see if + // it is the same as the cursor of the arena + // we are allocating from. + // + auto payload = lastShard->_data.ptr; + auto payloadSize = lastShard->_size; + auto payloadEnd = (Byte*)payload + payloadSize; + if (payloadEnd == arena.getCursor()) + { + // Now that we've confirmed that the shard's + // payload ends at an address the arena could + // conceivably allocate from, we need to ask + // the arena to allocate `size` bytes from + // the current block it is using, and see if + // doing so succeeds. + // + if (arena.allocateCurrentUnaligned(size)) + { + // At this point, we've confirmed that we + // are in our special case, and the relevant + // bytes have been allocated from the arena. + // + // Now we can simply write the new data at + // what used to be the end address for the + // shard's payload, and adjust its state + // to account for the new allocation. + // + ::memcpy(payloadEnd, data, size); + + lastShard->_size = payloadSize + size; + _contentSize += size; + return; + } + } + } + + // If the special case above doesn't apply, we simply + // allocate a new shard to hold the data that was + // passed in. + // + auto shard = _createDataShard(data, size); + _childShards.add(shard); + _contentSize += size; +} + +void ChunkBuilder::addContentsOf(ChunkBuilder* otherChunk) +{ + auto otherPrefixShard = otherChunk->_prefixShard; + auto otherChunkSize = otherChunk->getContentSize(); + auto otherChunkAlignment = otherChunk->getAlignment(); + auto otherChunkShards = otherChunk->_childShards; + + otherChunk->_prefixShard = nullptr; + otherChunk->_contentSize = 0; + otherChunk->_contentAlignment = 1; + otherChunk->_childShards = InternallyLinkedList<ShardBuilder>(); + + if (otherPrefixShard) + { + // If the other chunk included a prefix, then + // it only makes sense to append it in the case + // where *this* chunk is completely empty. + + SLANG_ASSERT(!_prefixShard); + SLANG_ASSERT(!_childShards.getFirst()); + _prefixShard = otherPrefixShard; + } + + writePaddingToAlignTo(otherChunkAlignment); + _childShards.append(otherChunkShards); + _contentSize += otherChunkSize; +} + +ChunkBuilder* ChunkBuilder::addChunkAfter() +{ + return getParentBlob()->addChunkAfter(this); +} + +void ChunkBuilder::_writeRelativePtr(ChunkBuilder* targetChunk, Size ptrSize) +{ + SLANG_ASSERT(ptrSize != 0); + SLANG_ASSERT(ptrSize <= sizeof(UInt64)); + + writePaddingToAlignTo(ptrSize); + + if (!targetChunk) + { + UInt64 value = 0; + writeData(&value, ptrSize); + return; + } + + auto shard = _createRelativePtrShard(targetChunk, ptrSize); + _childShards.add(shard); + _contentSize += ptrSize; +} + +void ChunkBuilder::_addPrefixRelativePtr(ChunkBuilder* targetChunk, Size ptrSize) +{ + SLANG_ASSERT(targetChunk != nullptr); + SLANG_ASSERT(ptrSize != 0); + + SLANG_ASSERT(!_prefixShard); + + setAlignmentToAtLeast(ptrSize); + + auto shard = _createRelativePtrShard(targetChunk, ptrSize); + _prefixShard = shard; +} + +void ChunkBuilder::addPrefixData(void const* data, Size size) +{ + SLANG_ASSERT(data != nullptr); + SLANG_ASSERT(size != 0); + + SLANG_ASSERT(!_prefixShard); + + setAlignmentToAtLeast(size); + + auto shard = _createDataShard(data, size); + _prefixShard = shard; +} + +ShardBuilder* ChunkBuilder::_createDataShard(void const* data, Size size) +{ + auto& arena = getParentBlob()->_getArena(); + auto shard = new (arena) ShardBuilder(ShardBuilder::Kind::Data); + + auto shardData = arena.allocateUnaligned(size); + ::memcpy(shardData, data, size); + + shard->_data.ptr = shardData; + shard->_size = size; + + return shard; +} + +ShardBuilder* ChunkBuilder::_createRelativePtrShard(ChunkBuilder* targetChunk, Size ptrSize) +{ + auto& arena = getParentBlob()->_getArena(); + auto shard = new (arena) ShardBuilder(ShardBuilder::Kind::RelativePtr); + + shard->_relativePtr.targetChunk = targetChunk; + shard->_size = ptrSize; + + return shard; +} + +void ChunkBuilder::_writeTo(Stream* stream) +{ + auto chunkOffset = _getCachedOffset(); + + if (_prefixShard) + { + // Note that the prefix is written *before* the + // starting offset of the chunk's content, so we + // compute the appropriate offset to pass down. + // + auto prefixSize = _prefixShard->getSize(); + auto prefixOffset = chunkOffset - prefixSize; + + _prefixShard->_writeTo(stream, prefixOffset); + } + + auto shardOffset = chunkOffset; + for (auto shard : _childShards) + { + shard->_writeTo(stream, shardOffset); + shardOffset += shard->getSize(); + } + SLANG_ASSERT(shardOffset == chunkOffset + getContentSize()); +} + + +// +// ShardBuilder +// + +ShardBuilder::ShardBuilder(Kind kind) + : _kind(kind) +{ +} + +void ShardBuilder::_writeTo(Stream* stream, Size inSelfOffset) +{ + switch (_kind) + { + case Kind::Data: + { + stream->write(_data.ptr, _size); + } + break; + + case Kind::RelativePtr: + { + auto targetChunk = _relativePtr.targetChunk; + SLANG_ASSERT(targetChunk); + + auto selfOffset = intptr_t(inSelfOffset); + auto targetOffset = intptr_t(targetChunk->_getCachedOffset()); + + intptr_t relativeOffset = targetOffset - selfOffset; + + switch (_size) + { + case sizeof(Int32): + { + auto value = Int32(relativeOffset); + stream->write(&value, sizeof(value)); + } + break; + + case sizeof(Int64): + { + auto value = Int64(relativeOffset); + stream->write(&value, sizeof(value)); + } + break; + + default: + SLANG_UNEXPECTED("unsupported relative pointer size"); + break; + } + } + break; + + default: + SLANG_UNEXPECTED("unknown Fossil::ShardBuilder::Kind"); + break; + } +} + +} // namespace Slang diff --git a/source/core/slang-blob-builder.h b/source/core/slang-blob-builder.h new file mode 100644 index 000000000..0b59ee7ab --- /dev/null +++ b/source/core/slang-blob-builder.h @@ -0,0 +1,352 @@ +// slang-blob-builder.h +#ifndef SLANG_BLOB_BUILDER_H +#define SLANG_BLOB_BUILDER_H + +// This file provides utilities for building "blobs" of data +// where, for purposes, a blob is a contiguous sequence of +// bytes where the interpretation *of* those bytes depends +// only on the bytes themselves, and not other factors like +// the in-memory address of the blob, or the address/contents +// of memory not in the blob. +// +// Superficially, the task seems simple: just maintain a +// dynamically-sized array of bytes and append to it until +// you're done. If that's what you need, you're probably +// better off just using an `OwnedMemoryStream`. +// +// The utilities in this file deal with the case where you +// want to build some kind of offset-based data structure, +// so that parts of the blob will store byte offsets to +// other parts, while also being able to build parts of +// that structure "out of order", so that the final offset +// of a particular piece of data in the blob may not be +// known until everything *before* it has been fully built. + +#include "slang-basic.h" +#include "slang-internally-linked-list.h" +#include "slang-io.h" +#include "slang-memory-arena.h" + +namespace Slang +{ + +inline constexpr bool isPowerOfTwo(Size value) +{ + return value > 0 && (value - 1 & value) == 0; +} + +inline constexpr Size roundUpToAlignment(Size size, Size alignment) +{ + SLANG_ASSERT(isPowerOfTwo(alignment)); + + auto alignmentMask = Size(alignment) - 1; + return (size + alignmentMask) & ~alignmentMask; +} + +class ShardBuilder; +class ChunkBuilder; +struct BlobBuilder; + + +/// A utility type for composing a binary blob. +/// +/// A blob builder allows a blob to be composed as a sequence of discrete +/// chunks, allowing chunks to be added and written to in any order. +/// +/// Chunks can contain (relative) pointers to one another, with the correct +/// relative offsets being computed as part of writing the entire blob out. +/// +struct BlobBuilder +{ +public: + /// Construct an empty blob builder. + BlobBuilder(); + + /// Write the contents of the blob to the given `stream`. + void writeTo(Stream* stream); + + /// Create a copy of the contents of the blob and assign to `outBlob`. + void writeToBlob(ISlangBlob** outBlob); + + /// Add a new empty chunk to the end of the blob. + ChunkBuilder* addChunk(); + + /// Add a new empty chunk after the given `chunk`. + ChunkBuilder* addChunkAfter(ChunkBuilder* chunk); + + /// Create a chunk that is not initially part of the blob. + /// + /// The contents of the returned chunk will only become + /// part of the full blob if `addChunk()` is called later, + /// *or* if the contents of the new chunk are moved into + /// another chunk that gets added. + /// + ChunkBuilder* createUnparentedChunk(); + + /// Add a chunk to the blob that was initially not part of the blob. + void addChunk(ChunkBuilder* chunk); + +private: + InternallyLinkedList<ChunkBuilder> _chunks; + MemoryArena _arena; + + friend class ChunkBuilder; + friend class ShardBuilder; + MemoryArena& _getArena() { return _arena; } + + Size _calcSizeAndSetCachedChunkOffsets(); + Size _writeChunksTo(Stream* stream); +}; + +/// A chunk is a logically contiguous unit, such that data can +/// only ever be appended to it. As a result, offsets that are +/// relative to the start of a chunk are meaningful, and can be +/// used to encode relative offsets for pointers. +/// +/// Every `ChunkBuilder` is owned by its parent `BlobBuilder`. +/// A pointer to a `ChunkBuilder` will only be valid during the +/// lifetime of the parent `BlobBuilder`. +/// +/// Conceptually, a `ChunkBuilder` has the following: +/// +/// * A minimum *alignment* in bytes (initially 1) +/// +/// * Zero or more bytes of *content* data (initially empty) +/// +/// * An optional *prefix* consisting of zero or more bytes (initially absent) +/// +/// The content may be a mix of raw data (e.g., added via `writeData()`), +/// relative pointers to other chunks (`writeRelativePtr()`) and padding +/// bytes (`writePaddingToAlignTo()`). +/// +/// The prefix may only be either a single relative pointer, or a +/// single range of raw data bytes. +/// +/// When the parent blob written out as a flat buffer of bytes, the +/// following are guaranteed: +/// +/// * The content of the chunk will be a contiguous range of bytes +/// starting at some offset, and will not overlap any other chunks. +/// +/// * The byte offset where the chunk contents start will be a multiple +/// of the chunk's minimum alignment. +/// +/// * The bytes of the prefix (if any) will immediately precede the +/// bytes of the content. +/// +class ChunkBuilder : public InternallyLinkedList<ChunkBuilder>::Node +{ +public: + /// Get the blob builder that this chunk belongs to. + BlobBuilder* getParentBlob() const { return _parentBlob; } + + /// Get the required alignment of this chunk. + /// + /// The minimum alignment for a chunk starts at 1, + /// and may be increased by operations such as + /// `setAlignmentToAtLeast()` and `writePaddingToAlinTo()`. + /// + Size getAlignment() const; + + /// Potentially increases the required alignment of this chunk. + /// + /// If the alignment of the chunk is less than `alignment`, + /// then it will be increased to `alignment`. + /// + /// The `alignment` passed in must be a power of two. + /// + void setAlignmentToAtLeast(Size alignment); + + /// Get the size in bytes of the content of this chunk. + /// + /// Note that the size is not necessarily a multiple of the + /// alignment of the chunk. + /// + Size getContentSize() const; + + /// Write data into the chunk. + /// + /// The chunk will retain a copy of the data passed in, + /// so the `data` pointer only needs to be valid for + /// the duration of the call. + /// + /// Note that this operation does *not* adjust the + /// alignment of the chunk in any way. + /// + /// The data must only contain types that can be copied + /// bit-for-bit, and that do not depend on addresses + /// in meory. In particular no pointers (absolute or + /// relative) should be written. + /// + void writeData(void const* data, Size size); + + /// Append padding bytes to this chunk until its content size + /// is a multiple of `alignment`. + /// + /// May also increase the alignment of the chunk, as if + /// calling `setAlignmentToAtLeast(alignment)`. + /// + /// The padding bytes will all be zero. + /// + void writePaddingToAlignTo(Size alignment); + + /// Write a relative pointer to the given `targetChunk`. + /// + /// The type parameter `T` is used to determine the size + /// of the relative pointer (should be either 4 or 8 bytes). + /// + /// The bytes that eventually get written will contain + /// the computed offset of `targetChunk` minus the computed + /// offset of the first byte of the relative pointer itself. + /// + /// Acts as is `writePaddingToAlignTo(sizeof(T))` were + /// called immediately before. + /// + template<typename T> + void writeRelativePtr(ChunkBuilder* targetChunk) + { + _writeRelativePtr(targetChunk, sizeof(T)); + } + + /// Append the contents of another chunk to this one. + /// + /// This *moves* all of the contents of `chunk` into `this`. + /// After the operation completes `chunk` will be an empty + /// chunk with one-byte alignment. + /// + /// This operation is useful when accumulating data that + /// needs to be appended to the chunk, but where the correct + /// alignment for that data is not yet known; a use can + /// effectively create a temporary "sub-chunk" and then append + /// it to the main chunk once its correct alignment is known. + /// + void addContentsOf(ChunkBuilder* chunk); + + /// Get the size in bytes of the prefix of this chunk, if any. + /// + /// The prefix will be written so that the *end* offset of the + /// prefix data is the same as the starting offset of the + /// chunk's content. + /// + Size getPrefixSize() const; + + /// Add a prefix to this chunk, consisting of raw data. + /// + /// This chunk must not already have a prefix. + /// + void addPrefixData(void const* data, Size size); + + /// Add a prefix to this chunk, consisting of a relative pointer. + /// + /// This chunk must not already have a prefix. + /// + /// Updates the alignment of the chunk as if making a call to + /// `setAlignmentToAtLeast(sizeof(T))`. + /// + template<typename T> + void addPrefixRelativePtr(ChunkBuilder* targetChunk) + { + _addPrefixRelativePtr(targetChunk, sizeof(T)); + } + + /// Insert a new chunk into the blob, immediately after this chunk. + /// + ChunkBuilder* addChunkAfter(); + +private: + ChunkBuilder() = delete; + ChunkBuilder(ChunkBuilder const&) = delete; + ChunkBuilder(ChunkBuilder&&) = delete; + + friend struct BlobBuilder; + friend class ShardBuilder; + + ChunkBuilder(BlobBuilder* parentBlob); + + Size _contentSize = 0; + Size _contentAlignment = 1; + + InternallyLinkedList<ShardBuilder> _childShards; + BlobBuilder* _parentBlob = nullptr; + + Size _cachedOffset = ~Size(0); + Size _getCachedOffset() { return _cachedOffset; } + void _setCachedOffset(Size offset) { _cachedOffset = offset; } + + void _writeRelativePtr(ChunkBuilder* targetChunk, Size ptrSize); + + ShardBuilder* _createDataShard(void const* data, Size size); + ShardBuilder* _createRelativePtrShard(ChunkBuilder* targetChunk, Size ptrSize); + + ShardBuilder* _prefixShard = nullptr; + + void _writeTo(Stream* stream); + + void _addPrefixRelativePtr(ChunkBuilder* targetChunk, Size ptrSize); +}; + +/// A shard is a unit of contiguously-allocated data that makes +/// up part of a chunk. +/// +/// Shards are *not* meant to be directly manipulated by users; +/// they are an implementation detail of `ChunkBuilder`. +/// +/// Every `ShardBuilder` is owned by its parent `ChunkBuilder`. +/// A pointer to a `ShardBuilder` will only be valid during the +/// lifetime of the parent `ChunkBuilder`. +/// +class ShardBuilder : public InternallyLinkedList<ShardBuilder>::Node +{ +public: + // There are two kinds of shards that may appear in a chunk: + // + // * Shards that hold plain data that will be part of the + // serialized chunk. + // + // * Shards that represent a relative pointer to some chunk, + // which cannot have their exact binary value determined + // until the offsets of chunk/shards have been finalized. + + enum class Kind + { + Data, + RelativePtr, + }; + + Size getSize() const { return _size; } + +private: + ShardBuilder() = delete; + ShardBuilder(ShardBuilder const&) = delete; + ShardBuilder(ShardBuilder&&) = delete; + + friend class ChunkBuilder; + ShardBuilder(Kind kind); + + void _writeTo(Stream* stream, Size selfOffset); + + /// Kind of this shard (data or relative pointer) + Kind _kind = Kind::Data; + + union + { + /// Used when `_kind == Kind::Data` + struct + { + void const* ptr; + } _data; + + /// Used when `_kind == Kind::RelativePtr` + struct + { + ChunkBuilder* targetChunk; + } _relativePtr; + }; + + // Size of this shard in bytes. + Size _size = 0; +}; + +} // namespace Slang + +#endif diff --git a/source/core/slang-internally-linked-list.cpp b/source/core/slang-internally-linked-list.cpp new file mode 100644 index 000000000..27bbd77b0 --- /dev/null +++ b/source/core/slang-internally-linked-list.cpp @@ -0,0 +1,2 @@ +// slang-internally-linked-list.cpp +#include "slang-internally-linked-list.h" diff --git a/source/core/slang-internally-linked-list.h b/source/core/slang-internally-linked-list.h new file mode 100644 index 000000000..168885531 --- /dev/null +++ b/source/core/slang-internally-linked-list.h @@ -0,0 +1,126 @@ +// slang-internally-linked-list.h +#ifndef SLANG_INTERNALLY_LINKED_LIST_H +#define SLANG_INTERNALLY_LINKED_LIST_H + +// This file provides support for the idiom of a linked +// list of values where the "next" pointer is stored in +// the values themselves (thus requiring no additional +// allocation for list nodes, at the price of any given +// value only being able to appear in a single list). + +#include "slang-basic.h" + +namespace Slang +{ + +/// A linked list where the elements are themselves the nodes. +/// +/// The type parameter `T` should be a type that publicly +/// inherits from `InternallyLinkedList<T>::Node`. +/// +template<typename T> +struct InternallyLinkedList +{ +public: + struct Node + { + public: + Node() {} + + private: + friend struct InternallyLinkedList<T>; + T* _next = nullptr; + }; + + struct Iterator + { + public: + Iterator() {} + + Iterator(T* node) + : _node(node) + { + } + + T* operator*() const { return _node; } + + void operator++() { _node = static_cast<Node const*>(_node)->_next; } + + bool operator!=(Iterator const& that) const { return _node != that._node; } + + private: + T* _node = nullptr; + }; + + Iterator begin() { return Iterator(_first); } + + Iterator end() { return Iterator(); } + + T* getFirst() const { return _first; } + + T* getLast() const { return _last; } + + void add(T* element) + { + SLANG_ASSERT(element != nullptr); + if (!_last) + { + SLANG_ASSERT(_first == nullptr); + + _first = element; + _last = element; + } + else + { + SLANG_ASSERT(_first != nullptr); + + _last->_next = element; + _last = element; + } + } + + void insertAfter(T* existingElement, T* newElement) + { + SLANG_ASSERT(existingElement != nullptr); + SLANG_ASSERT(newElement != nullptr); + if (existingElement == _last) + { + add(newElement); + } + else + { + newElement->_next = existingElement->_next; + existingElement->_next = newElement; + } + } + + void append(InternallyLinkedList<T> const& other) + { + if (!other._first) + { + } + else if (!_last) + { + _first = other._first; + _last = other._last; + } + else + { + SLANG_ASSERT(_first != nullptr); + + _last->_next = other._first; + _last = other._last; + } + } + +private: + T* _first = nullptr; + T* _last = nullptr; +}; + +template<typename T> +using InternallyLinkedListNode = InternallyLinkedList<T>::Node; + +} // namespace Slang + +#endif diff --git a/source/core/slang-memory-arena.h b/source/core/slang-memory-arena.h index 03631befe..714918b9a 100644 --- a/source/core/slang-memory-arena.h +++ b/source/core/slang-memory-arena.h @@ -476,4 +476,15 @@ SLANG_FORCE_INLINE void operator delete(void* memory, Slang::MemoryArena& arena) SLANG_UNUSED(arena); } +SLANG_FORCE_INLINE void* operator new[](size_t size, Slang::MemoryArena& arena) +{ + return arena.allocate(size); +} + +SLANG_FORCE_INLINE void operator delete[](void* memory, Slang::MemoryArena& arena) +{ + SLANG_UNUSED(memory); + SLANG_UNUSED(arena); +} + #endif // SLANG_MEMORY_ARENA_H diff --git a/source/core/slang-relative-ptr.cpp b/source/core/slang-relative-ptr.cpp new file mode 100644 index 000000000..0c4da2c6e --- /dev/null +++ b/source/core/slang-relative-ptr.cpp @@ -0,0 +1,2 @@ +// slang-relative-ptr.cpp +#include "slang-relative-ptr.h" diff --git a/source/core/slang-relative-ptr.h b/source/core/slang-relative-ptr.h new file mode 100644 index 000000000..6c4bc942c --- /dev/null +++ b/source/core/slang-relative-ptr.h @@ -0,0 +1,97 @@ +// slang-relative-ptr.h +#ifndef SLANG_RELATIVE_PTR_H +#define SLANG_RELATIVE_PTR_H + +// This file implements a smart pointer type `RelativePtr<T>` +// that, rather than storing the actual *address* of a value +// of type `T`, stores the relative offset (in bytes) between +// the target `T*` and the address of the `RelativePtr<T>` +// itself. +// +// This kind of pointer representation can be useful when +// implementing "memory-mappable" data structures that can +// still conveniently represent complicated object graphs. + +#include "slang-basic.h" + +namespace Slang +{ +namespace detail +{ +struct RelativePtr32Traits +{ + using Offset = Int32; + using UOffset = UInt32; +}; + +struct RelativePtr64Traits +{ + using Offset = Int64; + using UOffset = UInt64; +}; +} // namespace detail + +template<typename T, typename Traits> +struct RelativePtr +{ +public: + using This = RelativePtr<T, Traits>; + using Value = T; + using RawPtr = T*; + using Offset = typename Traits::Offset; + using UOffset = typename Traits::UOffset; + + RelativePtr() = default; + RelativePtr(RelativePtr const& ptr) { set(ptr); } + RelativePtr(RelativePtr&& ptr) { set(ptr); } + RelativePtr(T* ptr) { set(ptr); } + + void operator=(RelativePtr const& ptr) { set(ptr); } + void operator=(RelativePtr&& ptr) { set(ptr); } + void operator=(T* ptr) { set(ptr); } + + T* get() const + { + if (_offset == 0) + { + return nullptr; + } + + intptr_t thisAddr = intptr_t(this); + intptr_t targetAddr = thisAddr + intptr_t(_offset); + + return (T*)(targetAddr); + } + + void set(T* ptr) + { + if (ptr == nullptr) + { + _offset = 0; + return; + } + + intptr_t thisAddr = intptr_t(this); + intptr_t targetAddr = intptr_t(ptr); + intptr_t offsetVal = targetAddr - thisAddr; + + _offset = Offset(offsetVal); + SLANG_ASSERT(intptr_t(_offset) == offsetVal); + } + + operator T*() const { return get(); } + T* operator->() const { return get(); } + +private: + Offset _offset = 0; +}; + +template<typename T> +using RelativePtr32 = RelativePtr<T, detail::RelativePtr32Traits>; + +template<typename T> +using RelativePtr64 = RelativePtr<T, detail::RelativePtr64Traits>; + +} // namespace Slang + +#endif diff --git a/source/core/slang-riff.h b/source/core/slang-riff.h index 20b1d7c66..9459c8a55 100644 --- a/source/core/slang-riff.h +++ b/source/core/slang-riff.h @@ -16,6 +16,7 @@ // #include "slang-basic.h" +#include "slang-internally-linked-list.h" #include "slang-memory-arena.h" #include "slang-stream.h" #include "slang-writer.h" @@ -564,73 +565,6 @@ T const* as(Chunk const* chunk) return static_cast<T const*>(chunk); } -/// A linked list where the elements are themselves the nodes. -/// -template<typename T> -struct InternallyLinkedList -{ -public: - struct Node - { - public: - Node() {} - - private: - friend struct InternallyLinkedList<T>; - T* _next = nullptr; - }; - - struct Iterator - { - public: - Iterator() {} - - Iterator(T* node) - : _node(node) - { - } - - T* operator*() const { return _node; } - - void operator++() { _node = static_cast<Node const*>(_node)->_next; } - - bool operator!=(Iterator const& that) const { return _node != that._node; } - - private: - T* _node = nullptr; - }; - - Iterator begin() { return Iterator(_first); } - - Iterator end() { return Iterator(); } - - T* getFirst() const { return _first; } - - T* getLast() const { return _last; } - - void add(T* element) - { - if (!_last) - { - SLANG_ASSERT(_first == nullptr); - - _first = element; - _last = element; - } - else - { - SLANG_ASSERT(_first != nullptr); - - _last->_next = element; - _last = element; - } - } - -private: - T* _first = nullptr; - T* _last = nullptr; -}; - /// A builder for a chunk in a RIFF. class ChunkBuilder : public InternallyLinkedList<ChunkBuilder>::Node { diff --git a/source/slang/slang-fossil.cpp b/source/slang/slang-fossil.cpp new file mode 100644 index 000000000..e3b2e2c9d --- /dev/null +++ b/source/slang/slang-fossil.cpp @@ -0,0 +1,187 @@ +// slang-fossil.cpp +#include "slang-fossil.h" + +namespace Slang +{ +namespace Fossil +{ + +const char Fossil::Header::kMagic[16] = { + '\xAB', // byte 0 + 'f', // byte 1 + 'o', // byte 2 + 's', // byte 3 + 's', // byte 4 + 'i', // byte 5 + 'l', // byte 6 + ' ', // byte 7 + '1', // byte 8 + '0', // byte 9 + '0', // byte 10 + '\xBB', // byte 11 + '\r', // byte 12 + '\n', // byte 13 + '\x1A', // byte 14 + '\n' // byte 15 +}; + +FossilizedValRef getRootValue(ISlangBlob* blob) +{ + return getRootValue(blob->getBufferPointer(), blob->getBufferSize()); +} + +FossilizedValRef getRootValue(void const* data, Size size) +{ + if (!data) + { + SLANG_UNEXPECTED("bad format for fossil"); + } + + // There must be enough data to at least hold the header. + // + // (In practice there would need to be more data than + // just the header, but checking this invariant is a start). + // + if (size < sizeof(Fossil::Header)) + { + SLANG_UNEXPECTED("bad format for fossil"); + } + + // Once we've checked that there's enough data, we can read + // the contents of the header. + // + auto header = reinterpret_cast<Fossil::Header const*>(data); + + // The "magic" bytes at the start of the header must be + // what we expect (which is the contents of `Fossil::Header::kMagic`). + // + if (memcmp(header->magic, Fossil::Header::kMagic, sizeof(Fossil::Header::kMagic)) != 0) + { + SLANG_UNEXPECTED("bad format for fossil"); + } + + auto reportedSize = header->totalSizeIncludingHeader; + if (reportedSize > size) + { + SLANG_UNEXPECTED("bad format for fossil"); + } + + auto rootValueVariant = header->rootValue.get(); + if (!rootValueVariant) + { + SLANG_UNEXPECTED("bad format for fossil"); + } + + return FossilizedValRef( + rootValueVariant->getContentData(), + rootValueVariant->getContentLayout()); +} + +} // namespace Fossil + +Size FossilizedStringObj::getSize() const +{ + auto sizePtr = (FossilUInt*)this - 1; + return Size(*sizePtr); +} + +UnownedTerminatedStringSlice FossilizedStringObj::getValue() const +{ + auto size = getSize(); + return UnownedTerminatedStringSlice((char*)this, size); +} + +Count FossilizedContainerObj::getElementCount() const +{ + auto countPtr = (FossilUInt*)this - 1; + return Size(*countPtr); +} + +FossilizedValLayout* FossilizedVariantObj::getContentLayout() const +{ + auto layoutPtrPtr = (FossilizedPtr<FossilizedValLayout>*)this - 1; + return (*layoutPtrPtr).get(); +} + +FossilizedValRef getPtrTarget(FossilizedPtrValRef ptrRef) +{ + auto ptrLayout = ptrRef.getLayout(); + auto ptrPtr = ptrRef.getData(); + return FossilizedValRef(ptrPtr->getTargetData(), ptrLayout->elementLayout); +} + +bool hasValue(FossilizedOptionalObjRef optionalRef) +{ + return optionalRef.getData() != nullptr; +} + +FossilizedValRef getValue(FossilizedOptionalObjRef optionalRef) +{ + auto optionalLayout = optionalRef.getLayout(); + auto valuePtr = optionalRef.getData(); + return FossilizedValRef(valuePtr, optionalLayout->elementLayout); +} + +Count getElementCount(FossilizedContainerObjRef containerRef) +{ + if (!containerRef) + return 0; + + auto containerPtr = containerRef.getData(); + return containerPtr->getElementCount(); +} + +FossilizedValRef getElement(FossilizedContainerObjRef containerRef, Index index) +{ + SLANG_ASSERT(index >= 0); + SLANG_ASSERT(index < getElementCount(containerRef)); + + auto containerLayout = containerRef.getLayout(); + auto elementLayout = containerLayout->elementLayout.get(); + auto elementStride = containerLayout->elementStride; + + auto elementsPtr = (Byte*)containerRef.getData(); + auto elementPtr = (FossilizedVal*)(elementsPtr + elementStride * index); + return FossilizedValRef(elementPtr, elementLayout); +} + +Count getFieldCount(FossilizedRecordValRef recordRef) +{ + auto recordLayout = recordRef.getLayout(); + return recordLayout->fieldCount; +} + +FossilizedRecordElementLayout* FossilizedRecordLayout::getField(Index index) const +{ + SLANG_ASSERT(index >= 0); + SLANG_ASSERT(index < fieldCount); + + auto fieldsPtr = (FossilizedRecordElementLayout*)(this + 1); + return fieldsPtr + index; +} + + +FossilizedValRef getField(FossilizedRecordValRef recordRef, Index index) +{ + SLANG_ASSERT(index >= 0); + SLANG_ASSERT(index < getFieldCount(recordRef)); + + auto recordLayout = recordRef.getLayout(); + auto field = recordLayout->getField(index); + + auto fieldsPtr = (Byte*)recordRef.getData(); + auto fieldPtr = (FossilizedVal*)(fieldsPtr + field->offset); + return FossilizedValRef(fieldPtr, field->layout); +} + +FossilizedValRef getVariantContent(FossilizedVariantObjRef variantRef) +{ + return getVariantContent(variantRef.getData()); +} + +FossilizedValRef getVariantContent(FossilizedVariantObj* variantPtr) +{ + return FossilizedValRef(variantPtr->getContentData(), variantPtr->getContentLayout()); +} + +} // namespace Slang diff --git a/source/slang/slang-fossil.h b/source/slang/slang-fossil.h new file mode 100644 index 000000000..dcc12bacb --- /dev/null +++ b/source/slang/slang-fossil.h @@ -0,0 +1,488 @@ +// slang-fossil.h +#ifndef SLANG_FOSSIL_H +#define SLANG_FOSSIL_H + +// +// This file defines a memory-mappable binary format for +// serialized object graphs. To distinguish this specific +// format from other serialization formats used in the +// Slang project, we refer to these serialized objects +// as *fossilized* objects, and to the format as the +// *fossil format*. +// +// The term "fossil" is being used here to refer to formerly +// "live" objects that have been converted into an alternative +// form that can no longer perform their original functions, +// but that can still be inspected and dug through. +// + +#include "../core/slang-relative-ptr.h" + +namespace Slang +{ +// A key part of the fossil representation is the use of *relative pointers*, +// so that a fossilized object graph can be traversed dirctly in memory +// without having to deserialize any of the intermediate objects. +// +// Fossil uses 32-bit relative pointers, to keep the format compact. + +template<typename T> +using FossilizedPtr = RelativePtr32<T>; + +// Various other parts of the format need to store offsets or counts, +// and for consistency we will store them with the same number of +// bits as the relative pointers already used in the format. +// +// To make it easier for us to (potentially) change the relative +// pointer size down the line, we define type aliases for the +// general-purpose integer types that will be used in fossilized data. + +using FossilInt = FossilizedPtr<void>::Offset; +using FossilUInt = FossilizedPtr<void>::UOffset; + +// +// The fossil format supports data that is *self-describing*. +// +// A `FossilizedValLayout` describes the in-memory layout of a fossilized +// value. Given a `FossilizedValLayout` and a pointer to the data +// for a particular value, it is possible to inspect the structure +// of the fossilized data. +// +// If all you have is a `FossilizedVal*`, then there is no way to access +// its contents without assuming it is of some particular type and casting +// it. +// +// A `FossilizedVariantObj` is a fossilized value that is self-describing; +// it stores a (relative) pointer to a layout, which can be used to inspect +// its own data/state. +// + +struct FossilizedVal; +struct FossilizedValLayout; +struct FossilizedVariantObj; + +/// Kinds of values that can appear in fossilized data. +enum class FossilizedValKind : FossilUInt +{ + Bool, + Int8, + Int16, + Int32, + Int64, + UInt8, + UInt16, + UInt32, + UInt64, + Float32, + Float64, + String, + Array, + Optional, + Dictionary, + Tuple, + Struct, + Ptr, + Variant, +}; + +/// Layout information about a fossilized value in memory. +/// +/// +/// Every `FossilizedValLayout` stores the kind of the value. +/// Based on that kind, specific additional fields may be +/// available as part of the layout. +/// +struct FossilizedValLayout +{ + FossilizedValKind kind; +}; + +struct FossilizedPtrLikeLayout +{ + // Note: we aren't using inheritance in the definitions + // of these types, because per the letter of the law in + // C++, a type is only "standard layout" when there is + // only a single type in the inheritance hierarchy that + // has (non-static) data members. + + FossilizedValKind kind; + FossilizedPtr<FossilizedValLayout> elementLayout; +}; + +struct FossilizedContainerLayout +{ + FossilizedValKind kind; + FossilizedPtr<FossilizedValLayout> elementLayout; + FossilUInt elementStride; +}; + +struct FossilizedRecordElementLayout +{ + FossilizedPtr<FossilizedValLayout> layout; + FossilUInt offset; +}; + +struct FossilizedRecordLayout +{ + FossilizedValKind kind; + FossilUInt fieldCount; + + // FossilizedRecordElementLayout elements[]; + + FossilizedRecordElementLayout* getField(Index index) const; +}; + +/// A reference to a fossilized value in memory (of type T), and its layout. +/// +template<typename T> +struct FossilizedValRef_ +{ +public: + using Val = T; + using Layout = typename T::Layout; + + /// Construct a null reference. + /// + FossilizedValRef_() {} + + /// Construct a reference to the given `data`, assuming it has the given `layout`. + /// + FossilizedValRef_(T* data, Layout* layout) + : _data(data), _layout(layout) + { + } + + /// Get the kind of value being referenced. + /// + /// This reference must not be null. + /// + FossilizedValKind getKind() + { + SLANG_ASSERT(getLayout()); + return getLayout()->kind; + } + + /// Get the layout of the value being referenced. + /// + Layout* getLayout() { return _layout; } + + /// Get a pointer to the value being referenced. + /// + T* getData() { return _data; } + + operator T*() const { return _data; } + + T* operator->() { return _data; } + +private: + T* _data = nullptr; + Layout* _layout = nullptr; +}; + +using FossilizedValRef = FossilizedValRef_<FossilizedVal>; + +/// A fossilized value in memory. +/// +/// There isn't a lot that can be done with a bare pointer to +/// a `FossilizedVal`. This type is mostly declared to allow +/// us to make it explicit when a pointer points to a fossilized +/// value (even if we don't know anything about its layout). +/// +struct FossilizedVal +{ +public: + using Kind = FossilizedValKind; + using Layout = FossilizedValLayout; + + /// Determine if a value with the given `kind` should be allowed to cast to this type. + static bool _isMatchingKind(Kind kind) + { + SLANG_UNUSED(kind); + return true; + } + +protected: + FossilizedVal() = default; + FossilizedVal(FossilizedVal const&) = default; + FossilizedVal(FossilizedVal&&) = default; + ~FossilizedVal() = default; +}; + +template<typename T, FossilizedValKind kKind> +struct FossilizedSimpleVal : FossilizedVal +{ +public: + T getValue() const { return _value; } + + /// Determine if a value with the given `kind` should be allowed to cast to this type. + static bool _isMatchingKind(Kind kind) { return kind == kKind; } + +private: + T _value; +}; + +using FossilizedInt8Val = FossilizedSimpleVal<int8_t, FossilizedValKind::Int8>; +using FossilizedInt16Val = FossilizedSimpleVal<int16_t, FossilizedValKind::Int16>; +using FossilizedInt32Val = FossilizedSimpleVal<int32_t, FossilizedValKind::Int32>; +using FossilizedInt64Val = FossilizedSimpleVal<int64_t, FossilizedValKind::Int64>; + +using FossilizedUInt8Val = FossilizedSimpleVal<uint8_t, FossilizedValKind::UInt8>; +using FossilizedUInt16Val = FossilizedSimpleVal<uint16_t, FossilizedValKind::UInt16>; +using FossilizedUInt32Val = FossilizedSimpleVal<uint32_t, FossilizedValKind::UInt32>; +using FossilizedUInt64Val = FossilizedSimpleVal<uint64_t, FossilizedValKind::UInt64>; + +using FossilizedFloat32Val = FossilizedSimpleVal<float, FossilizedValKind::Float32>; +using FossilizedFloat64Val = FossilizedSimpleVal<double, FossilizedValKind::Float64>; + +struct FossilizedBoolVal : FossilizedVal +{ +public: + bool getValue() const { return _value != 0; } + + /// Determine if a value with the given `kind` should be allowed to cast to this type. + static bool _isMatchingKind(Kind kind) { return kind == Kind::Bool; } + +private: + uint8_t _value; +}; + +struct FossilizedPtrVal : FossilizedVal +{ +public: + using Layout = FossilizedPtrLikeLayout; + + FossilizedVal* getTargetData() const { return _value.get(); } + + /// Determine if a value with the given `kind` should be allowed to cast to this type. + static bool _isMatchingKind(Kind kind) { return kind == Kind::Ptr; } + +private: + FossilizedPtr<FossilizedVal> _value; +}; + + +struct FossilizedRecordVal : FossilizedVal +{ +public: + using Layout = FossilizedRecordLayout; + + /// Determine if a value with the given `kind` should be allowed to cast to this type. + static bool _isMatchingKind(Kind kind) + { + switch (kind) + { + default: + return false; + + case Kind::Struct: + case Kind::Tuple: + return true; + } + } +}; + +// +// Some of the following subtypes of `FossilizedVal` are +// named as `Fossilized*Obj` rather than `Fossilized*Val`, +// to indicate that they will only ever be located on the +// other side of a pointer indirection. +// +// E.g., a field of a fossilized struct value should never +// have a layout claiming it to be of kind `String`; instead +// it should show as a field of kind `Ptr`, where the +// pointed-to type is `String`. The same goes for `Optional`, +// `Array`, and `Dictionary`. +// +// This distinction only matters when dealing with things like +// an *optional* string, because instead of an in-memory +// layout like `Ptr -> Optional -> Ptr -> String`, the fossilized +// data will simply store `Ptr -> Optional -> String`. +// + +struct FossilizedStringObj : FossilizedVal +{ +public: + Size getSize() const; + UnownedTerminatedStringSlice getValue() const; + + /// Determine if a value with the given `kind` should be allowed to cast to this type. + static bool _isMatchingKind(Kind kind) { return kind == Kind::String; } + +private: + // Before the `this` address, there is a `FossilUInt` + // with the size of the string in bytes. + // + // At the `this` address there is a nul-terminated + // serquence of `getSize() + 1` bytes. +}; + +struct FossilizedOptionalObj : FossilizedVal +{ +public: + using Layout = FossilizedPtrLikeLayout; + + /// Determine if a value with the given `kind` should be allowed to cast to this type. + static bool _isMatchingKind(Kind kind) { return kind == Kind::Optional; } + + FossilizedVal* getValue() { return this; } + + FossilizedVal const* getValue() const { return this; } + +private: + // An absent optional is encoded as a null pointer + // (so `this` would be null), while a present value + // is encoded as a pointer to that value. Thus the + // held value is at the same address as `this`. +}; + +struct FossilizedContainerObj : FossilizedVal +{ +public: + using Layout = FossilizedContainerLayout; + + Count getElementCount() const; + + /// Determine if a value with the given `kind` should be allowed to cast to this type. + static bool _isMatchingKind(Kind kind) + { + switch (kind) + { + default: + return false; + + case Kind::Array: + case Kind::Dictionary: + return true; + } + } + +private: + // Before the `this` address, there is a `FossilUInt` + // with the number of elements. + // + // At the `this` address there is a sequence of + // `getCount()` elements. The layout of those elements + // cannot be determined without having a `FossilizedContainerLayout` + // for this container. +}; + +struct FossilizedVariantObj : FossilizedVal +{ +public: + FossilizedValLayout* getContentLayout() const; + + + FossilizedVal* getContentData() { return this; } + FossilizedVal const* getContentData() const { return this; } + + static bool _isMatchingKind(Kind kind) { return kind == Kind::Variant; } + +private: + // Before the `this` address, there is a `FossilizedPtr<FossilizedValLayout>` + // with the layout of the content. + // + // The content itself starts at the `this` address, with its + // layout determined by `getContentLayout()`. +}; + +/// Dynamic cast of a reference to a fossilized value. +/// +template<typename T, typename U> +FossilizedValRef_<T> as(FossilizedValRef_<U> valRef) +{ + if (!valRef || !T::_isMatchingKind(valRef.getKind())) + return FossilizedValRef_<T>(); + + return FossilizedValRef_<T>( + static_cast<T*>(valRef.getData()), + reinterpret_cast<typename T::Layout*>(valRef.getLayout())); +} + +using FossilizedInt8ValRef = FossilizedValRef_<FossilizedInt8Val>; +using FossilizedInt16ValRef = FossilizedValRef_<FossilizedInt16Val>; +using FossilizedInt32ValRef = FossilizedValRef_<FossilizedInt32Val>; +using FossilizedInt64ValRef = FossilizedValRef_<FossilizedInt64Val>; +using FossilizedUInt8ValRef = FossilizedValRef_<FossilizedUInt8Val>; +using FossilizedUInt16ValRef = FossilizedValRef_<FossilizedUInt16Val>; +using FossilizedUInt32ValRef = FossilizedValRef_<FossilizedUInt32Val>; +using FossilizedUInt64ValRef = FossilizedValRef_<FossilizedUInt64Val>; +using FossilizedFloat32ValRef = FossilizedValRef_<FossilizedFloat32Val>; +using FossilizedFloat64ValRef = FossilizedValRef_<FossilizedFloat64Val>; +using FossilizedBoolValRef = FossilizedValRef_<FossilizedBoolVal>; +using FossilizedStringObjRef = FossilizedValRef_<FossilizedStringObj>; +using FossilizedPtrValRef = FossilizedValRef_<FossilizedPtrVal>; +using FossilizedOptionalObjRef = FossilizedValRef_<FossilizedOptionalObj>; +using FossilizedContainerObjRef = FossilizedValRef_<FossilizedContainerObj>; +using FossilizedRecordValRef = FossilizedValRef_<FossilizedRecordVal>; +using FossilizedVariantObjRef = FossilizedValRef_<FossilizedVariantObj>; + +FossilizedValRef getPtrTarget(FossilizedPtrValRef ptrRef); + +bool hasValue(FossilizedOptionalObjRef optionalRef); +FossilizedValRef getValue(FossilizedOptionalObjRef optionalRef); + +Count getElementCount(FossilizedContainerObjRef containerRef); +FossilizedValRef getElement(FossilizedContainerObjRef containerRef, Index index); + +Count getFieldCount(FossilizedRecordValRef recordRef); +FossilizedValRef getField(FossilizedRecordValRef recordRef, Index index); + +FossilizedValRef getVariantContent(FossilizedVariantObjRef variantRef); +FossilizedValRef getVariantContent(FossilizedVariantObj* variantPtr); + + +namespace Fossil +{ +using RelativePtrOffset = FossilizedPtr<void>::Offset; + +/// Header for a fossil-format file or blob. +/// +/// A blob of fossilized data must start with a `Header` +/// that is properly formatted. +/// +struct Header +{ + /// The "magic" bytes used to identify this is a fossil-format blob. + char magic[16]; + + /// The expected bytes that should appear in `magic` + /// + static const char kMagic[16]; + + /// The total size of the fossil-format blob, including this header. + UInt64 totalSizeIncludingHeader; + + /// Flags; reserved for future use. + UInt32 flags; + + /// A relative pointer to the root value of the object graph. + /// + /// A fossil-format blob may only have one root value, and that + /// value *must* be a variant (so that it can reference the + /// layout information that describes itself). The *content* + /// of the root object can be arbitrary, so applications may + /// store multiple values using an array, struct, etc. + /// + FossilizedPtr<FossilizedVariantObj> rootValue; +}; + +/// Get the root object from a fossilized blob. +/// +/// This operation performs some basic validation on the blob to +/// ensure that it doesn't seem incorrectly sized or otherwise +/// corrupted/malformed. +/// +FossilizedValRef getRootValue(ISlangBlob* blob); + +/// Get the root object from a fossilized blob. +/// +/// This operation performs some basic validation on the blob to +/// ensure that it doesn't seem incorrectly sized or otherwise +/// corrupted/malformed. +/// +FossilizedValRef getRootValue(void const* data, Size size); +} // namespace Fossil + +} // namespace Slang + +#endif diff --git a/source/slang/slang-serialize-ast.cpp b/source/slang/slang-serialize-ast.cpp index 3a2d3818e..02dd374c1 100644 --- a/source/slang/slang-serialize-ast.cpp +++ b/source/slang/slang-serialize-ast.cpp @@ -5,6 +5,7 @@ #include "slang-compiler.h" #include "slang-diagnostics.h" #include "slang-mangle.h" +#include "slang-serialize-fossil.h" #include "slang-serialize-riff.h" namespace Slang @@ -202,7 +203,7 @@ using ASTSerializer = Serializer_<ASTSerializerImpl>; template<typename T> void serializeObject(ASTSerializer const& serializer, T*& value, NodeBase*) { - SLANG_SCOPED_SERIALIZER_STRUCT(serializer); + SLANG_SCOPED_SERIALIZER_VARIANT(serializer); serializer->handleASTNode(*(NodeBase**)&value); } @@ -224,7 +225,7 @@ void serialize(ASTSerializer const& serializer, SourceLoc& value) void serialize(ASTSerializer const& serializer, RequirementWitness& value) { - SLANG_SCOPED_SERIALIZER_TAGGED_UNION(serializer); + SLANG_SCOPED_SERIALIZER_VARIANT(serializer); serialize(serializer, value.m_flavor); switch (value.m_flavor) { @@ -406,7 +407,7 @@ void serialize(ASTSerializer const& serializer, SPIRVAsmInst& value) void serialize(ASTSerializer const& serializer, ValNodeOperand& value) { - SLANG_SCOPED_SERIALIZER_TAGGED_UNION(serializer); + SLANG_SCOPED_SERIALIZER_VARIANT(serializer); serialize(serializer, value.kind); switch (value.kind) { @@ -480,19 +481,19 @@ struct ASTEncodingContext : ASTSerializerImpl { public: ASTEncodingContext( - RIFF::BuildCursor& cursor, + ISerializerImpl* writer, ModuleDecl* module, SerialSourceLocWriter* sourceLocWriter) - : _writer(cursor.getCurrentChunk()), _module(module), _sourceLocWriter(sourceLocWriter) + : _writer(writer), _module(module), _sourceLocWriter(sourceLocWriter) { } private: - RIFFSerialWriter _writer; + ISerializerImpl* _writer = nullptr; ModuleDecl* _module = nullptr; SerialSourceLocWriter* _sourceLocWriter = nullptr; - virtual ISerializerImpl* getBaseSerializer() override { return &_writer; } + virtual ISerializerImpl* getBaseSerializer() override { return _writer; } virtual void handleName(Name*& value) override; virtual void handleSourceLoc(SourceLoc& value) override; @@ -531,7 +532,7 @@ public: Linkage* linkage, ASTBuilder* astBuilder, DiagnosticSink* sink, - RIFF::Chunk const* baseChunk, + ISerializerImpl* reader, SerialSourceLocReader* sourceLocReader, SourceLoc requestingSourceLoc) : _linkage(linkage) @@ -539,7 +540,7 @@ public: , _sink(sink) , _sourceLocReader(sourceLocReader) , _requestingSourceLoc(requestingSourceLoc) - , _riffReader(baseChunk) + , _reader(reader) { } @@ -549,9 +550,9 @@ private: DiagnosticSink* _sink = nullptr; SerialSourceLocReader* _sourceLocReader = nullptr; SourceLoc _requestingSourceLoc; - RIFFSerialReader _riffReader; + ISerializerImpl* _reader = nullptr; - virtual ISerializerImpl* getBaseSerializer() override { return &_riffReader; } + virtual ISerializerImpl* getBaseSerializer() override { return _reader; } virtual void handleName(Name*& value) override; virtual void handleSourceLoc(SourceLoc& value) override; @@ -910,8 +911,21 @@ void writeSerializedModuleAST( // TODO: we might want to have a more careful pass here, // where we only encode the public declarations. - ASTEncodingContext context(cursor, moduleDecl, sourceLocWriter); - serialize(ASTSerializer(&context), moduleDecl); + BlobBuilder blobBuilder; + { + Fossil::SerialWriter writer(blobBuilder); + + ASTEncodingContext context(&writer, moduleDecl, sourceLocWriter); + serialize(ASTSerializer(&context), moduleDecl); + } + + ComPtr<ISlangBlob> blob; + blobBuilder.writeToBlob(blob.writeRef()); + + void const* data = blob->getBufferPointer(); + size_t size = blob->getBufferSize(); + + cursor.addDataChunk(PropertyKeys<Module>::ASTModule, data, size); } ModuleDecl* readSerializedModuleAST( @@ -922,8 +936,14 @@ ModuleDecl* readSerializedModuleAST( SerialSourceLocReader* sourceLocReader, SourceLoc requestingSourceLoc) { + auto dataChunk = as<RIFF::DataChunk>(chunk); + + auto rootVal = Fossil::getRootValue(dataChunk->getPayload(), dataChunk->getPayloadSize()); + + Fossil::SerialReader reader(rootVal); + ASTDecodingContext - context(linkage, astBuilder, sink, chunk, sourceLocReader, requestingSourceLoc); + context(linkage, astBuilder, sink, &reader, sourceLocReader, requestingSourceLoc); ModuleDecl* moduleDecl = nullptr; serialize(ASTSerializer(&context), moduleDecl); diff --git a/source/slang/slang-serialize-fossil.cpp b/source/slang/slang-serialize-fossil.cpp new file mode 100644 index 000000000..5af9d2e6e --- /dev/null +++ b/source/slang/slang-serialize-fossil.cpp @@ -0,0 +1,1636 @@ +// slang-serialize-fossil.cpp +#include "slang-serialize-fossil.h" + +#include "../core/slang-blob.h" + +namespace Slang +{ +namespace Fossil +{ + +// +// SerialWriter +// + +SerialWriter::SerialWriter(ChunkBuilder* chunk) + : _arena(4096) +{ + _initialize(chunk); +} + +SerialWriter::SerialWriter(BlobBuilder& blobBuilder) + : _arena(4096) +{ + auto chunk = blobBuilder.addChunk(); + _initialize(chunk); +} + +void SerialWriter::_initialize(ChunkBuilder* chunk) +{ + _blobBuilder = chunk->getParentBlob(); + + // The top-level structure consists of a header, + // and a root value. We will allocate a distinct + // chunk for each of them, with the header coming + // first. + // + auto headerChunk = chunk; + auto rootValueChunk = headerChunk->addChunkAfter(); + + // We will write the fields of the header chunk manually, + // although we will use a temporary of type `Fossil::Header` + // to help make sure we write them with the correct sizes. + // + Fossil::Header header; + memcpy(header.magic, Fossil::Header::kMagic, sizeof(Fossil::Header::kMagic)); + header.totalSizeIncludingHeader = 0; + header.flags = 0; + + headerChunk->writeData(&header.magic, sizeof(header.magic)); + headerChunk->writeData( + &header.totalSizeIncludingHeader, + sizeof(header.totalSizeIncludingHeader)); + headerChunk->writeData(&header.flags, sizeof(header.flags)); + + // The main reason we are writing the fields manually is + // that the last field of the header is a relative pointer + // to the root-value chunk. + // + headerChunk->writeRelativePtr<Fossil::RelativePtrOffset>(rootValueChunk); + + // The root value should always be a variant, and we want to + // set up to write into it in a reasonable way. + // + auto rootPtrLayout = _createLayout(FossilizedValKind::Ptr); + _state = State(rootPtrLayout, rootValueChunk); + + _pushVariantScope(); +} + + +SerialWriter::~SerialWriter() +{ + _popVariantScope(); + + _flush(); +} + +SerializationMode SerialWriter::getMode() +{ + return SerializationMode::Write; +} + +void SerialWriter::handleBool(bool& value) +{ + // A boolean value will be serialized as a full byte. + uint8_t v = value; + _writeSimpleValue(FossilizedValKind::Bool, v); +} + +void SerialWriter::handleInt8(int8_t& value) +{ + _writeSimpleValue(FossilizedValKind::Int8, value); +} + +void SerialWriter::handleInt16(int16_t& value) +{ + _writeSimpleValue(FossilizedValKind::Int16, value); +} + +void SerialWriter::handleInt32(Int32& value) +{ + _writeSimpleValue(FossilizedValKind::Int32, value); +} + +void SerialWriter::handleInt64(Int64& value) +{ + _writeSimpleValue(FossilizedValKind::Int64, value); +} + +void SerialWriter::handleUInt8(uint8_t& value) +{ + _writeSimpleValue(FossilizedValKind::UInt8, value); +} + +void SerialWriter::handleUInt16(uint16_t& value) +{ + _writeSimpleValue(FossilizedValKind::UInt16, value); +} + +void SerialWriter::handleUInt32(UInt32& value) +{ + _writeSimpleValue(FossilizedValKind::UInt32, value); +} + +void SerialWriter::handleUInt64(UInt64& value) +{ + _writeSimpleValue(FossilizedValKind::UInt64, value); +} + +void SerialWriter::handleFloat32(float& value) +{ + _writeSimpleValue(FossilizedValKind::Float32, value); +} + +void SerialWriter::handleFloat64(double& value) +{ + _writeSimpleValue(FossilizedValKind::Float64, value); +} + +void SerialWriter::handleString(String& value) +{ + auto size = value.getLength(); + if (_shouldEmitWithPointerIndirection(FossilizedValKind::String)) + { + if (size == 0) + { + _writeNull(); + return; + } + + if (auto found = _mapStringToChunk.tryGetValue(value)) + { + auto existingChunk = *found; + + auto ptrLayout = + (ContainerLayoutObj*)_reserveDestinationForWrite(FossilizedValKind::Ptr); + _mergeLayout(ptrLayout->baseLayout, FossilizedValKind::String); + + _commitWrite(ValInfo::relativePtrTo(existingChunk)); + return; + } + } + + _pushPotentiallyIndirectValueScope(FossilizedValKind::String); + + auto data = value.getBuffer(); + _writeValueRaw(ValInfo::rawData(data, size + 1, 1)); + + auto chunk = _popPotentiallyIndirectValueScope(); + + auto rawSize = UInt32(size); + chunk->addPrefixData(&rawSize, sizeof(rawSize)); + + _mapStringToChunk.addIfNotExists(value, chunk); +} + +void SerialWriter::beginArray() +{ + _pushContainerScope(FossilizedValKind::Array); +} + +void SerialWriter::endArray() +{ + _popContainerScope(); +} + +void SerialWriter::beginDictionary() +{ + _pushContainerScope(FossilizedValKind::Dictionary); +} + +void SerialWriter::endDictionary() +{ + _popContainerScope(); +} + +void SerialWriter::_pushContainerScope(FossilizedValKind kind) +{ + _pushPotentiallyIndirectValueScope(kind); +} + +void SerialWriter::_popContainerScope() +{ + auto elementCount = _state.elementCount; + auto containerChunk = _popPotentiallyIndirectValueScope(); + + if (containerChunk) + { + auto rawElementCount = UInt32(elementCount); + containerChunk->addPrefixData(&rawElementCount, sizeof(rawElementCount)); + } +} + +bool SerialWriter::hasElements() +{ + return false; +} + +void SerialWriter::beginStruct() +{ + _pushInlineValueScope(FossilizedValKind::Struct); +} + +void SerialWriter::endStruct() +{ + _popInlineValueScope(); +} + +void SerialWriter::beginVariant() +{ + _pushVariantScope(); + _pushInlineValueScope(FossilizedValKind::Struct); +} + +void SerialWriter::endVariant() +{ + _popInlineValueScope(); + _popVariantScope(); +} + +void SerialWriter::handleFieldKey(char const* name, Int index) +{ + // For now we are ignoring field keys, and treating + // structs as basically equivalent to tuples. + SLANG_UNUSED(name); + SLANG_UNUSED(index); +} + +void SerialWriter::beginTuple() +{ + _pushInlineValueScope(FossilizedValKind::Tuple); +} + +void SerialWriter::endTuple() +{ + _popInlineValueScope(); +} + +void SerialWriter::beginOptional() +{ + _pushIndirectValueScope(FossilizedValKind::Optional); +} + +void SerialWriter::endOptional() +{ + _popIndirectValueScope(); +} + +void SerialWriter::handleSharedPtr(void*& value, Callback callback, void* userData) +{ + // Because we are writing, we only care about the + // pointer that is already present in `value`. + // + void* liveObjectPtr = value; + + // The first special case we check for is a null pointer, + // which we can serialize as an inline value. + // + if (liveObjectPtr == nullptr) + { + _writeNull(); + return; + } + + // Next, we check to see if we have encountered this + // pointer before, in which case we've already allocated + // an index for it in the object definition list, and + // we can simply write a reference to that object. + // + if (auto found = _mapLiveObjectPtrToFossilizedObject.tryGetValue(liveObjectPtr)) + { + auto fossilizedObject = *found; + + _reserveDestinationForWrite(fossilizedObject->ptrLayout); + _commitWrite(ValInfo::relativePtrTo(fossilizedObject->chunk)); + + return; + } + + auto ptrLayout = _reserveDestinationForWrite(FossilizedValKind::Ptr); + auto chunk = _blobBuilder->addChunk(); + + auto fossilizedObject = new (_arena) FossilizedObjectInfo(); + fossilizedObject->chunk = chunk; + fossilizedObject->ptrLayout = ptrLayout; + fossilizedObject->liveObjectPtr = liveObjectPtr; + fossilizedObject->callback = callback; + fossilizedObject->userData = userData; + + _fossilizedObjects.add(fossilizedObject); + _mapLiveObjectPtrToFossilizedObject.add(liveObjectPtr, fossilizedObject); + + _commitWrite(ValInfo::relativePtrTo(chunk)); +} + +void SerialWriter::handleUniquePtr(void*& value, Callback callback, void* userData) +{ + // We treat all pointers as shared pointers, because there isn't really + // an optimized representation we would want to use for the unique case. + // + handleSharedPtr(value, callback, userData); +} + +void SerialWriter::handleDeferredObjectContents(void* valuePtr, Callback callback, void* userData) +{ + // Because we are already deferring writing of the *entirety* of + // an object's members as part of how `handleSharedPtr()` works, + // we don't need to implement deferral at this juncture. + // + // (In practice the `handleDeferredObjectContents()` operation is + // more for the benefit of reading than writing). + // + callback(valuePtr, userData); +} + +SerialWriter::LayoutObj* SerialWriter::_createSimpleLayout(FossilizedValKind kind) +{ + switch (kind) + { + case FossilizedValKind::Bool: + case FossilizedValKind::Int8: + case FossilizedValKind::UInt8: + return new (_arena) SimpleLayoutObj(kind, 1); + + case FossilizedValKind::Int16: + case FossilizedValKind::UInt16: + return new (_arena) SimpleLayoutObj(kind, 2); + + case FossilizedValKind::Int32: + case FossilizedValKind::UInt32: + case FossilizedValKind::Float32: + return new (_arena) SimpleLayoutObj(kind, 4); + + case FossilizedValKind::Int64: + case FossilizedValKind::UInt64: + case FossilizedValKind::Float64: + return new (_arena) SimpleLayoutObj(kind, 8); + + case FossilizedValKind::String: + return new (_arena) SimpleLayoutObj(kind); + + default: + SLANG_UNEXPECTED("unhandled case"); + UNREACHABLE_RETURN(nullptr); + } +} + +SerialWriter::LayoutObj* SerialWriter::_createLayout(FossilizedValKind kind) +{ + switch (kind) + { + case FossilizedValKind::Array: + case FossilizedValKind::Optional: + case FossilizedValKind::Dictionary: + return new (_arena) ContainerLayoutObj(kind, nullptr); + + case FossilizedValKind::Ptr: + return new (_arena) ContainerLayoutObj( + kind, + nullptr, + sizeof(Fossil::RelativePtrOffset), + sizeof(Fossil::RelativePtrOffset)); + + case FossilizedValKind::Struct: + case FossilizedValKind::Tuple: + return new (_arena) RecordLayoutObj(kind); + + case FossilizedValKind::Variant: + // A variant is being treated like a container in this context, + // because it wants to be able to track the layout of what it + // ended up holding... + // + return new (_arena) ContainerLayoutObj(kind, nullptr); + + case FossilizedValKind::Bool: + case FossilizedValKind::Int8: + case FossilizedValKind::Int16: + case FossilizedValKind::Int32: + case FossilizedValKind::Int64: + case FossilizedValKind::UInt8: + case FossilizedValKind::UInt16: + case FossilizedValKind::UInt32: + case FossilizedValKind::UInt64: + case FossilizedValKind::Float32: + case FossilizedValKind::Float64: + case FossilizedValKind::String: + { + if (auto found = _simpleLayouts.tryGetValue(kind)) + return *found; + + auto layout = _createSimpleLayout(kind); + _simpleLayouts.add(kind, layout); + return layout; + } + + default: + SLANG_UNEXPECTED("unhandled case"); + UNREACHABLE_RETURN(nullptr); + } +} + +SerialWriter::LayoutObj* SerialWriter::_mergeLayout(LayoutObj*& dst, FossilizedValKind kind) +{ + if (!dst) + { + dst = _createLayout(kind); + } + + if (dst->kind != kind) + { + SLANG_UNEXPECTED("type mismatch during serialization"); + } + + // As a special case, if the right-hand-side is a variant, + // then we want to have a unique layout object for each + // instance. + // + if (kind == FossilizedValKind::Variant) + { + auto src = _createLayout(kind); + return src; + } + + return dst; +} + +void SerialWriter::_mergeLayout(LayoutObj*& dst, LayoutObj* src) +{ + if (dst == src) + return; + + if (!src) + return; + + if (!dst) + { + dst = src; + return; + } + + _mergeLayout(dst, src->getKind()); + + switch (src->getKind()) + { + case FossilizedValKind::Array: + case FossilizedValKind::Optional: + case FossilizedValKind::Dictionary: + case FossilizedValKind::Ptr: + { + auto dstContainer = (ContainerLayoutObj*)dst; + auto srcContainer = (ContainerLayoutObj*)src; + _mergeLayout(dstContainer->baseLayout, srcContainer->baseLayout); + } + break; + + case FossilizedValKind::String: + break; + + case FossilizedValKind::Variant: + // Recursive merging should not be applied to variants; + // each variant is unique until later deduplication. + break; + + default: + SLANG_UNEXPECTED("unhandled case"); + break; + } +} + +SerialWriter::RecordLayoutObj::FieldInfo& SerialWriter::_getOrAddField( + RecordLayoutObj* recordLayout, + Index index) +{ + // Note: we are doing all the allocation for `LayoutObj`s from + // an arena, so that we don't have to worry about managing + // their lifetimes carefully. + // + // One place where that is a bit tedious is handling the storage + // for the array of fields for a record. + // + // TODO(tfoley): see if there's allocator support on `List<T>` + // or similar, so that it can be made to just use the arena. + + SLANG_ASSERT(recordLayout); + SLANG_ASSERT(index >= 0); + + if (index < recordLayout->fieldCount) + return recordLayout->fields[index]; + + SLANG_ASSERT(index == recordLayout->fieldCount); + + if (index >= recordLayout->fieldCapacity) + { + if (recordLayout->fieldCapacity == 0) + recordLayout->fieldCapacity = 16; + + while (index >= recordLayout->fieldCapacity) + { + recordLayout->fieldCapacity = (recordLayout->fieldCapacity * 3) >> 1; + } + + auto newFields = new (_arena) RecordLayoutObj::FieldInfo[recordLayout->fieldCapacity]; + for (Index i = 0; i < recordLayout->fieldCount; ++i) + newFields[i] = recordLayout->fields[i]; + recordLayout->fields = newFields; + } + + recordLayout->fields[recordLayout->fieldCount++] = RecordLayoutObj::FieldInfo(); + return recordLayout->fields[index]; +} + +SerialWriter::ValInfo SerialWriter::ValInfo::rawData(void const* data, Size size, Size alignment) +{ + ValInfo val(Kind::RawData); + val.data.ptr = data; + val.data.size = size; + val.data.alignment = alignment; + return val; +} + +SerialWriter::ValInfo SerialWriter::ValInfo::relativePtrTo(ChunkBuilder* targetChunk) +{ + ValInfo val(Kind::RelativePtr); + val.chunk = targetChunk; + return val; +} + +SerialWriter::ValInfo SerialWriter::ValInfo::contentsOf(ChunkBuilder* chunk) +{ + ValInfo val(Kind::ContentsOfChunk); + val.chunk = chunk; + return val; +} + +Size SerialWriter::ValInfo::getAlignment() const +{ + switch (kind) + { + case Kind::RelativePtr: + return sizeof(Fossil::RelativePtrOffset); + + case Kind::ContentsOfChunk: + return chunk->getAlignment(); + + case Kind::RawData: + return data.alignment; + + default: + SLANG_UNEXPECTED("unhandled case"); + break; + } +} + +void SerialWriter::_pushInlineValueScope(FossilizedValKind kind) +{ + auto layout = _reserveDestinationForWrite(kind); + _pushState(layout); +} + +void SerialWriter::_popInlineValueScope() +{ + auto layout = _state.layout; + auto chunk = _state.chunk; + + if (chunk) + { + if (layout->size == 0) + { + layout->size = chunk->getContentSize(); + } + SLANG_ASSERT(layout->size == chunk->getContentSize()); + } + + _popState(); + + _commitWrite(ValInfo::contentsOf(chunk)); +} + +void SerialWriter::_pushVariantScope() +{ + _pushPotentiallyIndirectValueScope(FossilizedValKind::Variant); +} + +void SerialWriter::_popVariantScope() +{ + SLANG_ASSERT(_state.layout); + SLANG_ASSERT(_state.layout->kind == FossilizedValKind::Variant); + auto variantLayout = (ContainerLayoutObj*)_state.layout; + auto valueLayout = variantLayout->baseLayout; + SLANG_ASSERT(valueLayout); + + auto variantChunk = _popPotentiallyIndirectValueScope(); + + // The key feature of a variant is that it carries its own + // layout information. + // + // We need to insert a pointer to the serialized form + // of the layout information for the element type as a header + // *before* the content. + // + // The first step there is to turn the element layout into + // a handle such that we can write a relative pointer to it. + // + + VariantInfo variantInfo; + variantInfo.layout = valueLayout; + variantInfo.chunk = variantChunk; + _variants.add(variantInfo); +} + + +void SerialWriter::_pushPotentiallyIndirectValueScope(FossilizedValKind kind) +{ + if (_shouldEmitWithPointerIndirection(kind)) + { + _pushIndirectValueScope(kind); + } + else + { + _pushInlineValueScope(kind); + } +} + +ChunkBuilder* SerialWriter::_popPotentiallyIndirectValueScope() +{ + // TODO(tfoley): Try to make this function just be a simple + // conditional to select between the functions for the + // indirect and inline cases. + + auto valueLayout = _state.layout; + auto valueChunk = _state.chunk; + _popState(); + + auto valueKind = valueLayout->getKind(); + if (_shouldEmitWithPointerIndirection(valueKind)) + { + return _writeKnownIndirectValueSharedLogic(valueChunk); + } + else + { + _commitWrite(ValInfo::contentsOf(valueChunk)); + return _state.chunk; + } +} + +void SerialWriter::_pushIndirectValueScope(FossilizedValKind kind) +{ + auto ptrLayout = (ContainerLayoutObj*)_reserveDestinationForWrite(FossilizedValKind::Ptr); + auto valueLayout = _mergeLayout(ptrLayout->baseLayout, kind); + + _pushState(valueLayout); +} + +void SerialWriter::_popIndirectValueScope() +{ + auto valueChunk = _state.chunk; + _popState(); + + _writeKnownIndirectValueSharedLogic(valueChunk); +} + +ChunkBuilder* SerialWriter::_writeKnownIndirectValueSharedLogic(ChunkBuilder* valueChunk) +{ + if (!valueChunk) + { + _commitWrite(ValInfo::relativePtrTo(nullptr)); + return nullptr; + } + + _blobBuilder->addChunk(valueChunk); + + _commitWrite(ValInfo::relativePtrTo(valueChunk)); + return valueChunk; +} + + +void SerialWriter::_pushState(LayoutObj* layout) +{ + _stack.add(_state); + _state = State(layout); +} + +void SerialWriter::_popState() +{ + SLANG_ASSERT(_stack.getCount() != 0); + _state = _stack.getLast(); + _stack.removeLast(); +} + +void SerialWriter::_ensureChunkExists() +{ + if (_state.chunk != nullptr) + return; + + _state.chunk = _blobBuilder->createUnparentedChunk(); +} + +void SerialWriter::_writeValueRaw(ValInfo const& val) +{ + switch (val.kind) + { + case ValInfo::Kind::RawData: + if (val.data.size == 0) + return; + _ensureChunkExists(); + _state.chunk->writePaddingToAlignTo(val.data.alignment); + _state.chunk->writeData(val.data.ptr, val.data.size); + break; + + case ValInfo::Kind::RelativePtr: + _ensureChunkExists(); + _state.chunk->writeRelativePtr<Fossil::RelativePtrOffset>(val.chunk); + break; + + case ValInfo::Kind::ContentsOfChunk: + { + if (!_state.chunk) + { + _state.chunk = val.chunk; + } + else + { + _state.chunk->addContentsOf(val.chunk); + } + } + break; + + default: + SLANG_UNEXPECTED("unknown Fossil::SerialWriter::ValInfo::Kind"); + break; + } +} + +bool SerialWriter::_shouldEmitWithPointerIndirection(FossilizedValKind kind) +{ + switch (kind) + { + default: + return false; + + case FossilizedValKind::Optional: + return true; + + case FossilizedValKind::Array: + case FossilizedValKind::Dictionary: + case FossilizedValKind::String: + case FossilizedValKind::Variant: + break; + } + + switch (_state.layout->getKind()) + { + default: + return true; + + case FossilizedValKind::Optional: + case FossilizedValKind::Ptr: + return false; + } +} + +SerialWriter::LayoutObj*& SerialWriter::_reserveDestinationForWrite() +{ + switch (_state.layout->getKind()) + { + case FossilizedValKind::Struct: + case FossilizedValKind::Tuple: + { + auto recordLayout = (RecordLayoutObj*)_state.layout; + auto elementIndex = _state.elementCount; + auto& elementLayout = _getOrAddField(recordLayout, elementIndex).layout; + return elementLayout; + } + break; + + case FossilizedValKind::Ptr: + case FossilizedValKind::Optional: + case FossilizedValKind::Array: + case FossilizedValKind::Dictionary: + case FossilizedValKind::Variant: + { + auto containerLayout = (ContainerLayoutObj*)_state.layout; + auto& elementLayout = containerLayout->baseLayout; + return elementLayout; + } + break; + + default: + SLANG_UNEXPECTED("unhandled case"); + break; + } +} + +SerialWriter::LayoutObj* SerialWriter::_reserveDestinationForWrite(FossilizedValKind srcKind) +{ + return _mergeLayout(_reserveDestinationForWrite(), srcKind); +} + +SerialWriter::LayoutObj* SerialWriter::_reserveDestinationForWrite(LayoutObj* srcLayout) +{ + SLANG_ASSERT(srcLayout != nullptr); + _mergeLayout(_reserveDestinationForWrite(), srcLayout); + return srcLayout; +} + +void SerialWriter::_commitWrite(ValInfo const& val) +{ + auto outerKind = _state.layout->getKind(); + switch (outerKind) + { + case FossilizedValKind::Struct: + case FossilizedValKind::Tuple: + { + auto recordLayout = (RecordLayoutObj*)_state.layout; + auto elementIndex = _state.elementCount++; + auto& fieldInfo = _getOrAddField(recordLayout, elementIndex); + + Size fieldOffset = 0; + if (elementIndex != 0) + { + auto chunk = _state.chunk; + chunk->writePaddingToAlignTo(val.getAlignment()); + + fieldOffset = chunk->getContentSize(); + } + fieldInfo.offset = fieldOffset; + + _writeValueRaw(val); + } + break; + + case FossilizedValKind::Optional: + case FossilizedValKind::Ptr: + case FossilizedValKind::Array: + case FossilizedValKind::Dictionary: + case FossilizedValKind::Variant: + { + auto elementIndex = _state.elementCount++; + + switch (outerKind) + { + case FossilizedValKind::Optional: + case FossilizedValKind::Ptr: + if (elementIndex > 0) + { + SLANG_UNEXPECTED( + "error during serialization: optional with more than one value inside!!"); + } + break; + + default: + break; + } + + _writeValueRaw(val); + } + break; + + default: + SLANG_UNEXPECTED("unhandled case"); + break; + } +} + +void SerialWriter::_writeSimpleValue( + FossilizedValKind kind, + void const* data, + size_t size, + size_t alignment) +{ + auto layout = _reserveDestinationForWrite(kind); + SLANG_ASSERT(layout->size == size); + SLANG_ASSERT(layout->alignment = alignment); + _commitWrite(ValInfo::rawData(data, size, alignment)); +} + +void SerialWriter::_writeNull() +{ + RelativePtrOffset offset = 0; + _writeSimpleValue(FossilizedValKind::Ptr, offset); +} + +void SerialWriter::_flush() +{ + while (_writtenObjectDefinitionCount < _fossilizedObjects.getCount()) + { + auto objectIndex = _writtenObjectDefinitionCount++; + auto fossilizedObject = _fossilizedObjects[objectIndex]; + + SLANG_ASSERT(fossilizedObject->liveObjectPtr); + + _state = State(fossilizedObject->ptrLayout, fossilizedObject->chunk); + + fossilizedObject->callback(&fossilizedObject->liveObjectPtr, fossilizedObject->userData); + } + + // Once we've written out all the payload data, we can start to work on + // serializing layout information for all the variant values that were + // written. + // + for (auto variantInfo : _variants) + { + auto layoutChunk = _getOrCreateChunkForLayout(variantInfo.layout); + variantInfo.chunk->addPrefixRelativePtr<Fossil::RelativePtrOffset>(layoutChunk); + } +} + +ChunkBuilder* SerialWriter::_getOrCreateChunkForLayout(LayoutObj* layout) +{ + if (!layout) + return nullptr; + + // We start by looking for an existing chunk for `layout`, + // which would be cached on the object itself. + // + if (auto existingChunk = layout->chunk) + return existingChunk; + + // Next we look for an existing chunk that matches the + // structure of `layout`. + // + LayoutObjKey key = {layout}; + if (auto found = _mapLayoutObjToChunk.tryGetValue(key)) + { + auto existingChunk = *found; + layout->chunk = existingChunk; + return existingChunk; + } + + // If no existing layout has been written to a chunk, + // then we'll create one. + // + auto chunk = _blobBuilder->addChunk(); + layout->chunk = chunk; + _mapLayoutObjToChunk.add(key, chunk); + + auto kind = layout->getKind(); + auto rawKind = UInt32(kind); + chunk->writeData(&rawKind, sizeof(rawKind)); + + switch (kind) + { + default: + break; + + case FossilizedValKind::Ptr: + case FossilizedValKind::Optional: + { + auto containerLayout = (ContainerLayoutObj*)layout; + auto elementLayout = containerLayout->baseLayout; + auto elementLayoutChunk = _getOrCreateChunkForLayout(elementLayout); + chunk->writeRelativePtr<Fossil::RelativePtrOffset>(elementLayoutChunk); + } + break; + + case FossilizedValKind::Array: + case FossilizedValKind::Dictionary: + { + auto containerLayout = (ContainerLayoutObj*)layout; + auto elementLayout = containerLayout->baseLayout; + auto elementLayoutChunk = _getOrCreateChunkForLayout(elementLayout); + chunk->writeRelativePtr<Fossil::RelativePtrOffset>(elementLayoutChunk); + + UInt32 elementStride = 0; + if (elementLayout) + { + elementStride = + UInt32(roundUpToAlignment(elementLayout->size, elementLayout->alignment)); + SLANG_ASSERT(elementStride != 0); + } + chunk->writeData(&elementStride, sizeof(elementStride)); + } + break; + + case FossilizedValKind::Struct: + case FossilizedValKind::Tuple: + { + auto recordLayout = (RecordLayoutObj*)layout; + + auto fieldCount = UInt32(recordLayout->fieldCount); + chunk->writeData(&fieldCount, sizeof(fieldCount)); + + for (Index i = 0; i < fieldCount; ++i) + { + auto& field = recordLayout->fields[i]; + auto fieldLayoutChunk = _getOrCreateChunkForLayout(field.layout); + chunk->writeRelativePtr<Fossil::RelativePtrOffset>(fieldLayoutChunk); + + auto fieldOffset = UInt32(field.offset); + chunk->writeData(&fieldOffset, sizeof(fieldOffset)); + + if (i != 0) + { + // Make sure that all but the first field have + // a non-zero offset, to validate that offsets + // are being comptued at all. + // + SLANG_ASSERT(fieldOffset != 0); + } + } + } + break; + } + + return chunk; +} + +bool SerialWriter::LayoutObjKey::operator==(LayoutObjKey const& that) const +{ + if (obj == that.obj) + return true; + + if (!obj || !that.obj) + return false; + + SLANG_ASSERT(obj && that.obj); + + if (obj->kind != that.obj->kind) + return false; + + switch (obj->kind) + { + default: + break; + + case FossilizedValKind::Array: + case FossilizedValKind::Dictionary: + case FossilizedValKind::Optional: + case FossilizedValKind::Ptr: + { + auto thisContainer = (ContainerLayoutObj*)obj; + auto thatContainer = (ContainerLayoutObj*)that.obj; + + LayoutObjKey thisElement = thisContainer->baseLayout; + LayoutObjKey thatElement = thatContainer->baseLayout; + + if (thisElement != thatElement) + return false; + } + break; + + case FossilizedValKind::Tuple: + case FossilizedValKind::Struct: + { + auto thisRecord = (RecordLayoutObj*)obj; + auto thatRecord = (RecordLayoutObj*)that.obj; + + if (thisRecord->fieldCount != thatRecord->fieldCount) + return false; + + auto fieldCount = thisRecord->fieldCount; + for (Index i = 0; i < fieldCount; ++i) + { + auto thisField = thisRecord->fields[i]; + auto thatField = thatRecord->fields[i]; + + if (thisField.offset != thatField.offset) + return false; + + LayoutObjKey thisFieldLayout = thisField.layout; + LayoutObjKey thatFieldLayout = thatField.layout; + + if (thisFieldLayout != thatFieldLayout) + return false; + } + } + break; + } + + return true; +} + +bool SerialWriter::LayoutObjKey::operator!=(LayoutObjKey const& that) const +{ + return !(*this == that); +} + +HashCode64 SerialWriter::LayoutObjKey::getHashCode() const +{ + Hasher hasher; + hashInto(hasher); + return hasher.getResult(); +} + +void SerialWriter::LayoutObjKey::hashInto(Hasher& hasher) const +{ + if (!obj) + { + hasher.hashValue(obj); + return; + } + + hasher.hashValue(obj->kind); + + switch (obj->kind) + { + default: + break; + + case FossilizedValKind::Array: + case FossilizedValKind::Dictionary: + case FossilizedValKind::Optional: + case FossilizedValKind::Ptr: + { + auto container = (ContainerLayoutObj*)obj; + + LayoutObjKey(container->baseLayout).hashInto(hasher); + } + break; + + case FossilizedValKind::Tuple: + case FossilizedValKind::Struct: + { + auto record = (RecordLayoutObj*)obj; + + auto fieldCount = record->fieldCount; + hasher.hashValue(record->fieldCount); + + for (Index i = 0; i < fieldCount; ++i) + { + auto& field = record->fields[i]; + hasher.hashValue(field.offset); + LayoutObjKey(field.layout).hashInto(hasher); + } + } + break; + } +} + + +// +// SerialReader +// + +SerialReader::SerialReader(FossilizedValRef valRef) +{ + _state.type = State::Type::Root; + _state.baseValue = valRef; + _state.elementIndex = 0; + _state.elementCount = 1; +} + +SerialReader::~SerialReader() +{ + _flush(); +} + +SerializationMode SerialReader::getMode() +{ + return SerializationMode::Read; +} + +void SerialReader::handleBool(bool& value) +{ + auto valRef = _readValRef(); + value = as<FossilizedBoolVal>(valRef)->getValue(); +} + +void SerialReader::handleInt8(int8_t& value) +{ + auto valRef = _readValRef(); + value = as<FossilizedInt8Val>(valRef)->getValue(); +} + +void SerialReader::handleInt16(int16_t& value) +{ + auto valRef = _readValRef(); + value = as<FossilizedInt16Val>(valRef)->getValue(); +} + +void SerialReader::handleInt32(Int32& value) +{ + auto valRef = _readValRef(); + value = as<FossilizedInt32Val>(valRef)->getValue(); +} + +void SerialReader::handleInt64(Int64& value) +{ + auto valRef = _readValRef(); + value = as<FossilizedInt64Val>(valRef)->getValue(); +} + +void SerialReader::handleUInt8(uint8_t& value) +{ + auto valRef = _readValRef(); + value = as<FossilizedUInt8Val>(valRef)->getValue(); +} + +void SerialReader::handleUInt16(uint16_t& value) +{ + auto valRef = _readValRef(); + value = as<FossilizedUInt16Val>(valRef)->getValue(); +} + +void SerialReader::handleUInt32(UInt32& value) +{ + auto valRef = _readValRef(); + value = as<FossilizedUInt32Val>(valRef)->getValue(); +} + +void SerialReader::handleUInt64(UInt64& value) +{ + auto valRef = _readValRef(); + value = as<FossilizedUInt64Val>(valRef)->getValue(); +} + +void SerialReader::handleFloat32(float& value) +{ + auto valRef = _readValRef(); + value = as<FossilizedFloat32Val>(valRef)->getValue(); +} + +void SerialReader::handleFloat64(double& value) +{ + auto valRef = _readValRef(); + value = as<FossilizedFloat64Val>(valRef)->getValue(); +} + +void SerialReader::handleString(String& value) +{ + auto valRef = _readPotentiallyIndirectValRef(); + if (!valRef) + { + value = String(); + } + else + { + value = as<FossilizedStringObj>(valRef)->getValue(); + } +} + +void SerialReader::beginArray() +{ + auto valRef = _readPotentiallyIndirectValRef(); + auto arrayRef = as<FossilizedContainerObj>(valRef); + + _pushState(); + + _state.type = State::Type::Array; + _state.baseValue = valRef; + _state.elementIndex = 0; + _state.elementCount = getElementCount(arrayRef); +} + +void SerialReader::endArray() +{ + _popState(); +} + +void SerialReader::beginDictionary() +{ + auto valRef = _readPotentiallyIndirectValRef(); + auto dictionaryRef = as<FossilizedContainerObj>(valRef); + + _pushState(); + + _state.type = State::Type::Dictionary; + _state.baseValue = valRef; + _state.elementIndex = 0; + _state.elementCount = getElementCount(dictionaryRef); +} + +void SerialReader::endDictionary() +{ + _popState(); +} + +bool SerialReader::hasElements() +{ + return _state.elementIndex < _state.elementCount; +} + +void SerialReader::beginStruct() +{ + auto valRef = _readValRef(); + auto recordRef = as<FossilizedRecordVal>(valRef); + + _pushState(); + + _state.type = State::Type::Struct; + _state.baseValue = valRef; + _state.elementIndex = 0; + _state.elementCount = getFieldCount(recordRef); +} + +void SerialReader::endStruct() +{ + _popState(); +} + +void SerialReader::beginVariant() +{ + auto valRef = _readPotentiallyIndirectValRef(); + auto variantRef = as<FossilizedVariantObj>(valRef); + + auto contentValRef = getVariantContent(variantRef); + auto contentRecordRef = as<FossilizedRecordVal>(contentValRef); + + _pushState(); + + _state.type = State::Type::Struct; + _state.baseValue = contentValRef; + _state.elementIndex = 0; + _state.elementCount = getFieldCount(contentRecordRef); +} + +void SerialReader::endVariant() +{ + _popState(); +} + +void SerialReader::handleFieldKey(char const* name, Int index) +{ + // For now we are ignoring field keys, and treating + // structs as basically equivalent to tuples. + SLANG_UNUSED(name); + SLANG_UNUSED(index); +} + +void SerialReader::beginTuple() +{ + auto valRef = _readValRef(); + auto recordRef = as<FossilizedRecordVal>(valRef); + + _pushState(); + + _state.type = State::Type::Tuple; + _state.baseValue = valRef; + _state.elementIndex = 0; + _state.elementCount = getFieldCount(recordRef); +} + +void SerialReader::endTuple() +{ + _popState(); +} + +void SerialReader::beginOptional() +{ + auto valRef = _readIndirectValRef(); + auto optionalRef = as<FossilizedOptionalObj>(valRef); + + _pushState(); + + _state.type = State::Type::Optional; + _state.baseValue = valRef; + _state.elementIndex = 0; + _state.elementCount = Count(hasValue(optionalRef)); +} + +void SerialReader::endOptional() +{ + _popState(); +} + +void SerialReader::handleSharedPtr(void*& value, Callback callback, void* userData) +{ + // The fossilized value at our cursor must be a pointer, + // and we can resolve what it is pointing to easily enough. + // + auto valRef = _readValRef(); + auto ptrRef = as<FossilizedPtrVal>(valRef); + auto targetValRef = getPtrTarget(ptrRef); + + // The logic here largely mirrors what appears in + // `SerialWriter::handleSharedPtr`. + // + // We first check for an explicitly written null pointer. + // If we find one our work is very easy. + // + if (!targetValRef) + { + value = nullptr; + return; + } + + // Now we need to check if we've previously read in + // a reference to the same object. + // + if (auto found = _mapFossilizedObjectPtrToObjectInfo.tryGetValue(targetValRef.getData())) + { + auto objectInfo = *found; + + // We've seen this object before, although it + // is still possible that we are in the middle + // of reading it as part of an invocation + // of `handleSharedPtr()` further up the call + // stack. + // + // If a non-nullpointer value has already been + // written into the `objectInfo`, then that means + // the callback that was run for the prior (or + // in-flight) read operation has already allocated + // or found an object and written it out. + // In that case we will trust the value. + // + if (objectInfo->resurrectedObjectPtr == nullptr) + { + // It is possible that the pointer is null because + // the callback that was invoked explicitly *chose* + // to yield a null pointer (perhaps the application + // is choosing not to deserialize some optional + // piece of state). + // + // However, if there is still a callback in-flight + // to read this object, and the pointer is null, + // then we have reached a circular reference, + // and need to signal an error. + // + if (objectInfo->state == ObjectState::ReadingInProgress) + { + SLANG_UNEXPECTED("circularity detected in fossil deserialization"); + } + } + value = objectInfo->resurrectedObjectPtr; + return; + } + + // At this point we are reading a reference to an + // object index that has not yet been read at all. + // + auto objectInfo = RefPtr(new ObjectInfo()); + _mapFossilizedObjectPtrToObjectInfo.add(targetValRef.getData(), objectInfo); + + objectInfo->fossilizedObjectRef = targetValRef; + + // We cannot return from this function until we have + // stored a pointer into `value`, to represent the + // deserialized object. + // + // Thus we will set ourselves up to start reading + // from the relevant object definition, and invoke + // the callback that was passed in. + // + // Calling into user-defined serialization logic from + // within this function creates the possibility of + // unbounded/infinite recursion, so it is vital that + // the user is properly using `deferSerializeObjectContents()` + // to delay reading data that isn't immediately + // necessary. + // + // We will still set the `objectInfo.state` to reflect + // this in-flight operation so that we can detect + // a cirularity if one occurs at runtime. + // + objectInfo->state = ObjectState::ReadingInProgress; + + // We save/restore the current cursor around + // the callback, because we need to be able + // to return to the current state to continue + // reading whatever comes after the pointer + // we were invoked to read. + // + _pushState(); + _state.type = State::Type::Object; + _state.baseValue = objectInfo->fossilizedObjectRef; + _state.elementIndex = 0; + _state.elementCount = 1; + + // Note that we are passing the address of `objectInfo.ptr`, + // and `objectInfo` is a reference to an element of the + // `_objects` array. Thus whenever the `callback` stores + // a pointer into that output parameter, the value it writes + // will automatically be visible to any subsequent calls + // to `handleSharedPtr()`, even if they occur before + // `callback` returns. + // + // Thus a "true" circularity can only occur if the callback + // recursively reads a reference to the same object again + // *before* it allocates the in-memory representation of + // that objects and stores a pointer to it into the output + // parameter. + // + callback(&objectInfo->resurrectedObjectPtr, userData); + + _popState(); + + objectInfo->state = ObjectState::ReadingComplete; + + value = objectInfo->resurrectedObjectPtr; +} + +void SerialReader::handleUniquePtr(void*& value, Callback callback, void* userData) +{ + // We treat all pointers as shared pointers, because there isn't really + // an optimized representation we would want to use for the unique case. + // + handleSharedPtr(value, callback, userData); +} + +void SerialReader::handleDeferredObjectContents(void* valuePtr, Callback callback, void* userData) +{ + // Unlike the case in `SerialWriter::handleDeferredObjectContents()`, + // we very much *do* want to delay invoking the callback until later. + // + // There is a kind of symmetry going on, where the writer delays the + // callback passed to `handleSharedPtr()`, but *not* the callback + // passed to `handleDeferredObjectContents()`, while the reader + // does the opposite: immediately calls the callback in `handleSharedPtr()` + // but delays calling it here. + + // We make sure to save the current `_cursor` value along with + // the arguments that will be passed into the callback, so that + // we can restore the reader to this state before invoking + // the callbak in `_flush()`. + + DeferredAction deferredAction; + deferredAction.savedState = _state; + deferredAction.resurrectedObjectPtr = valuePtr; + deferredAction.callback = callback; + deferredAction.userData = userData; + + _deferredActions.add(deferredAction); +} + +void SerialReader::_flush() +{ + // We need to flush any actions that were deferred + // and are still pending. + // + while (_deferredActions.getCount() != 0) + { + // TODO: For simplicity we are using the `_deferredActions` + // array as a stack (LIFO), but it would be good to + // check whether there is a menaingful difference in how + // large the array would need to grow for a FIFO vs. LIFO, + // and pick the better option. + // + auto deferredAction = _deferredActions.getLast(); + _deferredActions.removeLast(); + + _state = deferredAction.savedState; + deferredAction.callback(deferredAction.resurrectedObjectPtr, deferredAction.userData); + } +} + +FossilizedValRef SerialReader::_readValRef() +{ + switch (_state.type) + { + case State::Type::Root: + case State::Type::Object: + SLANG_ASSERT(_state.elementCount == 1); + SLANG_ASSERT(_state.elementIndex == 0); + _state.elementIndex++; + return _state.baseValue; + + case State::Type::Struct: + case State::Type::Tuple: + { + SLANG_ASSERT(_state.elementIndex < _state.elementCount); + auto index = _state.elementIndex++; + + auto recordRef = as<FossilizedRecordVal>(_state.baseValue); + return getField(recordRef, index); + } + + case State::Type::Optional: + { + SLANG_ASSERT(_state.elementCount == 1); + SLANG_ASSERT(_state.elementIndex == 0); + + auto optionalRef = as<FossilizedOptionalObj>(_state.baseValue); + return getValue(optionalRef); + } + + case State::Type::Array: + case State::Type::Dictionary: + { + SLANG_ASSERT(_state.elementIndex < _state.elementCount); + auto index = _state.elementIndex++; + + auto containerRef = as<FossilizedContainerObj>(_state.baseValue); + return getElement(containerRef, index); + } + + default: + SLANG_UNEXPECTED("unhandled case"); + break; + } +} + +FossilizedValRef SerialReader::_readIndirectValRef() +{ + auto ptrValRef = _readValRef(); + auto ptrRef = as<FossilizedPtrVal>(ptrValRef); + + auto valRef = getPtrTarget(ptrRef); + return valRef; +} + + +FossilizedValRef SerialReader::_readPotentiallyIndirectValRef() +{ + auto valRef = _readValRef(); + if (auto ptrRef = as<FossilizedPtrVal>(valRef)) + { + return getPtrTarget(ptrRef); + } + return valRef; +} + +void SerialReader::_pushState() +{ + _stack.add(_state); +} + +void SerialReader::_popState() +{ + SLANG_ASSERT(_stack.getCount() != 0); + _state = _stack.getLast(); + _stack.removeLast(); +} + +} // namespace Fossil +} // namespace Slang diff --git a/source/slang/slang-serialize-fossil.h b/source/slang/slang-serialize-fossil.h new file mode 100644 index 000000000..0393c5784 --- /dev/null +++ b/source/slang/slang-serialize-fossil.h @@ -0,0 +1,741 @@ +// slang-serialize-fossil.h +#ifndef SLANG_SERIALIZE_FOSSIL_H +#define SLANG_SERIALIZE_FOSSIL_H + +// +// This file provides implementations of `ISerializerImpl` that +// serialize hierarchical data in the "memory-mappable" binary +// format defined in `slang-fossil.h`. +// + +#include "../core/slang-blob-builder.h" +#include "../core/slang-internally-linked-list.h" +#include "../core/slang-io.h" +#include "../core/slang-memory-arena.h" +#include "../core/slang-relative-ptr.h" +#include "slang-fossil.h" +#include "slang-serialize.h" + +namespace Slang +{ +namespace Fossil +{ + +/// Serializer implementation for writing objects to a fossil-format blob. +struct SerialWriter : ISerializerImpl +{ +public: + SerialWriter(ChunkBuilder* chunk); + SerialWriter(BlobBuilder& blobBuilder); + + ~SerialWriter(); + +private: + SerialWriter() = delete; + + void _initialize(ChunkBuilder* chunk); + + // The fossil format stores layout information, but that + // information is kept separate from the values themselves. + // + // The nature of the `ISerializer` interface means that we + // can only discover the layout as it is first being written, + // so we need an intermediate representation of layouts + // that we compute during the serialization process, before + // we can write those layouts out as their own bytes. + // + // Two related issues make this task a little intricate: + // + // * We don't want to redundantly serialize many copies of + // the same layout (since the whole point of keeping the + // layout information separate from the content is to + // save on space), and ideally we don't want to *create* + // a large number of intermediate layouts that will end + // up getting deduplicated out of existence. + // + // * If the same C++ type is getting serialized multiple times + // (e.g., in a loop serializing an array) we both want to + // re-use the layout from the first element for subsequent + // elements *and* we want to handle the case where parts of + // the layout get expanded on subsequent iterations (e.g., + // the first element in an array might have contained a null + // pointer, so there is no layout info for what it points to, + // but a later element might fill in that gap). + // + // The `_mergeLayouts()` operation is central to how these + // issues are handled, allowing code to attach new information + // to an existing layout as it goes. + + /// Representation of a layout for data that has been serialized. + class LayoutObj + { + public: + LayoutObj(FossilizedValKind kind, Size size = 0, Size alignment = 1) + : kind(kind), size(size), alignment(alignment) + { + } + + virtual ~LayoutObj() {} + + FossilizedValKind getKind() const { return kind; } + + Size getSize() const { return size; } + Size getAlignment() const { return alignment; } + + FossilizedValKind kind; + Size size = 0; + Size alignment = 1; + + /// If this layout is getting serialized out, then this + /// is a pointer to the chunk that will store the `FossilizedValLayout`. + /// + ChunkBuilder* chunk = nullptr; + }; + + /// Create a layout of the given `kind`. + /// + /// If `kind` is one of the simple layout kinds, then this will + /// return a singleton layout. + /// + LayoutObj* _createLayout(FossilizedValKind kind); + + LayoutObj* _createSimpleLayout(FossilizedValKind kind); + Dictionary<FossilizedValKind, LayoutObj*> _simpleLayouts; + + // Rather than try to do detailed memory management for + // layouts, we simply allocate them from an arena. + + MemoryArena _arena; + + /// Merge the `dst` layout object with the given `kind`. + /// + /// This more or less ensures that the layout *exists* + /// and has the right kind. + /// + /// If `dst` is null, it will be initialized via `_createLayout`. + /// + /// If `dst` is non-null, it will be checked against `kind`. + /// + LayoutObj* _mergeLayout(LayoutObj*& dst, FossilizedValKind kind); + + /// Merge the `src` layout into the `dst` layout. + /// + /// If `dst` is null, sets it to `src`. + /// + /// If `dst` is non-null, validates that `dst` and + /// `src` have the same kind, and then may recursively + /// merge their contents (e.g., if both are arrays, + /// it will merge the element layouts). + /// + void _mergeLayout(LayoutObj*& dst, LayoutObj* src); + + /// Layout for simple types (integers, strings, etc.) + class SimpleLayoutObj : public LayoutObj + { + public: + SimpleLayoutObj(FossilizedValKind kind, Size size) + : LayoutObj(kind, size, size) + { + } + + SimpleLayoutObj(FossilizedValKind kind) + : LayoutObj(kind) + { + } + }; + + /// Layouts for objects that have one conceptual type parameter. + /// + /// The obvious cases include pointers, arrays, and optionals. + /// + /// This is also used for dictionaries (the element type is + /// a pair). + /// + /// This is also used for variants (the element type is the type + /// of data that a *particular* variant used, whether or not + /// it matches any others). + /// + class ContainerLayoutObj : public LayoutObj + { + public: + ContainerLayoutObj( + FossilizedValKind kind, + LayoutObj* baseLayout, + Size size = 0, + Size alignment = 1) + : LayoutObj(kind, size, alignment), baseLayout(baseLayout) + { + } + + LayoutObj* baseLayout = nullptr; + }; + + /// Layouts for tuples and structs. + /// + class RecordLayoutObj : public LayoutObj + { + public: + RecordLayoutObj(FossilizedValKind kind) + : LayoutObj(kind) + { + } + + struct FieldInfo + { + LayoutObj* layout = nullptr; + Size offset = 0; + }; + + Count fieldCount = 0; + Count fieldCapacity = 0; + FieldInfo* fields = nullptr; + }; + + /// Get or add a field to the given `recordLayout` at the given `index`. + /// + /// If there is not already a field at `index`, then `index` must be + /// equal to the number of existing fields. + /// + RecordLayoutObj::FieldInfo& _getOrAddField(RecordLayoutObj* layout, Index index); + + // The serialized representation only references layouts as part of + // its encoding of variants, with each variant having a prefix field + // that is a relative pointer to its serialized layout. + // + // Because we want to deduplicate layouts, we keep track of all of + // the variant values we have serialized (each of which should be its + // own chunk), and use that array to come back later and write out + // their final layouts (after deduplication). + + struct VariantInfo + { + LayoutObj* layout = nullptr; + ChunkBuilder* chunk = nullptr; + }; + List<VariantInfo> _variants; + + /// Create a chunk to represent `layout`, or return a pre-existing one. + ChunkBuilder* _getOrCreateChunkForLayout(LayoutObj* layout); + + /// Key for deduplication of `LayoutObj`s. + struct LayoutObjKey + { + LayoutObjKey() {} + + LayoutObjKey(LayoutObj* obj) + : obj(obj) + { + } + + LayoutObj* obj = nullptr; + + bool operator==(LayoutObjKey const& that) const; + bool operator!=(LayoutObjKey const& that) const; + + HashCode64 getHashCode() const; + void hashInto(Hasher& hasher) const; + }; + Dictionary<LayoutObjKey, ChunkBuilder*> _mapLayoutObjToChunk; + + // We also go ahead and deduplicate strings as part of serialization, + // since it is easy to do so. + + Dictionary<String, ChunkBuilder*> _mapStringToChunk; + + // Like almost any implementation of `ISerializer`, we need to track + // information on the objects that have been encountered on the other + // side of pointers, so that we can delay serializing their contents + // until an appropriate time. + + struct FossilizedObjectInfo + { + /// Pointer to the "live" object. + void* liveObjectPtr = nullptr; + + /// Chunk that will store the bytes of the fossilized object. + ChunkBuilder* chunk = nullptr; + + /// Layout for a pointer to the fossilized `chunk`. + LayoutObj* ptrLayout = nullptr; + + /// Callback information used by the ISerializer interface. + Callback callback = nullptr; + void* userData = nullptr; + }; + + List<FossilizedObjectInfo*> _fossilizedObjects; + Dictionary<void*, FossilizedObjectInfo*> _mapLiveObjectPtrToFossilizedObject; + Index _writtenObjectDefinitionCount = 0; + + /// Flush all pending operations. + /// + /// This function ensures that all of the to-be-writen objects have + /// been written out, and that all of the variants that need a pointer + /// to a serialized layout get one. + /// + void _flush(); + + // + // As the user makes various begin/end calls on this `SerialWriter`, + // we need to push/pop state information so that we don't lose it. + // + + struct State + { + /// The layout for the value being composed. + LayoutObj* layout = nullptr; + + /// The number of elements/fields or other sub-values written so var. + Count elementCount = 0; + + /// The chunk that holds the data for the value. + /// + /// Can be null if nothing has been written yet, in which + /// case it may be allocated on teh first write. + /// + ChunkBuilder* chunk = nullptr; + + State() {} + + State(LayoutObj* layout, ChunkBuilder* chunk = nullptr) + : layout(layout), chunk(chunk) + { + } + }; + + /// The current state. + State _state; + + /// Stack of suspended states. + List<State> _stack; + + /// The underlying blob builder that we are writing to. + BlobBuilder* _blobBuilder = nullptr; + + // + // Depending on the kind of value being written, it may + // require a different representation. The `Val + + /// Represents a conceptual value to be written. + /// + /// Depending on the kind of value being written, it may + /// require a different representation. The `ValInfo` type + /// abstracts over these differences. + /// + /// Simple values that just consist of bytes can use the + /// `RawData` case. + /// + /// Values that are encoded as a relative pointer use the + /// `RelativePtr` case (unsurprisingly). + /// + /// The `ContentsOfChunk` case is used when the conceptual + /// value is some kind of aggregate that is stored inline + /// rather than indirectly. + /// + struct ValInfo + { + public: + enum class Kind + { + RawData, + RelativePtr, + ContentsOfChunk, + }; + + static ValInfo rawData(void const* data, Size size, Size alignment); + static ValInfo relativePtrTo(ChunkBuilder* targetChunk); + static ValInfo contentsOf(ChunkBuilder* chunk); + + Size getAlignment() const; + + Kind kind; + union + { + struct + { + void const* ptr; + Size size; + Size alignment; + } data; + ChunkBuilder* chunk; + }; + + private: + ValInfo() = default; + ValInfo(const ValInfo&) = default; + ValInfo(ValInfo&&) = default; + ValInfo(Kind kind) + : kind(kind) + { + } + }; + + // In order to allow building up layout information as values are + // being written, the process of writing a value is broken into + // two parts: + // + // * First, the code conceptually "reserves" a destination for the + // value it will write, passing in what it knows about the expected + // layout for the value. The reserve operation returns a layout + // to use (which may be a pre-existing one). + // + // * Second, once the value is ready as a `ValInfo`, the code "commits" + // the write and puts actual data in a chunk somewhere. + // + // For simple values these operations occur on after the other in + // the same function. For complex things that need a begin/end pair, + // the reserve usually happens in a `begin*()` or `push*()` function, + // while the commit happens in an `end*()` or `pop*()` function. + + LayoutObj*& _reserveDestinationForWrite(); + LayoutObj* _reserveDestinationForWrite(FossilizedValKind srcKind); + LayoutObj* _reserveDestinationForWrite(LayoutObj* srcLayout); + + void _commitWrite(ValInfo const& val); + + /// Write a value without doing any of the checks that `_commitWrite` does. + /// + /// (Usually this is called because `_commitWrite()` has already been called) + void _writeValueRaw(ValInfo const& val); + + /// Ensure that the current `State` has a non-null chunk that data + /// can be written to. + /// + void _ensureChunkExists(); + + // There are various different categories of values that each + // need slightly different handling, so each gets its own + // operations that the various `ISerializer::begin()/end()` + // functions will delegate to. + // + // The easiest case is simple values that consist of nothing + // but plain data and have a layout that can be fully summarized + // by the kind. + + void _writeSimpleValue(FossilizedValKind kind, void const* data, size_t size, size_t alignment); + + template<typename T> + void _writeSimpleValue(FossilizedValKind kind, T const& value) + { + _writeSimpleValue(kind, &value, sizeof(value), sizeof(value)); + } + + /// Write a null (relative) pointer. + /// + /// Use this case when there is no more refined type information + /// available about what the layout of the pointed-to data *would* + /// be if the pointer were non-null. + /// + void _writeNull(); + + // + // "Inline" values are aggregates like tuple and structs that + // are always stored by-value in their parent. + // + + void _pushInlineValueScope(FossilizedValKind kind); + void _popInlineValueScope(); + + // + // "Indirect" values are those like optionals that are + // stored as a pointer to an (optional) out-of-line value. + // + + void _pushIndirectValueScope(FossilizedValKind kind); + void _popIndirectValueScope(); + + // + // Many cases of values are *potentially* indirect, in that + // they should be stored via pointer indirection *unless* + // their immediate parent is something that already introduced + // an indirection. + // + // A simple example is a string. A string will by default + // be stored as a (relative) pointer to its content. However, + // if there happens to be an *optional* string, then there is + // no need for a second indirection. + // + // Arrays, dictionaries, strings, and variants are all + // potentially-indirect values. + // + // TODO: This is one aspect of the current design that may need + // to be revisited, if it proves to add too much complexity. + // + + void _pushPotentiallyIndirectValueScope(FossilizedValKind kind); + ChunkBuilder* _popPotentiallyIndirectValueScope(); + + /// Determine if a potentially-indirect value of `kind` should be + /// emitted indirectly, in the current state. + /// + bool _shouldEmitWithPointerIndirection(FossilizedValKind kind); + + /// Helper function to share details between `_popIndirectValueScope` + /// and `_popPotentiallyIndirectValueScope`. + /// + ChunkBuilder* _writeKnownIndirectValueSharedLogic(ChunkBuilder* valueChunk); + + // + // Containers like arrays and dictionaries are potentially-indirect + // values where the chunk that stores their content needs to + // be given a prefix with the element count. + // + + void _pushContainerScope(FossilizedValKind kind); + void _popContainerScope(); + + // + // A variant is a potentially-indirect value where the chunk + // that stores its content needs to be given a prefix with + // the layout of the content. + // + + void _pushVariantScope(); + void _popVariantScope(); + + // + // All of the above operations ultimately bottleneck through + // `_pushState()`/`_popState()`. + // + + void _pushState(LayoutObj* layout); + void _popState(); + +private: + // + // The following declarations are the requirements + // of the `ISerializerImpl` interface: + // + + virtual SerializationMode getMode() override; + + virtual void handleBool(bool& value) override; + + virtual void handleInt8(int8_t& value) override; + virtual void handleInt16(int16_t& value) override; + virtual void handleInt32(Int32& value) override; + virtual void handleInt64(Int64& value) override; + + virtual void handleUInt8(uint8_t& value) override; + virtual void handleUInt16(uint16_t& value) override; + virtual void handleUInt32(UInt32& value) override; + virtual void handleUInt64(UInt64& value) override; + + virtual void handleFloat32(float& value) override; + virtual void handleFloat64(double& value) override; + + virtual void handleString(String& value) override; + + virtual void beginArray() override; + virtual void endArray() override; + + virtual void beginOptional() override; + virtual void endOptional() override; + + virtual void beginDictionary() override; + virtual void endDictionary() override; + + virtual bool hasElements() override; + + virtual void beginTuple() override; + virtual void endTuple() override; + + virtual void beginStruct() override; + virtual void endStruct() override; + + virtual void beginVariant() override; + virtual void endVariant() override; + + virtual void handleFieldKey(char const* name, Int index) override; + + virtual void handleSharedPtr(void*& value, Callback callback, void* userData) override; + virtual void handleUniquePtr(void*& value, Callback callback, void* userData) override; + + virtual void handleDeferredObjectContents(void* valuePtr, Callback callback, void* userData) + override; +}; + +/// Serializer implementation for reading objects from a fossil-format blob. +struct SerialReader : ISerializerImpl +{ +public: + SerialReader(FossilizedValRef valRef); + ~SerialReader(); + +private: + /// A state that the reader can be in. + struct State + { + /// Type of state; related to the kind of value being read from. + /// + enum class Type + { + Root, + Array, + Dictionary, + Optional, + Tuple, + Struct, + Object, + }; + + /// The type of state. + Type type = Type::Root; + + /// The fossilized value (data and layout) that is being read from. + /// + /// Depending on the `type` of state, this might either be the next value + /// that will be read (e.g., for the `Root` case), or it might be + /// a container that is a parent of the next value to be read. + /// + FossilizedValRef baseValue; + + /// Index of next element to read. + /// + /// This is used in the case where `baseValue` is some kind of + /// container or record. + /// + Index elementIndex = 0; + + /// Total number of values that can be read. + /// + /// If `baseValue` is a container, this is the element count. + /// If `baseValue` is a tuple/struct, this is the field count. + /// If `baseValue` is an optional, this is either zero or one. + /// If this state is a singleton case like `Root`, will be one. + /// + Count elementCount = 0; + }; + + /// The current state. + State _state; + + /// Stack of saved states. + List<State> _stack; + + void _pushState(); + void _popState(); + + // + // Like other `ISerializerImpl`s for reading, we track objects + // that are in the process of being read in, to avoid possible + // unbounded recursion (and detect circularities when they + // occur). + // + + enum class ObjectState + { + Unread, + ReadingInProgress, + ReadingComplete, + }; + struct ObjectInfo : public RefObject + { + ObjectState state = ObjectState::Unread; + + void* resurrectedObjectPtr = nullptr; + FossilizedValRef fossilizedObjectRef; + }; + Dictionary<void*, RefPtr<ObjectInfo>> _mapFossilizedObjectPtrToObjectInfo; + + // + // Again, like other `ISerializerImpl`s for reading, we + // maintain a list of deferred serialization actions that + // need to be performed to finish reading the state of + // in-memory objects. + // + + struct DeferredAction + { + void* resurrectedObjectPtr; + + State savedState; + + Callback callback; + void* userData; + }; + List<DeferredAction> _deferredActions; + + /// Execute all deferred actions that are still pending. + void _flush(); + + /// Read a simple/inline value. + /// + /// This is the case for scalars, tuples, and structs. + /// + FossilizedValRef _readValRef(); + + /// Read an indirect value. + /// + /// This is the case for things like optionals, that are + /// always encoded as a pointer. + /// + FossilizedValRef _readIndirectValRef(); + + /// Read a potentially-indirect value. + /// + /// If the value that gets read is a pointer, then this + /// function will return a reference to whatever it points to. + /// + /// Otherwise, this will return a reference to the value itself. + /// + FossilizedValRef _readPotentiallyIndirectValRef(); + +private: + // + // The following declarations are the requirements + // of the `ISerializerImpl` interface: + // + + virtual SerializationMode getMode() override; + + virtual void handleBool(bool& value) override; + + virtual void handleInt8(int8_t& value) override; + virtual void handleInt16(int16_t& value) override; + virtual void handleInt32(Int32& value) override; + virtual void handleInt64(Int64& value) override; + + virtual void handleUInt8(uint8_t& value) override; + virtual void handleUInt16(uint16_t& value) override; + virtual void handleUInt32(UInt32& value) override; + virtual void handleUInt64(UInt64& value) override; + + virtual void handleFloat32(float& value) override; + virtual void handleFloat64(double& value) override; + + virtual void handleString(String& value) override; + + virtual void beginArray() override; + virtual void endArray() override; + + virtual void beginDictionary() override; + virtual void endDictionary() override; + + virtual bool hasElements() override; + + virtual void beginStruct() override; + virtual void endStruct() override; + + virtual void beginVariant() override; + virtual void endVariant() override; + + virtual void handleFieldKey(char const* name, Int index) override; + + virtual void beginTuple() override; + virtual void endTuple() override; + + virtual void beginOptional() override; + virtual void endOptional() override; + + virtual void handleSharedPtr(void*& value, Callback callback, void* userData) override; + virtual void handleUniquePtr(void*& value, Callback callback, void* userData) override; + + virtual void handleDeferredObjectContents(void* valuePtr, Callback callback, void* userData) + override; +}; + +} // namespace Fossil +} // namespace Slang + +#endif diff --git a/source/slang/slang-serialize-riff.cpp b/source/slang/slang-serialize-riff.cpp index 01b39e825..469803c2a 100644 --- a/source/slang/slang-serialize-riff.cpp +++ b/source/slang/slang-serialize-riff.cpp @@ -174,6 +174,21 @@ void RIFFSerialWriter::beginStruct() _cursor.beginListChunk(RIFFSerial::kStructFourCC); } +void RIFFSerialWriter::endStruct() +{ + _cursor.endChunk(); +} + +void RIFFSerialWriter::beginVariant() +{ + beginStruct(); +} + +void RIFFSerialWriter::endVariant() +{ + endStruct(); +} + void RIFFSerialWriter::handleFieldKey(char const* name, Int index) { // For now we are ignoring field keys, and treating @@ -182,11 +197,6 @@ void RIFFSerialWriter::handleFieldKey(char const* name, Int index) SLANG_UNUSED(index); } -void RIFFSerialWriter::endStruct() -{ - _cursor.endChunk(); -} - void RIFFSerialWriter::beginTuple() { _cursor.beginListChunk(RIFFSerial::kTupleFourCC); @@ -506,6 +516,21 @@ void RIFFSerialReader::beginStruct() _beginListChunk(RIFFSerial::kStructFourCC); } +void RIFFSerialReader::endStruct() +{ + _endListChunk(); +} + +void RIFFSerialReader::beginVariant() +{ + beginStruct(); +} + +void RIFFSerialReader::endVariant() +{ + endStruct(); +} + void RIFFSerialReader::handleFieldKey(char const* name, Int index) { // For now we are ignoring field keys, and treating @@ -514,11 +539,6 @@ void RIFFSerialReader::handleFieldKey(char const* name, Int index) SLANG_UNUSED(index); } -void RIFFSerialReader::endStruct() -{ - _endListChunk(); -} - void RIFFSerialReader::beginTuple() { _beginListChunk(RIFFSerial::kTupleFourCC); diff --git a/source/slang/slang-serialize-riff.h b/source/slang/slang-serialize-riff.h index a464a5ded..87f83f3f0 100644 --- a/source/slang/slang-serialize-riff.h +++ b/source/slang/slang-serialize-riff.h @@ -224,9 +224,13 @@ private: virtual bool hasElements() override; virtual void beginStruct() override; - virtual void handleFieldKey(char const* name, Int index) override; virtual void endStruct() override; + virtual void beginVariant() override; + virtual void endVariant() override; + + virtual void handleFieldKey(char const* name, Int index) override; + virtual void beginTuple() override; virtual void endTuple() override; @@ -410,9 +414,13 @@ private: virtual bool hasElements() override; virtual void beginStruct() override; - virtual void handleFieldKey(char const* name, Int index) override; virtual void endStruct() override; + virtual void beginVariant() override; + virtual void endVariant() override; + + virtual void handleFieldKey(char const* name, Int index) override; + virtual void beginTuple() override; virtual void endTuple() override; diff --git a/source/slang/slang-serialize.h b/source/slang/slang-serialize.h index d76ab8338..b962ee2b7 100644 --- a/source/slang/slang-serialize.h +++ b/source/slang/slang-serialize.h @@ -335,6 +335,30 @@ struct ISerializerImpl /// End serializing a struct value. virtual void endStruct() = 0; + /// Begin serializing a variant value. + /// + /// A variant should be used to serialize any type + /// that behaves like a "tagged union," where different + /// instances may have different sequences of members, + /// of different types. + /// + /// User code reading from a variant must be able to + /// use the members read so far to determine what + /// members it should read next (e.g., by serializing + /// a tag enumerant first, followed by the tag-dependent + /// members). + /// + /// A variant is otherwise like a struct. Some serializer + /// implementations may treat variants just like structs, + /// while others may rely on any type serialized as a + /// struct always including the same members in the same + /// order. + /// + virtual void beginVariant() = 0; + + /// End serializing a variant value. + virtual void endVariant() = 0; + /// Set the key for the next struct field to be serialized. /// /// If no name is available for the field, `name` may be `nullptr`. @@ -623,6 +647,21 @@ private: Serializer _serializer; }; +struct ScopedSerializerVariant +{ +public: + ScopedSerializerVariant(Serializer const& serializer) + : _serializer(serializer) + { + serializer->beginVariant(); + } + + ~ScopedSerializerVariant() { _serializer->endVariant(); } + +private: + Serializer _serializer; +}; + struct ScopedSerializerTuple { public: @@ -667,8 +706,8 @@ private: #define SLANG_SCOPED_SERIALIZER_STRUCT(SERIALIZER) \ ::Slang::ScopedSerializerStruct SLANG_CONCAT(_scopedSerializerStruct, __LINE__)(SERIALIZER) -#define SLANG_SCOPED_SERIALIZER_TAGGED_UNION(SERIALIZER) \ - ::Slang::ScopedSerializerStruct SLANG_CONCAT(_scopedSerializerStruct, __LINE__)(SERIALIZER) +#define SLANG_SCOPED_SERIALIZER_VARIANT(SERIALIZER) \ + ::Slang::ScopedSerializerVariant SLANG_CONCAT(_scopedSerializerVariant, __LINE__)(SERIALIZER) #define SLANG_SCOPED_SERIALIZER_TUPLE(SERIALIZER) \ ::Slang::ScopedSerializerTuple SLANG_CONCAT(_scopedSerializerTuple, __LINE__)(SERIALIZER) |
