diff options
Diffstat (limited to 'source')
| -rw-r--r-- | source/core/slang-blob-builder.cpp | 473 | ||||
| -rw-r--r-- | source/core/slang-blob-builder.h | 352 | ||||
| -rw-r--r-- | source/core/slang-internally-linked-list.cpp | 2 | ||||
| -rw-r--r-- | source/core/slang-internally-linked-list.h | 126 | ||||
| -rw-r--r-- | source/core/slang-memory-arena.h | 11 | ||||
| -rw-r--r-- | source/core/slang-relative-ptr.cpp | 2 | ||||
| -rw-r--r-- | source/core/slang-relative-ptr.h | 97 | ||||
| -rw-r--r-- | source/core/slang-riff.h | 68 | ||||
| -rw-r--r-- | source/slang/slang-fossil.cpp | 187 | ||||
| -rw-r--r-- | source/slang/slang-fossil.h | 488 | ||||
| -rw-r--r-- | source/slang/slang-serialize-ast.cpp | 48 | ||||
| -rw-r--r-- | source/slang/slang-serialize-fossil.cpp | 1636 | ||||
| -rw-r--r-- | source/slang/slang-serialize-fossil.h | 741 | ||||
| -rw-r--r-- | source/slang/slang-serialize-riff.cpp | 40 | ||||
| -rw-r--r-- | source/slang/slang-serialize-riff.h | 12 | ||||
| -rw-r--r-- | source/slang/slang-serialize.h | 43 |
16 files changed, 4231 insertions, 95 deletions
diff --git a/source/core/slang-blob-builder.cpp b/source/core/slang-blob-builder.cpp new file mode 100644 index 000000000..10dd4003e --- /dev/null +++ b/source/core/slang-blob-builder.cpp @@ -0,0 +1,473 @@ +// slang-blob-builder.cpp +#include "slang-blob-builder.h" + +namespace Slang +{ + +// +// BlobBuilder +// + +BlobBuilder::BlobBuilder() + : _arena(4096) +{ +} + +void BlobBuilder::writeTo(Stream* stream) +{ + // First we scan through all the chunks to set their + // absolute offsets, which enables us to compute the + // correct values for relative pointers when we write + // them out. + // + SLANG_MAYBE_UNUSED + Size sizeComputed = _calcSizeAndSetCachedChunkOffsets(); + + // Now we can scan through the chunks again and write + // the bytes of each of their shards. + // + SLANG_MAYBE_UNUSED + Size sizeWritten = _writeChunksTo(stream); + + SLANG_ASSERT(sizeComputed == sizeWritten); +} + +Size BlobBuilder::_calcSizeAndSetCachedChunkOffsets() +{ + Size totalSize = 0; + for (auto chunk : _chunks) + { + auto chunkPrefixSize = chunk->getPrefixSize(); + auto chunkContentSize = chunk->getContentSize(); + auto chunkAlignment = chunk->getAlignment(); + + // We add the size of the chunk prefix (if any) *before* + // aligning the current offset. Doing it this way + // means that sometimes the prefix can fit "for free" + // in space that would otherwise be padding. + // + // For example, if the current `totalSize` is 4, the + // `chunkAlignment` is 16, and the `chunkPrefixSize` is 8, + // then the sequence will be: + // + // * Add `totalSize += chunkPrefixSize`, resulting in a `totalSize` + // of 12. + // + // * Round the `totalSize` up to the `chunkAlignment`, to compute + // a `chunkOffset` of 16. + // + // The result is that the chunk's content starts at offset 16, + // and the prefix can occupy the 8 bytes before that (so the + // prefix is at offset 8). + // + // In the best case this approach can save a few bytes here or + // there when `chunkAlignment` is larger than the `chunkPrefixSize`, + // and in the worst case it does no harm. + + totalSize += chunkPrefixSize; + auto chunkOffset = roundUpToAlignment(totalSize, chunkAlignment); + + chunk->_setCachedOffset(chunkOffset); + + totalSize = chunkOffset + chunkContentSize; + } + return totalSize; +} + +Size BlobBuilder::_writeChunksTo(Stream* stream) +{ + Size totalSize = 0; + for (auto chunk : _chunks) + { + auto chunkPrefixSize = chunk->getPrefixSize(); + auto chunkContentSize = chunk->getContentSize(); + auto chunkOffset = chunk->_getCachedOffset(); + + SLANG_ASSERT( + chunkOffset == roundUpToAlignment(totalSize + chunkPrefixSize, chunk->getAlignment())); + + auto paddingSize = chunkOffset - totalSize; + + // The "padding" before the chunk's content also + // includes the space reserved for the chunk's prefix. + // + // We can thus subtract the prefix size from the number + // of pad bytes to write. + + SLANG_ASSERT(paddingSize >= chunkPrefixSize); + paddingSize -= chunkPrefixSize; + + while (paddingSize--) + { + Byte padding = 0; + stream->write(&padding, sizeof(padding)); + } + + // The `ChunkBuilder::_writeTo()` call will write the + // prefix *and* the chunk content, so it is appropriate + // to call it here even when the total number of bytes + // written to the stream is not equal to the `chunkOffset` + // (because in that case the total bytes written so far + // should be `chunkOffset - chunkPrefixSize`). + // + chunk->_writeTo(stream); + + totalSize = chunkOffset + chunkContentSize; + } + + return totalSize; +} + +void BlobBuilder::writeToBlob(ISlangBlob** outBlob) +{ + OwnedMemoryStream stream(FileAccess::Write); + writeTo(&stream); + + List<uint8_t> data; + stream.swapContents(data); + + *outBlob = ListBlob::moveCreate(data).detach(); +} + +ChunkBuilder* BlobBuilder::createUnparentedChunk() +{ + auto chunk = new (_arena) ChunkBuilder(this); + return chunk; +} + +void BlobBuilder::addChunk(ChunkBuilder* chunk) +{ + // TODO(tfoley): it would be good to have a way to assert + // that the chunk has not already been added. + + _chunks.add(chunk); +} + +ChunkBuilder* BlobBuilder::addChunk() +{ + auto chunk = new (_arena) ChunkBuilder(this); + _chunks.add(chunk); + return chunk; +} + +ChunkBuilder* BlobBuilder::addChunkAfter(ChunkBuilder* existingChunk) +{ + auto newChunk = new (_arena) ChunkBuilder(this); + _chunks.insertAfter(existingChunk, newChunk); + return newChunk; +} + +// +// ChunkBuilder +// + +void ChunkBuilder::setAlignmentToAtLeast(Size alignment) +{ + SLANG_ASSERT(alignment > 0); + SLANG_ASSERT(isPowerOfTwo(alignment)); + + _contentAlignment = std::max(_contentAlignment, alignment); +} + +void ChunkBuilder::writePaddingToAlignTo(Size alignment) +{ + setAlignmentToAtLeast(alignment); + + auto alignedSize = roundUpToAlignment(_contentSize, alignment); + + auto requiredPaddingSize = alignedSize - _contentSize; + + while (requiredPaddingSize) + { + Byte padByte = 0; + writeData(&padByte, sizeof(padByte)); + requiredPaddingSize -= sizeof(padByte); + } +} + +ChunkBuilder::ChunkBuilder(BlobBuilder* parentBlob) + : _parentBlob(parentBlob) +{ +} + +Size ChunkBuilder::getContentSize() const +{ + return _contentSize; +} + +Size ChunkBuilder::getPrefixSize() const +{ + if (!_prefixShard) + return 0; + + return _prefixShard->getSize(); +} + +Size ChunkBuilder::getAlignment() const +{ + return _contentAlignment; +} + + +void ChunkBuilder::writeData(void const* data, Size size) +{ + // Adding no data should be a no-op. + // + if (size == 0) + return; + + + // The most interesting implementation detail here + // is that we will try to detect cases where we + // can re-use an existing `ShardBuilder` by adding the data + // to the end of that shard's allocation. + // + // This is only possible because of the way that + // we are using a single `MemoryArena` to allocate + // everything, which makes it possible that the + // next address the arena would return for an allocation + // of `size` bytes is the same as the ending address + // of the payload for the last shard of this chunk. + // + auto& arena = getParentBlob()->_getArena(); + + // We start by checking if this chunk already has + // a last shard that we could consider appending to. + // + auto lastShard = _childShards.getLast(); + if (lastShard && lastShard->_kind == ShardBuilder::Kind::Data) + { + // If there is a last shard, then we can compute + // the end address of its payload, and see if + // it is the same as the cursor of the arena + // we are allocating from. + // + auto payload = lastShard->_data.ptr; + auto payloadSize = lastShard->_size; + auto payloadEnd = (Byte*)payload + payloadSize; + if (payloadEnd == arena.getCursor()) + { + // Now that we've confirmed that the shard's + // payload ends at an address the arena could + // conceivably allocate from, we need to ask + // the arena to allocate `size` bytes from + // the current block it is using, and see if + // doing so succeeds. + // + if (arena.allocateCurrentUnaligned(size)) + { + // At this point, we've confirmed that we + // are in our special case, and the relevant + // bytes have been allocated from the arena. + // + // Now we can simply write the new data at + // what used to be the end address for the + // shard's payload, and adjust its state + // to account for the new allocation. + // + ::memcpy(payloadEnd, data, size); + + lastShard->_size = payloadSize + size; + _contentSize += size; + return; + } + } + } + + // If the special case above doesn't apply, we simply + // allocate a new shard to hold the data that was + // passed in. + // + auto shard = _createDataShard(data, size); + _childShards.add(shard); + _contentSize += size; +} + +void ChunkBuilder::addContentsOf(ChunkBuilder* otherChunk) +{ + auto otherPrefixShard = otherChunk->_prefixShard; + auto otherChunkSize = otherChunk->getContentSize(); + auto otherChunkAlignment = otherChunk->getAlignment(); + auto otherChunkShards = otherChunk->_childShards; + + otherChunk->_prefixShard = nullptr; + otherChunk->_contentSize = 0; + otherChunk->_contentAlignment = 1; + otherChunk->_childShards = InternallyLinkedList<ShardBuilder>(); + + if (otherPrefixShard) + { + // If the other chunk included a prefix, then + // it only makes sense to append it in the case + // where *this* chunk is completely empty. + + SLANG_ASSERT(!_prefixShard); + SLANG_ASSERT(!_childShards.getFirst()); + _prefixShard = otherPrefixShard; + } + + writePaddingToAlignTo(otherChunkAlignment); + _childShards.append(otherChunkShards); + _contentSize += otherChunkSize; +} + +ChunkBuilder* ChunkBuilder::addChunkAfter() +{ + return getParentBlob()->addChunkAfter(this); +} + +void ChunkBuilder::_writeRelativePtr(ChunkBuilder* targetChunk, Size ptrSize) +{ + SLANG_ASSERT(ptrSize != 0); + SLANG_ASSERT(ptrSize <= sizeof(UInt64)); + + writePaddingToAlignTo(ptrSize); + + if (!targetChunk) + { + UInt64 value = 0; + writeData(&value, ptrSize); + return; + } + + auto shard = _createRelativePtrShard(targetChunk, ptrSize); + _childShards.add(shard); + _contentSize += ptrSize; +} + +void ChunkBuilder::_addPrefixRelativePtr(ChunkBuilder* targetChunk, Size ptrSize) +{ + SLANG_ASSERT(targetChunk != nullptr); + SLANG_ASSERT(ptrSize != 0); + + SLANG_ASSERT(!_prefixShard); + + setAlignmentToAtLeast(ptrSize); + + auto shard = _createRelativePtrShard(targetChunk, ptrSize); + _prefixShard = shard; +} + +void ChunkBuilder::addPrefixData(void const* data, Size size) +{ + SLANG_ASSERT(data != nullptr); + SLANG_ASSERT(size != 0); + + SLANG_ASSERT(!_prefixShard); + + setAlignmentToAtLeast(size); + + auto shard = _createDataShard(data, size); + _prefixShard = shard; +} + +ShardBuilder* ChunkBuilder::_createDataShard(void const* data, Size size) +{ + auto& arena = getParentBlob()->_getArena(); + auto shard = new (arena) ShardBuilder(ShardBuilder::Kind::Data); + + auto shardData = arena.allocateUnaligned(size); + ::memcpy(shardData, data, size); + + shard->_data.ptr = shardData; + shard->_size = size; + + return shard; +} + +ShardBuilder* ChunkBuilder::_createRelativePtrShard(ChunkBuilder* targetChunk, Size ptrSize) +{ + auto& arena = getParentBlob()->_getArena(); + auto shard = new (arena) ShardBuilder(ShardBuilder::Kind::RelativePtr); + + shard->_relativePtr.targetChunk = targetChunk; + shard->_size = ptrSize; + + return shard; +} + +void ChunkBuilder::_writeTo(Stream* stream) +{ + auto chunkOffset = _getCachedOffset(); + + if (_prefixShard) + { + // Note that the prefix is written *before* the + // starting offset of the chunk's content, so we + // compute the appropriate offset to pass down. + // + auto prefixSize = _prefixShard->getSize(); + auto prefixOffset = chunkOffset - prefixSize; + + _prefixShard->_writeTo(stream, prefixOffset); + } + + auto shardOffset = chunkOffset; + for (auto shard : _childShards) + { + shard->_writeTo(stream, shardOffset); + shardOffset += shard->getSize(); + } + SLANG_ASSERT(shardOffset == chunkOffset + getContentSize()); +} + + +// +// ShardBuilder +// + +ShardBuilder::ShardBuilder(Kind kind) + : _kind(kind) +{ +} + +void ShardBuilder::_writeTo(Stream* stream, Size inSelfOffset) +{ + switch (_kind) + { + case Kind::Data: + { + stream->write(_data.ptr, _size); + } + break; + + case Kind::RelativePtr: + { + auto targetChunk = _relativePtr.targetChunk; + SLANG_ASSERT(targetChunk); + + auto selfOffset = intptr_t(inSelfOffset); + auto targetOffset = intptr_t(targetChunk->_getCachedOffset()); + + intptr_t relativeOffset = targetOffset - selfOffset; + + switch (_size) + { + case sizeof(Int32): + { + auto value = Int32(relativeOffset); + stream->write(&value, sizeof(value)); + } + break; + + case sizeof(Int64): + { + auto value = Int64(relativeOffset); + stream->write(&value, sizeof(value)); + } + break; + + default: + SLANG_UNEXPECTED("unsupported relative pointer size"); + break; + } + } + break; + + default: + SLANG_UNEXPECTED("unknown Fossil::ShardBuilder::Kind"); + break; + } +} + +} // namespace Slang diff --git a/source/core/slang-blob-builder.h b/source/core/slang-blob-builder.h new file mode 100644 index 000000000..0b59ee7ab --- /dev/null +++ b/source/core/slang-blob-builder.h @@ -0,0 +1,352 @@ +// slang-blob-builder.h +#ifndef SLANG_BLOB_BUILDER_H +#define SLANG_BLOB_BUILDER_H + +// This file provides utilities for building "blobs" of data +// where, for purposes, a blob is a contiguous sequence of +// bytes where the interpretation *of* those bytes depends +// only on the bytes themselves, and not other factors like +// the in-memory address of the blob, or the address/contents +// of memory not in the blob. +// +// Superficially, the task seems simple: just maintain a +// dynamically-sized array of bytes and append to it until +// you're done. If that's what you need, you're probably +// better off just using an `OwnedMemoryStream`. +// +// The utilities in this file deal with the case where you +// want to build some kind of offset-based data structure, +// so that parts of the blob will store byte offsets to +// other parts, while also being able to build parts of +// that structure "out of order", so that the final offset +// of a particular piece of data in the blob may not be +// known until everything *before* it has been fully built. + +#include "slang-basic.h" +#include "slang-internally-linked-list.h" +#include "slang-io.h" +#include "slang-memory-arena.h" + +namespace Slang +{ + +inline constexpr bool isPowerOfTwo(Size value) +{ + return value > 0 && (value - 1 & value) == 0; +} + +inline constexpr Size roundUpToAlignment(Size size, Size alignment) +{ + SLANG_ASSERT(isPowerOfTwo(alignment)); + + auto alignmentMask = Size(alignment) - 1; + return (size + alignmentMask) & ~alignmentMask; +} + +class ShardBuilder; +class ChunkBuilder; +struct BlobBuilder; + + +/// A utility type for composing a binary blob. +/// +/// A blob builder allows a blob to be composed as a sequence of discrete +/// chunks, allowing chunks to be added and written to in any order. +/// +/// Chunks can contain (relative) pointers to one another, with the correct +/// relative offsets being computed as part of writing the entire blob out. +/// +struct BlobBuilder +{ +public: + /// Construct an empty blob builder. + BlobBuilder(); + + /// Write the contents of the blob to the given `stream`. + void writeTo(Stream* stream); + + /// Create a copy of the contents of the blob and assign to `outBlob`. + void writeToBlob(ISlangBlob** outBlob); + + /// Add a new empty chunk to the end of the blob. + ChunkBuilder* addChunk(); + + /// Add a new empty chunk after the given `chunk`. + ChunkBuilder* addChunkAfter(ChunkBuilder* chunk); + + /// Create a chunk that is not initially part of the blob. + /// + /// The contents of the returned chunk will only become + /// part of the full blob if `addChunk()` is called later, + /// *or* if the contents of the new chunk are moved into + /// another chunk that gets added. + /// + ChunkBuilder* createUnparentedChunk(); + + /// Add a chunk to the blob that was initially not part of the blob. + void addChunk(ChunkBuilder* chunk); + +private: + InternallyLinkedList<ChunkBuilder> _chunks; + MemoryArena _arena; + + friend class ChunkBuilder; + friend class ShardBuilder; + MemoryArena& _getArena() { return _arena; } + + Size _calcSizeAndSetCachedChunkOffsets(); + Size _writeChunksTo(Stream* stream); +}; + +/// A chunk is a logically contiguous unit, such that data can +/// only ever be appended to it. As a result, offsets that are +/// relative to the start of a chunk are meaningful, and can be +/// used to encode relative offsets for pointers. +/// +/// Every `ChunkBuilder` is owned by its parent `BlobBuilder`. +/// A pointer to a `ChunkBuilder` will only be valid during the +/// lifetime of the parent `BlobBuilder`. +/// +/// Conceptually, a `ChunkBuilder` has the following: +/// +/// * A minimum *alignment* in bytes (initially 1) +/// +/// * Zero or more bytes of *content* data (initially empty) +/// +/// * An optional *prefix* consisting of zero or more bytes (initially absent) +/// +/// The content may be a mix of raw data (e.g., added via `writeData()`), +/// relative pointers to other chunks (`writeRelativePtr()`) and padding +/// bytes (`writePaddingToAlignTo()`). +/// +/// The prefix may only be either a single relative pointer, or a +/// single range of raw data bytes. +/// +/// When the parent blob written out as a flat buffer of bytes, the +/// following are guaranteed: +/// +/// * The content of the chunk will be a contiguous range of bytes +/// starting at some offset, and will not overlap any other chunks. +/// +/// * The byte offset where the chunk contents start will be a multiple +/// of the chunk's minimum alignment. +/// +/// * The bytes of the prefix (if any) will immediately precede the +/// bytes of the content. +/// +class ChunkBuilder : public InternallyLinkedList<ChunkBuilder>::Node +{ +public: + /// Get the blob builder that this chunk belongs to. + BlobBuilder* getParentBlob() const { return _parentBlob; } + + /// Get the required alignment of this chunk. + /// + /// The minimum alignment for a chunk starts at 1, + /// and may be increased by operations such as + /// `setAlignmentToAtLeast()` and `writePaddingToAlinTo()`. + /// + Size getAlignment() const; + + /// Potentially increases the required alignment of this chunk. + /// + /// If the alignment of the chunk is less than `alignment`, + /// then it will be increased to `alignment`. + /// + /// The `alignment` passed in must be a power of two. + /// + void setAlignmentToAtLeast(Size alignment); + + /// Get the size in bytes of the content of this chunk. + /// + /// Note that the size is not necessarily a multiple of the + /// alignment of the chunk. + /// + Size getContentSize() const; + + /// Write data into the chunk. + /// + /// The chunk will retain a copy of the data passed in, + /// so the `data` pointer only needs to be valid for + /// the duration of the call. + /// + /// Note that this operation does *not* adjust the + /// alignment of the chunk in any way. + /// + /// The data must only contain types that can be copied + /// bit-for-bit, and that do not depend on addresses + /// in meory. In particular no pointers (absolute or + /// relative) should be written. + /// + void writeData(void const* data, Size size); + + /// Append padding bytes to this chunk until its content size + /// is a multiple of `alignment`. + /// + /// May also increase the alignment of the chunk, as if + /// calling `setAlignmentToAtLeast(alignment)`. + /// + /// The padding bytes will all be zero. + /// + void writePaddingToAlignTo(Size alignment); + + /// Write a relative pointer to the given `targetChunk`. + /// + /// The type parameter `T` is used to determine the size + /// of the relative pointer (should be either 4 or 8 bytes). + /// + /// The bytes that eventually get written will contain + /// the computed offset of `targetChunk` minus the computed + /// offset of the first byte of the relative pointer itself. + /// + /// Acts as is `writePaddingToAlignTo(sizeof(T))` were + /// called immediately before. + /// + template<typename T> + void writeRelativePtr(ChunkBuilder* targetChunk) + { + _writeRelativePtr(targetChunk, sizeof(T)); + } + + /// Append the contents of another chunk to this one. + /// + /// This *moves* all of the contents of `chunk` into `this`. + /// After the operation completes `chunk` will be an empty + /// chunk with one-byte alignment. + /// + /// This operation is useful when accumulating data that + /// needs to be appended to the chunk, but where the correct + /// alignment for that data is not yet known; a use can + /// effectively create a temporary "sub-chunk" and then append + /// it to the main chunk once its correct alignment is known. + /// + void addContentsOf(ChunkBuilder* chunk); + + /// Get the size in bytes of the prefix of this chunk, if any. + /// + /// The prefix will be written so that the *end* offset of the + /// prefix data is the same as the starting offset of the + /// chunk's content. + /// + Size getPrefixSize() const; + + /// Add a prefix to this chunk, consisting of raw data. + /// + /// This chunk must not already have a prefix. + /// + void addPrefixData(void const* data, Size size); + + /// Add a prefix to this chunk, consisting of a relative pointer. + /// + /// This chunk must not already have a prefix. + /// + /// Updates the alignment of the chunk as if making a call to + /// `setAlignmentToAtLeast(sizeof(T))`. + /// + template<typename T> + void addPrefixRelativePtr(ChunkBuilder* targetChunk) + { + _addPrefixRelativePtr(targetChunk, sizeof(T)); + } + + /// Insert a new chunk into the blob, immediately after this chunk. + /// + ChunkBuilder* addChunkAfter(); + +private: + ChunkBuilder() = delete; + ChunkBuilder(ChunkBuilder const&) = delete; + ChunkBuilder(ChunkBuilder&&) = delete; + + friend struct BlobBuilder; + friend class ShardBuilder; + + ChunkBuilder(BlobBuilder* parentBlob); + + Size _contentSize = 0; + Size _contentAlignment = 1; + + InternallyLinkedList<ShardBuilder> _childShards; + BlobBuilder* _parentBlob = nullptr; + + Size _cachedOffset = ~Size(0); + Size _getCachedOffset() { return _cachedOffset; } + void _setCachedOffset(Size offset) { _cachedOffset = offset; } + + void _writeRelativePtr(ChunkBuilder* targetChunk, Size ptrSize); + + ShardBuilder* _createDataShard(void const* data, Size size); + ShardBuilder* _createRelativePtrShard(ChunkBuilder* targetChunk, Size ptrSize); + + ShardBuilder* _prefixShard = nullptr; + + void _writeTo(Stream* stream); + + void _addPrefixRelativePtr(ChunkBuilder* targetChunk, Size ptrSize); +}; + +/// A shard is a unit of contiguously-allocated data that makes +/// up part of a chunk. +/// +/// Shards are *not* meant to be directly manipulated by users; +/// they are an implementation detail of `ChunkBuilder`. +/// +/// Every `ShardBuilder` is owned by its parent `ChunkBuilder`. +/// A pointer to a `ShardBuilder` will only be valid during the +/// lifetime of the parent `ChunkBuilder`. +/// +class ShardBuilder : public InternallyLinkedList<ShardBuilder>::Node +{ +public: + // There are two kinds of shards that may appear in a chunk: + // + // * Shards that hold plain data that will be part of the + // serialized chunk. + // + // * Shards that represent a relative pointer to some chunk, + // which cannot have their exact binary value determined + // until the offsets of chunk/shards have been finalized. + + enum class Kind + { + Data, + RelativePtr, + }; + + Size getSize() const { return _size; } + +private: + ShardBuilder() = delete; + ShardBuilder(ShardBuilder const&) = delete; + ShardBuilder(ShardBuilder&&) = delete; + + friend class ChunkBuilder; + ShardBuilder(Kind kind); + + void _writeTo(Stream* stream, Size selfOffset); + + /// Kind of this shard (data or relative pointer) + Kind _kind = Kind::Data; + + union + { + /// Used when `_kind == Kind::Data` + struct + { + void const* ptr; + } _data; + + /// Used when `_kind == Kind::RelativePtr` + struct + { + ChunkBuilder* targetChunk; + } _relativePtr; + }; + + // Size of this shard in bytes. + Size _size = 0; +}; + +} // namespace Slang + +#endif diff --git a/source/core/slang-internally-linked-list.cpp b/source/core/slang-internally-linked-list.cpp new file mode 100644 index 000000000..27bbd77b0 --- /dev/null +++ b/source/core/slang-internally-linked-list.cpp @@ -0,0 +1,2 @@ +// slang-internally-linked-list.cpp +#include "slang-internally-linked-list.h" diff --git a/source/core/slang-internally-linked-list.h b/source/core/slang-internally-linked-list.h new file mode 100644 index 000000000..168885531 --- /dev/null +++ b/source/core/slang-internally-linked-list.h @@ -0,0 +1,126 @@ +// slang-internally-linked-list.h +#ifndef SLANG_INTERNALLY_LINKED_LIST_H +#define SLANG_INTERNALLY_LINKED_LIST_H + +// This file provides support for the idiom of a linked +// list of values where the "next" pointer is stored in +// the values themselves (thus requiring no additional +// allocation for list nodes, at the price of any given +// value only being able to appear in a single list). + +#include "slang-basic.h" + +namespace Slang +{ + +/// A linked list where the elements are themselves the nodes. +/// +/// The type parameter `T` should be a type that publicly +/// inherits from `InternallyLinkedList<T>::Node`. +/// +template<typename T> +struct InternallyLinkedList +{ +public: + struct Node + { + public: + Node() {} + + private: + friend struct InternallyLinkedList<T>; + T* _next = nullptr; + }; + + struct Iterator + { + public: + Iterator() {} + + Iterator(T* node) + : _node(node) + { + } + + T* operator*() const { return _node; } + + void operator++() { _node = static_cast<Node const*>(_node)->_next; } + + bool operator!=(Iterator const& that) const { return _node != that._node; } + + private: + T* _node = nullptr; + }; + + Iterator begin() { return Iterator(_first); } + + Iterator end() { return Iterator(); } + + T* getFirst() const { return _first; } + + T* getLast() const { return _last; } + + void add(T* element) + { + SLANG_ASSERT(element != nullptr); + if (!_last) + { + SLANG_ASSERT(_first == nullptr); + + _first = element; + _last = element; + } + else + { + SLANG_ASSERT(_first != nullptr); + + _last->_next = element; + _last = element; + } + } + + void insertAfter(T* existingElement, T* newElement) + { + SLANG_ASSERT(existingElement != nullptr); + SLANG_ASSERT(newElement != nullptr); + if (existingElement == _last) + { + add(newElement); + } + else + { + newElement->_next = existingElement->_next; + existingElement->_next = newElement; + } + } + + void append(InternallyLinkedList<T> const& other) + { + if (!other._first) + { + } + else if (!_last) + { + _first = other._first; + _last = other._last; + } + else + { + SLANG_ASSERT(_first != nullptr); + + _last->_next = other._first; + _last = other._last; + } + } + +private: + T* _first = nullptr; + T* _last = nullptr; +}; + +template<typename T> +using InternallyLinkedListNode = InternallyLinkedList<T>::Node; + +} // namespace Slang + +#endif diff --git a/source/core/slang-memory-arena.h b/source/core/slang-memory-arena.h index 03631befe..714918b9a 100644 --- a/source/core/slang-memory-arena.h +++ b/source/core/slang-memory-arena.h @@ -476,4 +476,15 @@ SLANG_FORCE_INLINE void operator delete(void* memory, Slang::MemoryArena& arena) SLANG_UNUSED(arena); } +SLANG_FORCE_INLINE void* operator new[](size_t size, Slang::MemoryArena& arena) +{ + return arena.allocate(size); +} + +SLANG_FORCE_INLINE void operator delete[](void* memory, Slang::MemoryArena& arena) +{ + SLANG_UNUSED(memory); + SLANG_UNUSED(arena); +} + #endif // SLANG_MEMORY_ARENA_H diff --git a/source/core/slang-relative-ptr.cpp b/source/core/slang-relative-ptr.cpp new file mode 100644 index 000000000..0c4da2c6e --- /dev/null +++ b/source/core/slang-relative-ptr.cpp @@ -0,0 +1,2 @@ +// slang-relative-ptr.cpp +#include "slang-relative-ptr.h" diff --git a/source/core/slang-relative-ptr.h b/source/core/slang-relative-ptr.h new file mode 100644 index 000000000..6c4bc942c --- /dev/null +++ b/source/core/slang-relative-ptr.h @@ -0,0 +1,97 @@ +// slang-relative-ptr.h +#ifndef SLANG_RELATIVE_PTR_H +#define SLANG_RELATIVE_PTR_H + +// This file implements a smart pointer type `RelativePtr<T>` +// that, rather than storing the actual *address* of a value +// of type `T`, stores the relative offset (in bytes) between +// the target `T*` and the address of the `RelativePtr<T>` +// itself. +// +// This kind of pointer representation can be useful when +// implementing "memory-mappable" data structures that can +// still conveniently represent complicated object graphs. + +#include "slang-basic.h" + +namespace Slang +{ +namespace detail +{ +struct RelativePtr32Traits +{ + using Offset = Int32; + using UOffset = UInt32; +}; + +struct RelativePtr64Traits +{ + using Offset = Int64; + using UOffset = UInt64; +}; +} // namespace detail + +template<typename T, typename Traits> +struct RelativePtr +{ +public: + using This = RelativePtr<T, Traits>; + using Value = T; + using RawPtr = T*; + using Offset = typename Traits::Offset; + using UOffset = typename Traits::UOffset; + + RelativePtr() = default; + RelativePtr(RelativePtr const& ptr) { set(ptr); } + RelativePtr(RelativePtr&& ptr) { set(ptr); } + RelativePtr(T* ptr) { set(ptr); } + + void operator=(RelativePtr const& ptr) { set(ptr); } + void operator=(RelativePtr&& ptr) { set(ptr); } + void operator=(T* ptr) { set(ptr); } + + T* get() const + { + if (_offset == 0) + { + return nullptr; + } + + intptr_t thisAddr = intptr_t(this); + intptr_t targetAddr = thisAddr + intptr_t(_offset); + + return (T*)(targetAddr); + } + + void set(T* ptr) + { + if (ptr == nullptr) + { + _offset = 0; + return; + } + + intptr_t thisAddr = intptr_t(this); + intptr_t targetAddr = intptr_t(ptr); + intptr_t offsetVal = targetAddr - thisAddr; + + _offset = Offset(offsetVal); + SLANG_ASSERT(intptr_t(_offset) == offsetVal); + } + + operator T*() const { return get(); } + T* operator->() const { return get(); } + +private: + Offset _offset = 0; +}; + +template<typename T> +using RelativePtr32 = RelativePtr<T, detail::RelativePtr32Traits>; + +template<typename T> +using RelativePtr64 = RelativePtr<T, detail::RelativePtr64Traits>; + +} // namespace Slang + +#endif diff --git a/source/core/slang-riff.h b/source/core/slang-riff.h index 20b1d7c66..9459c8a55 100644 --- a/source/core/slang-riff.h +++ b/source/core/slang-riff.h @@ -16,6 +16,7 @@ // #include "slang-basic.h" +#include "slang-internally-linked-list.h" #include "slang-memory-arena.h" #include "slang-stream.h" #include "slang-writer.h" @@ -564,73 +565,6 @@ T const* as(Chunk const* chunk) return static_cast<T const*>(chunk); } -/// A linked list where the elements are themselves the nodes. -/// -template<typename T> -struct InternallyLinkedList -{ -public: - struct Node - { - public: - Node() {} - - private: - friend struct InternallyLinkedList<T>; - T* _next = nullptr; - }; - - struct Iterator - { - public: - Iterator() {} - - Iterator(T* node) - : _node(node) - { - } - - T* operator*() const { return _node; } - - void operator++() { _node = static_cast<Node const*>(_node)->_next; } - - bool operator!=(Iterator const& that) const { return _node != that._node; } - - private: - T* _node = nullptr; - }; - - Iterator begin() { return Iterator(_first); } - - Iterator end() { return Iterator(); } - - T* getFirst() const { return _first; } - - T* getLast() const { return _last; } - - void add(T* element) - { - if (!_last) - { - SLANG_ASSERT(_first == nullptr); - - _first = element; - _last = element; - } - else - { - SLANG_ASSERT(_first != nullptr); - - _last->_next = element; - _last = element; - } - } - -private: - T* _first = nullptr; - T* _last = nullptr; -}; - /// A builder for a chunk in a RIFF. class ChunkBuilder : public InternallyLinkedList<ChunkBuilder>::Node { diff --git a/source/slang/slang-fossil.cpp b/source/slang/slang-fossil.cpp new file mode 100644 index 000000000..e3b2e2c9d --- /dev/null +++ b/source/slang/slang-fossil.cpp @@ -0,0 +1,187 @@ +// slang-fossil.cpp +#include "slang-fossil.h" + +namespace Slang +{ +namespace Fossil +{ + +const char Fossil::Header::kMagic[16] = { + '\xAB', // byte 0 + 'f', // byte 1 + 'o', // byte 2 + 's', // byte 3 + 's', // byte 4 + 'i', // byte 5 + 'l', // byte 6 + ' ', // byte 7 + '1', // byte 8 + '0', // byte 9 + '0', // byte 10 + '\xBB', // byte 11 + '\r', // byte 12 + '\n', // byte 13 + '\x1A', // byte 14 + '\n' // byte 15 +}; + +FossilizedValRef getRootValue(ISlangBlob* blob) +{ + return getRootValue(blob->getBufferPointer(), blob->getBufferSize()); +} + +FossilizedValRef getRootValue(void const* data, Size size) +{ + if (!data) + { + SLANG_UNEXPECTED("bad format for fossil"); + } + + // There must be enough data to at least hold the header. + // + // (In practice there would need to be more data than + // just the header, but checking this invariant is a start). + // + if (size < sizeof(Fossil::Header)) + { + SLANG_UNEXPECTED("bad format for fossil"); + } + + // Once we've checked that there's enough data, we can read + // the contents of the header. + // + auto header = reinterpret_cast<Fossil::Header const*>(data); + + // The "magic" bytes at the start of the header must be + // what we expect (which is the contents of `Fossil::Header::kMagic`). + // + if (memcmp(header->magic, Fossil::Header::kMagic, sizeof(Fossil::Header::kMagic)) != 0) + { + SLANG_UNEXPECTED("bad format for fossil"); + } + + auto reportedSize = header->totalSizeIncludingHeader; + if (reportedSize > size) + { + SLANG_UNEXPECTED("bad format for fossil"); + } + + auto rootValueVariant = header->rootValue.get(); + if (!rootValueVariant) + { + SLANG_UNEXPECTED("bad format for fossil"); + } + + return FossilizedValRef( + rootValueVariant->getContentData(), + rootValueVariant->getContentLayout()); +} + +} // namespace Fossil + +Size FossilizedStringObj::getSize() const +{ + auto sizePtr = (FossilUInt*)this - 1; + return Size(*sizePtr); +} + +UnownedTerminatedStringSlice FossilizedStringObj::getValue() const +{ + auto size = getSize(); + return UnownedTerminatedStringSlice((char*)this, size); +} + +Count FossilizedContainerObj::getElementCount() const +{ + auto countPtr = (FossilUInt*)this - 1; + return Size(*countPtr); +} + +FossilizedValLayout* FossilizedVariantObj::getContentLayout() const +{ + auto layoutPtrPtr = (FossilizedPtr<FossilizedValLayout>*)this - 1; + return (*layoutPtrPtr).get(); +} + +FossilizedValRef getPtrTarget(FossilizedPtrValRef ptrRef) +{ + auto ptrLayout = ptrRef.getLayout(); + auto ptrPtr = ptrRef.getData(); + return FossilizedValRef(ptrPtr->getTargetData(), ptrLayout->elementLayout); +} + +bool hasValue(FossilizedOptionalObjRef optionalRef) +{ + return optionalRef.getData() != nullptr; +} + +FossilizedValRef getValue(FossilizedOptionalObjRef optionalRef) +{ + auto optionalLayout = optionalRef.getLayout(); + auto valuePtr = optionalRef.getData(); + return FossilizedValRef(valuePtr, optionalLayout->elementLayout); +} + +Count getElementCount(FossilizedContainerObjRef containerRef) +{ + if (!containerRef) + return 0; + + auto containerPtr = containerRef.getData(); + return containerPtr->getElementCount(); +} + +FossilizedValRef getElement(FossilizedContainerObjRef containerRef, Index index) +{ + SLANG_ASSERT(index >= 0); + SLANG_ASSERT(index < getElementCount(containerRef)); + + auto containerLayout = containerRef.getLayout(); + auto elementLayout = containerLayout->elementLayout.get(); + auto elementStride = containerLayout->elementStride; + + auto elementsPtr = (Byte*)containerRef.getData(); + auto elementPtr = (FossilizedVal*)(elementsPtr + elementStride * index); + return FossilizedValRef(elementPtr, elementLayout); +} + +Count getFieldCount(FossilizedRecordValRef recordRef) +{ + auto recordLayout = recordRef.getLayout(); + return recordLayout->fieldCount; +} + +FossilizedRecordElementLayout* FossilizedRecordLayout::getField(Index index) const +{ + SLANG_ASSERT(index >= 0); + SLANG_ASSERT(index < fieldCount); + + auto fieldsPtr = (FossilizedRecordElementLayout*)(this + 1); + return fieldsPtr + index; +} + + +FossilizedValRef getField(FossilizedRecordValRef recordRef, Index index) +{ + SLANG_ASSERT(index >= 0); + SLANG_ASSERT(index < getFieldCount(recordRef)); + + auto recordLayout = recordRef.getLayout(); + auto field = recordLayout->getField(index); + + auto fieldsPtr = (Byte*)recordRef.getData(); + auto fieldPtr = (FossilizedVal*)(fieldsPtr + field->offset); + return FossilizedValRef(fieldPtr, field->layout); +} + +FossilizedValRef getVariantContent(FossilizedVariantObjRef variantRef) +{ + return getVariantContent(variantRef.getData()); +} + +FossilizedValRef getVariantContent(FossilizedVariantObj* variantPtr) +{ + return FossilizedValRef(variantPtr->getContentData(), variantPtr->getContentLayout()); +} + +} // namespace Slang diff --git a/source/slang/slang-fossil.h b/source/slang/slang-fossil.h new file mode 100644 index 000000000..dcc12bacb --- /dev/null +++ b/source/slang/slang-fossil.h @@ -0,0 +1,488 @@ +// slang-fossil.h +#ifndef SLANG_FOSSIL_H +#define SLANG_FOSSIL_H + +// +// This file defines a memory-mappable binary format for +// serialized object graphs. To distinguish this specific +// format from other serialization formats used in the +// Slang project, we refer to these serialized objects +// as *fossilized* objects, and to the format as the +// *fossil format*. +// +// The term "fossil" is being used here to refer to formerly +// "live" objects that have been converted into an alternative +// form that can no longer perform their original functions, +// but that can still be inspected and dug through. +// + +#include "../core/slang-relative-ptr.h" + +namespace Slang +{ +// A key part of the fossil representation is the use of *relative pointers*, +// so that a fossilized object graph can be traversed dirctly in memory +// without having to deserialize any of the intermediate objects. +// +// Fossil uses 32-bit relative pointers, to keep the format compact. + +template<typename T> +using FossilizedPtr = RelativePtr32<T>; + +// Various other parts of the format need to store offsets or counts, +// and for consistency we will store them with the same number of +// bits as the relative pointers already used in the format. +// +// To make it easier for us to (potentially) change the relative +// pointer size down the line, we define type aliases for the +// general-purpose integer types that will be used in fossilized data. + +using FossilInt = FossilizedPtr<void>::Offset; +using FossilUInt = FossilizedPtr<void>::UOffset; + +// +// The fossil format supports data that is *self-describing*. +// +// A `FossilizedValLayout` describes the in-memory layout of a fossilized +// value. Given a `FossilizedValLayout` and a pointer to the data +// for a particular value, it is possible to inspect the structure +// of the fossilized data. +// +// If all you have is a `FossilizedVal*`, then there is no way to access +// its contents without assuming it is of some particular type and casting +// it. +// +// A `FossilizedVariantObj` is a fossilized value that is self-describing; +// it stores a (relative) pointer to a layout, which can be used to inspect +// its own data/state. +// + +struct FossilizedVal; +struct FossilizedValLayout; +struct FossilizedVariantObj; + +/// Kinds of values that can appear in fossilized data. +enum class FossilizedValKind : FossilUInt +{ + Bool, + Int8, + Int16, + Int32, + Int64, + UInt8, + UInt16, + UInt32, + UInt64, + Float32, + Float64, + String, + Array, + Optional, + Dictionary, + Tuple, + Struct, + Ptr, + Variant, +}; + +/// Layout information about a fossilized value in memory. +/// +/// +/// Every `FossilizedValLayout` stores the kind of the value. +/// Based on that kind, specific additional fields may be +/// available as part of the layout. +/// +struct FossilizedValLayout +{ + FossilizedValKind kind; +}; + +struct FossilizedPtrLikeLayout +{ + // Note: we aren't using inheritance in the definitions + // of these types, because per the letter of the law in + // C++, a type is only "standard layout" when there is + // only a single type in the inheritance hierarchy that + // has (non-static) data members. + + FossilizedValKind kind; + FossilizedPtr<FossilizedValLayout> elementLayout; +}; + +struct FossilizedContainerLayout +{ + FossilizedValKind kind; + FossilizedPtr<FossilizedValLayout> elementLayout; + FossilUInt elementStride; +}; + +struct FossilizedRecordElementLayout +{ + FossilizedPtr<FossilizedValLayout> layout; + FossilUInt offset; +}; + +struct FossilizedRecordLayout +{ + FossilizedValKind kind; + FossilUInt fieldCount; + + // FossilizedRecordElementLayout elements[]; + + FossilizedRecordElementLayout* getField(Index index) const; +}; + +/// A reference to a fossilized value in memory (of type T), and its layout. +/// +template<typename T> +struct FossilizedValRef_ +{ +public: + using Val = T; + using Layout = typename T::Layout; + + /// Construct a null reference. + /// + FossilizedValRef_() {} + + /// Construct a reference to the given `data`, assuming it has the given `layout`. + /// + FossilizedValRef_(T* data, Layout* layout) + : _data(data), _layout(layout) + { + } + + /// Get the kind of value being referenced. + /// + /// This reference must not be null. + /// + FossilizedValKind getKind() + { + SLANG_ASSERT(getLayout()); + return getLayout()->kind; + } + + /// Get the layout of the value being referenced. + /// + Layout* getLayout() { return _layout; } + + /// Get a pointer to the value being referenced. + /// + T* getData() { return _data; } + + operator T*() const { return _data; } + + T* operator->() { return _data; } + +private: + T* _data = nullptr; + Layout* _layout = nullptr; +}; + +using FossilizedValRef = FossilizedValRef_<FossilizedVal>; + +/// A fossilized value in memory. +/// +/// There isn't a lot that can be done with a bare pointer to +/// a `FossilizedVal`. This type is mostly declared to allow +/// us to make it explicit when a pointer points to a fossilized +/// value (even if we don't know anything about its layout). +/// +struct FossilizedVal +{ +public: + using Kind = FossilizedValKind; + using Layout = FossilizedValLayout; + + /// Determine if a value with the given `kind` should be allowed to cast to this type. + static bool _isMatchingKind(Kind kind) + { + SLANG_UNUSED(kind); + return true; + } + +protected: + FossilizedVal() = default; + FossilizedVal(FossilizedVal const&) = default; + FossilizedVal(FossilizedVal&&) = default; + ~FossilizedVal() = default; +}; + +template<typename T, FossilizedValKind kKind> +struct FossilizedSimpleVal : FossilizedVal +{ +public: + T getValue() const { return _value; } + + /// Determine if a value with the given `kind` should be allowed to cast to this type. + static bool _isMatchingKind(Kind kind) { return kind == kKind; } + +private: + T _value; +}; + +using FossilizedInt8Val = FossilizedSimpleVal<int8_t, FossilizedValKind::Int8>; +using FossilizedInt16Val = FossilizedSimpleVal<int16_t, FossilizedValKind::Int16>; +using FossilizedInt32Val = FossilizedSimpleVal<int32_t, FossilizedValKind::Int32>; +using FossilizedInt64Val = FossilizedSimpleVal<int64_t, FossilizedValKind::Int64>; + +using FossilizedUInt8Val = FossilizedSimpleVal<uint8_t, FossilizedValKind::UInt8>; +using FossilizedUInt16Val = FossilizedSimpleVal<uint16_t, FossilizedValKind::UInt16>; +using FossilizedUInt32Val = FossilizedSimpleVal<uint32_t, FossilizedValKind::UInt32>; +using FossilizedUInt64Val = FossilizedSimpleVal<uint64_t, FossilizedValKind::UInt64>; + +using FossilizedFloat32Val = FossilizedSimpleVal<float, FossilizedValKind::Float32>; +using FossilizedFloat64Val = FossilizedSimpleVal<double, FossilizedValKind::Float64>; + +struct FossilizedBoolVal : FossilizedVal +{ +public: + bool getValue() const { return _value != 0; } + + /// Determine if a value with the given `kind` should be allowed to cast to this type. + static bool _isMatchingKind(Kind kind) { return kind == Kind::Bool; } + +private: + uint8_t _value; +}; + +struct FossilizedPtrVal : FossilizedVal +{ +public: + using Layout = FossilizedPtrLikeLayout; + + FossilizedVal* getTargetData() const { return _value.get(); } + + /// Determine if a value with the given `kind` should be allowed to cast to this type. + static bool _isMatchingKind(Kind kind) { return kind == Kind::Ptr; } + +private: + FossilizedPtr<FossilizedVal> _value; +}; + + +struct FossilizedRecordVal : FossilizedVal +{ +public: + using Layout = FossilizedRecordLayout; + + /// Determine if a value with the given `kind` should be allowed to cast to this type. + static bool _isMatchingKind(Kind kind) + { + switch (kind) + { + default: + return false; + + case Kind::Struct: + case Kind::Tuple: + return true; + } + } +}; + +// +// Some of the following subtypes of `FossilizedVal` are +// named as `Fossilized*Obj` rather than `Fossilized*Val`, +// to indicate that they will only ever be located on the +// other side of a pointer indirection. +// +// E.g., a field of a fossilized struct value should never +// have a layout claiming it to be of kind `String`; instead +// it should show as a field of kind `Ptr`, where the +// pointed-to type is `String`. The same goes for `Optional`, +// `Array`, and `Dictionary`. +// +// This distinction only matters when dealing with things like +// an *optional* string, because instead of an in-memory +// layout like `Ptr -> Optional -> Ptr -> String`, the fossilized +// data will simply store `Ptr -> Optional -> String`. +// + +struct FossilizedStringObj : FossilizedVal +{ +public: + Size getSize() const; + UnownedTerminatedStringSlice getValue() const; + + /// Determine if a value with the given `kind` should be allowed to cast to this type. + static bool _isMatchingKind(Kind kind) { return kind == Kind::String; } + +private: + // Before the `this` address, there is a `FossilUInt` + // with the size of the string in bytes. + // + // At the `this` address there is a nul-terminated + // serquence of `getSize() + 1` bytes. +}; + +struct FossilizedOptionalObj : FossilizedVal +{ +public: + using Layout = FossilizedPtrLikeLayout; + + /// Determine if a value with the given `kind` should be allowed to cast to this type. + static bool _isMatchingKind(Kind kind) { return kind == Kind::Optional; } + + FossilizedVal* getValue() { return this; } + + FossilizedVal const* getValue() const { return this; } + +private: + // An absent optional is encoded as a null pointer + // (so `this` would be null), while a present value + // is encoded as a pointer to that value. Thus the + // held value is at the same address as `this`. +}; + +struct FossilizedContainerObj : FossilizedVal +{ +public: + using Layout = FossilizedContainerLayout; + + Count getElementCount() const; + + /// Determine if a value with the given `kind` should be allowed to cast to this type. + static bool _isMatchingKind(Kind kind) + { + switch (kind) + { + default: + return false; + + case Kind::Array: + case Kind::Dictionary: + return true; + } + } + +private: + // Before the `this` address, there is a `FossilUInt` + // with the number of elements. + // + // At the `this` address there is a sequence of + // `getCount()` elements. The layout of those elements + // cannot be determined without having a `FossilizedContainerLayout` + // for this container. +}; + +struct FossilizedVariantObj : FossilizedVal +{ +public: + FossilizedValLayout* getContentLayout() const; + + + FossilizedVal* getContentData() { return this; } + FossilizedVal const* getContentData() const { return this; } + + static bool _isMatchingKind(Kind kind) { return kind == Kind::Variant; } + +private: + // Before the `this` address, there is a `FossilizedPtr<FossilizedValLayout>` + // with the layout of the content. + // + // The content itself starts at the `this` address, with its + // layout determined by `getContentLayout()`. +}; + +/// Dynamic cast of a reference to a fossilized value. +/// +template<typename T, typename U> +FossilizedValRef_<T> as(FossilizedValRef_<U> valRef) +{ + if (!valRef || !T::_isMatchingKind(valRef.getKind())) + return FossilizedValRef_<T>(); + + return FossilizedValRef_<T>( + static_cast<T*>(valRef.getData()), + reinterpret_cast<typename T::Layout*>(valRef.getLayout())); +} + +using FossilizedInt8ValRef = FossilizedValRef_<FossilizedInt8Val>; +using FossilizedInt16ValRef = FossilizedValRef_<FossilizedInt16Val>; +using FossilizedInt32ValRef = FossilizedValRef_<FossilizedInt32Val>; +using FossilizedInt64ValRef = FossilizedValRef_<FossilizedInt64Val>; +using FossilizedUInt8ValRef = FossilizedValRef_<FossilizedUInt8Val>; +using FossilizedUInt16ValRef = FossilizedValRef_<FossilizedUInt16Val>; +using FossilizedUInt32ValRef = FossilizedValRef_<FossilizedUInt32Val>; +using FossilizedUInt64ValRef = FossilizedValRef_<FossilizedUInt64Val>; +using FossilizedFloat32ValRef = FossilizedValRef_<FossilizedFloat32Val>; +using FossilizedFloat64ValRef = FossilizedValRef_<FossilizedFloat64Val>; +using FossilizedBoolValRef = FossilizedValRef_<FossilizedBoolVal>; +using FossilizedStringObjRef = FossilizedValRef_<FossilizedStringObj>; +using FossilizedPtrValRef = FossilizedValRef_<FossilizedPtrVal>; +using FossilizedOptionalObjRef = FossilizedValRef_<FossilizedOptionalObj>; +using FossilizedContainerObjRef = FossilizedValRef_<FossilizedContainerObj>; +using FossilizedRecordValRef = FossilizedValRef_<FossilizedRecordVal>; +using FossilizedVariantObjRef = FossilizedValRef_<FossilizedVariantObj>; + +FossilizedValRef getPtrTarget(FossilizedPtrValRef ptrRef); + +bool hasValue(FossilizedOptionalObjRef optionalRef); +FossilizedValRef getValue(FossilizedOptionalObjRef optionalRef); + +Count getElementCount(FossilizedContainerObjRef containerRef); +FossilizedValRef getElement(FossilizedContainerObjRef containerRef, Index index); + +Count getFieldCount(FossilizedRecordValRef recordRef); +FossilizedValRef getField(FossilizedRecordValRef recordRef, Index index); + +FossilizedValRef getVariantContent(FossilizedVariantObjRef variantRef); +FossilizedValRef getVariantContent(FossilizedVariantObj* variantPtr); + + +namespace Fossil +{ +using RelativePtrOffset = FossilizedPtr<void>::Offset; + +/// Header for a fossil-format file or blob. +/// +/// A blob of fossilized data must start with a `Header` +/// that is properly formatted. +/// +struct Header +{ + /// The "magic" bytes used to identify this is a fossil-format blob. + char magic[16]; + + /// The expected bytes that should appear in `magic` + /// + static const char kMagic[16]; + + /// The total size of the fossil-format blob, including this header. + UInt64 totalSizeIncludingHeader; + + /// Flags; reserved for future use. + UInt32 flags; + + /// A relative pointer to the root value of the object graph. + /// + /// A fossil-format blob may only have one root value, and that + /// value *must* be a variant (so that it can reference the + /// layout information that describes itself). The *content* + /// of the root object can be arbitrary, so applications may + /// store multiple values using an array, struct, etc. + /// + FossilizedPtr<FossilizedVariantObj> rootValue; +}; + +/// Get the root object from a fossilized blob. +/// +/// This operation performs some basic validation on the blob to +/// ensure that it doesn't seem incorrectly sized or otherwise +/// corrupted/malformed. +/// +FossilizedValRef getRootValue(ISlangBlob* blob); + +/// Get the root object from a fossilized blob. +/// +/// This operation performs some basic validation on the blob to +/// ensure that it doesn't seem incorrectly sized or otherwise +/// corrupted/malformed. +/// +FossilizedValRef getRootValue(void const* data, Size size); +} // namespace Fossil + +} // namespace Slang + +#endif diff --git a/source/slang/slang-serialize-ast.cpp b/source/slang/slang-serialize-ast.cpp index 3a2d3818e..02dd374c1 100644 --- a/source/slang/slang-serialize-ast.cpp +++ b/source/slang/slang-serialize-ast.cpp @@ -5,6 +5,7 @@ #include "slang-compiler.h" #include "slang-diagnostics.h" #include "slang-mangle.h" +#include "slang-serialize-fossil.h" #include "slang-serialize-riff.h" namespace Slang @@ -202,7 +203,7 @@ using ASTSerializer = Serializer_<ASTSerializerImpl>; template<typename T> void serializeObject(ASTSerializer const& serializer, T*& value, NodeBase*) { - SLANG_SCOPED_SERIALIZER_STRUCT(serializer); + SLANG_SCOPED_SERIALIZER_VARIANT(serializer); serializer->handleASTNode(*(NodeBase**)&value); } @@ -224,7 +225,7 @@ void serialize(ASTSerializer const& serializer, SourceLoc& value) void serialize(ASTSerializer const& serializer, RequirementWitness& value) { - SLANG_SCOPED_SERIALIZER_TAGGED_UNION(serializer); + SLANG_SCOPED_SERIALIZER_VARIANT(serializer); serialize(serializer, value.m_flavor); switch (value.m_flavor) { @@ -406,7 +407,7 @@ void serialize(ASTSerializer const& serializer, SPIRVAsmInst& value) void serialize(ASTSerializer const& serializer, ValNodeOperand& value) { - SLANG_SCOPED_SERIALIZER_TAGGED_UNION(serializer); + SLANG_SCOPED_SERIALIZER_VARIANT(serializer); serialize(serializer, value.kind); switch (value.kind) { @@ -480,19 +481,19 @@ struct ASTEncodingContext : ASTSerializerImpl { public: ASTEncodingContext( - RIFF::BuildCursor& cursor, + ISerializerImpl* writer, ModuleDecl* module, SerialSourceLocWriter* sourceLocWriter) - : _writer(cursor.getCurrentChunk()), _module(module), _sourceLocWriter(sourceLocWriter) + : _writer(writer), _module(module), _sourceLocWriter(sourceLocWriter) { } private: - RIFFSerialWriter _writer; + ISerializerImpl* _writer = nullptr; ModuleDecl* _module = nullptr; SerialSourceLocWriter* _sourceLocWriter = nullptr; - virtual ISerializerImpl* getBaseSerializer() override { return &_writer; } + virtual ISerializerImpl* getBaseSerializer() override { return _writer; } virtual void handleName(Name*& value) override; virtual void handleSourceLoc(SourceLoc& value) override; @@ -531,7 +532,7 @@ public: Linkage* linkage, ASTBuilder* astBuilder, DiagnosticSink* sink, - RIFF::Chunk const* baseChunk, + ISerializerImpl* reader, SerialSourceLocReader* sourceLocReader, SourceLoc requestingSourceLoc) : _linkage(linkage) @@ -539,7 +540,7 @@ public: , _sink(sink) , _sourceLocReader(sourceLocReader) , _requestingSourceLoc(requestingSourceLoc) - , _riffReader(baseChunk) + , _reader(reader) { } @@ -549,9 +550,9 @@ private: DiagnosticSink* _sink = nullptr; SerialSourceLocReader* _sourceLocReader = nullptr; SourceLoc _requestingSourceLoc; - RIFFSerialReader _riffReader; + ISerializerImpl* _reader = nullptr; - virtual ISerializerImpl* getBaseSerializer() override { return &_riffReader; } + virtual ISerializerImpl* getBaseSerializer() override { return _reader; } virtual void handleName(Name*& value) override; virtual void handleSourceLoc(SourceLoc& value) override; @@ -910,8 +911,21 @@ void writeSerializedModuleAST( // TODO: we might want to have a more careful pass here, // where we only encode the public declarations. - ASTEncodingContext context(cursor, moduleDecl, sourceLocWriter); - serialize(ASTSerializer(&context), moduleDecl); + BlobBuilder blobBuilder; + { + Fossil::SerialWriter writer(blobBuilder); + + ASTEncodingContext context(&writer, moduleDecl, sourceLocWriter); + serialize(ASTSerializer(&context), moduleDecl); + } + + ComPtr<ISlangBlob> blob; + blobBuilder.writeToBlob(blob.writeRef()); + + void const* data = blob->getBufferPointer(); + size_t size = blob->getBufferSize(); + + cursor.addDataChunk(PropertyKeys<Module>::ASTModule, data, size); } ModuleDecl* readSerializedModuleAST( @@ -922,8 +936,14 @@ ModuleDecl* readSerializedModuleAST( SerialSourceLocReader* sourceLocReader, SourceLoc requestingSourceLoc) { + auto dataChunk = as<RIFF::DataChunk>(chunk); + + auto rootVal = Fossil::getRootValue(dataChunk->getPayload(), dataChunk->getPayloadSize()); + + Fossil::SerialReader reader(rootVal); + ASTDecodingContext - context(linkage, astBuilder, sink, chunk, sourceLocReader, requestingSourceLoc); + context(linkage, astBuilder, sink, &reader, sourceLocReader, requestingSourceLoc); ModuleDecl* moduleDecl = nullptr; serialize(ASTSerializer(&context), moduleDecl); diff --git a/source/slang/slang-serialize-fossil.cpp b/source/slang/slang-serialize-fossil.cpp new file mode 100644 index 000000000..5af9d2e6e --- /dev/null +++ b/source/slang/slang-serialize-fossil.cpp @@ -0,0 +1,1636 @@ +// slang-serialize-fossil.cpp +#include "slang-serialize-fossil.h" + +#include "../core/slang-blob.h" + +namespace Slang +{ +namespace Fossil +{ + +// +// SerialWriter +// + +SerialWriter::SerialWriter(ChunkBuilder* chunk) + : _arena(4096) +{ + _initialize(chunk); +} + +SerialWriter::SerialWriter(BlobBuilder& blobBuilder) + : _arena(4096) +{ + auto chunk = blobBuilder.addChunk(); + _initialize(chunk); +} + +void SerialWriter::_initialize(ChunkBuilder* chunk) +{ + _blobBuilder = chunk->getParentBlob(); + + // The top-level structure consists of a header, + // and a root value. We will allocate a distinct + // chunk for each of them, with the header coming + // first. + // + auto headerChunk = chunk; + auto rootValueChunk = headerChunk->addChunkAfter(); + + // We will write the fields of the header chunk manually, + // although we will use a temporary of type `Fossil::Header` + // to help make sure we write them with the correct sizes. + // + Fossil::Header header; + memcpy(header.magic, Fossil::Header::kMagic, sizeof(Fossil::Header::kMagic)); + header.totalSizeIncludingHeader = 0; + header.flags = 0; + + headerChunk->writeData(&header.magic, sizeof(header.magic)); + headerChunk->writeData( + &header.totalSizeIncludingHeader, + sizeof(header.totalSizeIncludingHeader)); + headerChunk->writeData(&header.flags, sizeof(header.flags)); + + // The main reason we are writing the fields manually is + // that the last field of the header is a relative pointer + // to the root-value chunk. + // + headerChunk->writeRelativePtr<Fossil::RelativePtrOffset>(rootValueChunk); + + // The root value should always be a variant, and we want to + // set up to write into it in a reasonable way. + // + auto rootPtrLayout = _createLayout(FossilizedValKind::Ptr); + _state = State(rootPtrLayout, rootValueChunk); + + _pushVariantScope(); +} + + +SerialWriter::~SerialWriter() +{ + _popVariantScope(); + + _flush(); +} + +SerializationMode SerialWriter::getMode() +{ + return SerializationMode::Write; +} + +void SerialWriter::handleBool(bool& value) +{ + // A boolean value will be serialized as a full byte. + uint8_t v = value; + _writeSimpleValue(FossilizedValKind::Bool, v); +} + +void SerialWriter::handleInt8(int8_t& value) +{ + _writeSimpleValue(FossilizedValKind::Int8, value); +} + +void SerialWriter::handleInt16(int16_t& value) +{ + _writeSimpleValue(FossilizedValKind::Int16, value); +} + +void SerialWriter::handleInt32(Int32& value) +{ + _writeSimpleValue(FossilizedValKind::Int32, value); +} + +void SerialWriter::handleInt64(Int64& value) +{ + _writeSimpleValue(FossilizedValKind::Int64, value); +} + +void SerialWriter::handleUInt8(uint8_t& value) +{ + _writeSimpleValue(FossilizedValKind::UInt8, value); +} + +void SerialWriter::handleUInt16(uint16_t& value) +{ + _writeSimpleValue(FossilizedValKind::UInt16, value); +} + +void SerialWriter::handleUInt32(UInt32& value) +{ + _writeSimpleValue(FossilizedValKind::UInt32, value); +} + +void SerialWriter::handleUInt64(UInt64& value) +{ + _writeSimpleValue(FossilizedValKind::UInt64, value); +} + +void SerialWriter::handleFloat32(float& value) +{ + _writeSimpleValue(FossilizedValKind::Float32, value); +} + +void SerialWriter::handleFloat64(double& value) +{ + _writeSimpleValue(FossilizedValKind::Float64, value); +} + +void SerialWriter::handleString(String& value) +{ + auto size = value.getLength(); + if (_shouldEmitWithPointerIndirection(FossilizedValKind::String)) + { + if (size == 0) + { + _writeNull(); + return; + } + + if (auto found = _mapStringToChunk.tryGetValue(value)) + { + auto existingChunk = *found; + + auto ptrLayout = + (ContainerLayoutObj*)_reserveDestinationForWrite(FossilizedValKind::Ptr); + _mergeLayout(ptrLayout->baseLayout, FossilizedValKind::String); + + _commitWrite(ValInfo::relativePtrTo(existingChunk)); + return; + } + } + + _pushPotentiallyIndirectValueScope(FossilizedValKind::String); + + auto data = value.getBuffer(); + _writeValueRaw(ValInfo::rawData(data, size + 1, 1)); + + auto chunk = _popPotentiallyIndirectValueScope(); + + auto rawSize = UInt32(size); + chunk->addPrefixData(&rawSize, sizeof(rawSize)); + + _mapStringToChunk.addIfNotExists(value, chunk); +} + +void SerialWriter::beginArray() +{ + _pushContainerScope(FossilizedValKind::Array); +} + +void SerialWriter::endArray() +{ + _popContainerScope(); +} + +void SerialWriter::beginDictionary() +{ + _pushContainerScope(FossilizedValKind::Dictionary); +} + +void SerialWriter::endDictionary() +{ + _popContainerScope(); +} + +void SerialWriter::_pushContainerScope(FossilizedValKind kind) +{ + _pushPotentiallyIndirectValueScope(kind); +} + +void SerialWriter::_popContainerScope() +{ + auto elementCount = _state.elementCount; + auto containerChunk = _popPotentiallyIndirectValueScope(); + + if (containerChunk) + { + auto rawElementCount = UInt32(elementCount); + containerChunk->addPrefixData(&rawElementCount, sizeof(rawElementCount)); + } +} + +bool SerialWriter::hasElements() +{ + return false; +} + +void SerialWriter::beginStruct() +{ + _pushInlineValueScope(FossilizedValKind::Struct); +} + +void SerialWriter::endStruct() +{ + _popInlineValueScope(); +} + +void SerialWriter::beginVariant() +{ + _pushVariantScope(); + _pushInlineValueScope(FossilizedValKind::Struct); +} + +void SerialWriter::endVariant() +{ + _popInlineValueScope(); + _popVariantScope(); +} + +void SerialWriter::handleFieldKey(char const* name, Int index) +{ + // For now we are ignoring field keys, and treating + // structs as basically equivalent to tuples. + SLANG_UNUSED(name); + SLANG_UNUSED(index); +} + +void SerialWriter::beginTuple() +{ + _pushInlineValueScope(FossilizedValKind::Tuple); +} + +void SerialWriter::endTuple() +{ + _popInlineValueScope(); +} + +void SerialWriter::beginOptional() +{ + _pushIndirectValueScope(FossilizedValKind::Optional); +} + +void SerialWriter::endOptional() +{ + _popIndirectValueScope(); +} + +void SerialWriter::handleSharedPtr(void*& value, Callback callback, void* userData) +{ + // Because we are writing, we only care about the + // pointer that is already present in `value`. + // + void* liveObjectPtr = value; + + // The first special case we check for is a null pointer, + // which we can serialize as an inline value. + // + if (liveObjectPtr == nullptr) + { + _writeNull(); + return; + } + + // Next, we check to see if we have encountered this + // pointer before, in which case we've already allocated + // an index for it in the object definition list, and + // we can simply write a reference to that object. + // + if (auto found = _mapLiveObjectPtrToFossilizedObject.tryGetValue(liveObjectPtr)) + { + auto fossilizedObject = *found; + + _reserveDestinationForWrite(fossilizedObject->ptrLayout); + _commitWrite(ValInfo::relativePtrTo(fossilizedObject->chunk)); + + return; + } + + auto ptrLayout = _reserveDestinationForWrite(FossilizedValKind::Ptr); + auto chunk = _blobBuilder->addChunk(); + + auto fossilizedObject = new (_arena) FossilizedObjectInfo(); + fossilizedObject->chunk = chunk; + fossilizedObject->ptrLayout = ptrLayout; + fossilizedObject->liveObjectPtr = liveObjectPtr; + fossilizedObject->callback = callback; + fossilizedObject->userData = userData; + + _fossilizedObjects.add(fossilizedObject); + _mapLiveObjectPtrToFossilizedObject.add(liveObjectPtr, fossilizedObject); + + _commitWrite(ValInfo::relativePtrTo(chunk)); +} + +void SerialWriter::handleUniquePtr(void*& value, Callback callback, void* userData) +{ + // We treat all pointers as shared pointers, because there isn't really + // an optimized representation we would want to use for the unique case. + // + handleSharedPtr(value, callback, userData); +} + +void SerialWriter::handleDeferredObjectContents(void* valuePtr, Callback callback, void* userData) +{ + // Because we are already deferring writing of the *entirety* of + // an object's members as part of how `handleSharedPtr()` works, + // we don't need to implement deferral at this juncture. + // + // (In practice the `handleDeferredObjectContents()` operation is + // more for the benefit of reading than writing). + // + callback(valuePtr, userData); +} + +SerialWriter::LayoutObj* SerialWriter::_createSimpleLayout(FossilizedValKind kind) +{ + switch (kind) + { + case FossilizedValKind::Bool: + case FossilizedValKind::Int8: + case FossilizedValKind::UInt8: + return new (_arena) SimpleLayoutObj(kind, 1); + + case FossilizedValKind::Int16: + case FossilizedValKind::UInt16: + return new (_arena) SimpleLayoutObj(kind, 2); + + case FossilizedValKind::Int32: + case FossilizedValKind::UInt32: + case FossilizedValKind::Float32: + return new (_arena) SimpleLayoutObj(kind, 4); + + case FossilizedValKind::Int64: + case FossilizedValKind::UInt64: + case FossilizedValKind::Float64: + return new (_arena) SimpleLayoutObj(kind, 8); + + case FossilizedValKind::String: + return new (_arena) SimpleLayoutObj(kind); + + default: + SLANG_UNEXPECTED("unhandled case"); + UNREACHABLE_RETURN(nullptr); + } +} + +SerialWriter::LayoutObj* SerialWriter::_createLayout(FossilizedValKind kind) +{ + switch (kind) + { + case FossilizedValKind::Array: + case FossilizedValKind::Optional: + case FossilizedValKind::Dictionary: + return new (_arena) ContainerLayoutObj(kind, nullptr); + + case FossilizedValKind::Ptr: + return new (_arena) ContainerLayoutObj( + kind, + nullptr, + sizeof(Fossil::RelativePtrOffset), + sizeof(Fossil::RelativePtrOffset)); + + case FossilizedValKind::Struct: + case FossilizedValKind::Tuple: + return new (_arena) RecordLayoutObj(kind); + + case FossilizedValKind::Variant: + // A variant is being treated like a container in this context, + // because it wants to be able to track the layout of what it + // ended up holding... + // + return new (_arena) ContainerLayoutObj(kind, nullptr); + + case FossilizedValKind::Bool: + case FossilizedValKind::Int8: + case FossilizedValKind::Int16: + case FossilizedValKind::Int32: + case FossilizedValKind::Int64: + case FossilizedValKind::UInt8: + case FossilizedValKind::UInt16: + case FossilizedValKind::UInt32: + case FossilizedValKind::UInt64: + case FossilizedValKind::Float32: + case FossilizedValKind::Float64: + case FossilizedValKind::String: + { + if (auto found = _simpleLayouts.tryGetValue(kind)) + return *found; + + auto layout = _createSimpleLayout(kind); + _simpleLayouts.add(kind, layout); + return layout; + } + + default: + SLANG_UNEXPECTED("unhandled case"); + UNREACHABLE_RETURN(nullptr); + } +} + +SerialWriter::LayoutObj* SerialWriter::_mergeLayout(LayoutObj*& dst, FossilizedValKind kind) +{ + if (!dst) + { + dst = _createLayout(kind); + } + + if (dst->kind != kind) + { + SLANG_UNEXPECTED("type mismatch during serialization"); + } + + // As a special case, if the right-hand-side is a variant, + // then we want to have a unique layout object for each + // instance. + // + if (kind == FossilizedValKind::Variant) + { + auto src = _createLayout(kind); + return src; + } + + return dst; +} + +void SerialWriter::_mergeLayout(LayoutObj*& dst, LayoutObj* src) +{ + if (dst == src) + return; + + if (!src) + return; + + if (!dst) + { + dst = src; + return; + } + + _mergeLayout(dst, src->getKind()); + + switch (src->getKind()) + { + case FossilizedValKind::Array: + case FossilizedValKind::Optional: + case FossilizedValKind::Dictionary: + case FossilizedValKind::Ptr: + { + auto dstContainer = (ContainerLayoutObj*)dst; + auto srcContainer = (ContainerLayoutObj*)src; + _mergeLayout(dstContainer->baseLayout, srcContainer->baseLayout); + } + break; + + case FossilizedValKind::String: + break; + + case FossilizedValKind::Variant: + // Recursive merging should not be applied to variants; + // each variant is unique until later deduplication. + break; + + default: + SLANG_UNEXPECTED("unhandled case"); + break; + } +} + +SerialWriter::RecordLayoutObj::FieldInfo& SerialWriter::_getOrAddField( + RecordLayoutObj* recordLayout, + Index index) +{ + // Note: we are doing all the allocation for `LayoutObj`s from + // an arena, so that we don't have to worry about managing + // their lifetimes carefully. + // + // One place where that is a bit tedious is handling the storage + // for the array of fields for a record. + // + // TODO(tfoley): see if there's allocator support on `List<T>` + // or similar, so that it can be made to just use the arena. + + SLANG_ASSERT(recordLayout); + SLANG_ASSERT(index >= 0); + + if (index < recordLayout->fieldCount) + return recordLayout->fields[index]; + + SLANG_ASSERT(index == recordLayout->fieldCount); + + if (index >= recordLayout->fieldCapacity) + { + if (recordLayout->fieldCapacity == 0) + recordLayout->fieldCapacity = 16; + + while (index >= recordLayout->fieldCapacity) + { + recordLayout->fieldCapacity = (recordLayout->fieldCapacity * 3) >> 1; + } + + auto newFields = new (_arena) RecordLayoutObj::FieldInfo[recordLayout->fieldCapacity]; + for (Index i = 0; i < recordLayout->fieldCount; ++i) + newFields[i] = recordLayout->fields[i]; + recordLayout->fields = newFields; + } + + recordLayout->fields[recordLayout->fieldCount++] = RecordLayoutObj::FieldInfo(); + return recordLayout->fields[index]; +} + +SerialWriter::ValInfo SerialWriter::ValInfo::rawData(void const* data, Size size, Size alignment) +{ + ValInfo val(Kind::RawData); + val.data.ptr = data; + val.data.size = size; + val.data.alignment = alignment; + return val; +} + +SerialWriter::ValInfo SerialWriter::ValInfo::relativePtrTo(ChunkBuilder* targetChunk) +{ + ValInfo val(Kind::RelativePtr); + val.chunk = targetChunk; + return val; +} + +SerialWriter::ValInfo SerialWriter::ValInfo::contentsOf(ChunkBuilder* chunk) +{ + ValInfo val(Kind::ContentsOfChunk); + val.chunk = chunk; + return val; +} + +Size SerialWriter::ValInfo::getAlignment() const +{ + switch (kind) + { + case Kind::RelativePtr: + return sizeof(Fossil::RelativePtrOffset); + + case Kind::ContentsOfChunk: + return chunk->getAlignment(); + + case Kind::RawData: + return data.alignment; + + default: + SLANG_UNEXPECTED("unhandled case"); + break; + } +} + +void SerialWriter::_pushInlineValueScope(FossilizedValKind kind) +{ + auto layout = _reserveDestinationForWrite(kind); + _pushState(layout); +} + +void SerialWriter::_popInlineValueScope() +{ + auto layout = _state.layout; + auto chunk = _state.chunk; + + if (chunk) + { + if (layout->size == 0) + { + layout->size = chunk->getContentSize(); + } + SLANG_ASSERT(layout->size == chunk->getContentSize()); + } + + _popState(); + + _commitWrite(ValInfo::contentsOf(chunk)); +} + +void SerialWriter::_pushVariantScope() +{ + _pushPotentiallyIndirectValueScope(FossilizedValKind::Variant); +} + +void SerialWriter::_popVariantScope() +{ + SLANG_ASSERT(_state.layout); + SLANG_ASSERT(_state.layout->kind == FossilizedValKind::Variant); + auto variantLayout = (ContainerLayoutObj*)_state.layout; + auto valueLayout = variantLayout->baseLayout; + SLANG_ASSERT(valueLayout); + + auto variantChunk = _popPotentiallyIndirectValueScope(); + + // The key feature of a variant is that it carries its own + // layout information. + // + // We need to insert a pointer to the serialized form + // of the layout information for the element type as a header + // *before* the content. + // + // The first step there is to turn the element layout into + // a handle such that we can write a relative pointer to it. + // + + VariantInfo variantInfo; + variantInfo.layout = valueLayout; + variantInfo.chunk = variantChunk; + _variants.add(variantInfo); +} + + +void SerialWriter::_pushPotentiallyIndirectValueScope(FossilizedValKind kind) +{ + if (_shouldEmitWithPointerIndirection(kind)) + { + _pushIndirectValueScope(kind); + } + else + { + _pushInlineValueScope(kind); + } +} + +ChunkBuilder* SerialWriter::_popPotentiallyIndirectValueScope() +{ + // TODO(tfoley): Try to make this function just be a simple + // conditional to select between the functions for the + // indirect and inline cases. + + auto valueLayout = _state.layout; + auto valueChunk = _state.chunk; + _popState(); + + auto valueKind = valueLayout->getKind(); + if (_shouldEmitWithPointerIndirection(valueKind)) + { + return _writeKnownIndirectValueSharedLogic(valueChunk); + } + else + { + _commitWrite(ValInfo::contentsOf(valueChunk)); + return _state.chunk; + } +} + +void SerialWriter::_pushIndirectValueScope(FossilizedValKind kind) +{ + auto ptrLayout = (ContainerLayoutObj*)_reserveDestinationForWrite(FossilizedValKind::Ptr); + auto valueLayout = _mergeLayout(ptrLayout->baseLayout, kind); + + _pushState(valueLayout); +} + +void SerialWriter::_popIndirectValueScope() +{ + auto valueChunk = _state.chunk; + _popState(); + + _writeKnownIndirectValueSharedLogic(valueChunk); +} + +ChunkBuilder* SerialWriter::_writeKnownIndirectValueSharedLogic(ChunkBuilder* valueChunk) +{ + if (!valueChunk) + { + _commitWrite(ValInfo::relativePtrTo(nullptr)); + return nullptr; + } + + _blobBuilder->addChunk(valueChunk); + + _commitWrite(ValInfo::relativePtrTo(valueChunk)); + return valueChunk; +} + + +void SerialWriter::_pushState(LayoutObj* layout) +{ + _stack.add(_state); + _state = State(layout); +} + +void SerialWriter::_popState() +{ + SLANG_ASSERT(_stack.getCount() != 0); + _state = _stack.getLast(); + _stack.removeLast(); +} + +void SerialWriter::_ensureChunkExists() +{ + if (_state.chunk != nullptr) + return; + + _state.chunk = _blobBuilder->createUnparentedChunk(); +} + +void SerialWriter::_writeValueRaw(ValInfo const& val) +{ + switch (val.kind) + { + case ValInfo::Kind::RawData: + if (val.data.size == 0) + return; + _ensureChunkExists(); + _state.chunk->writePaddingToAlignTo(val.data.alignment); + _state.chunk->writeData(val.data.ptr, val.data.size); + break; + + case ValInfo::Kind::RelativePtr: + _ensureChunkExists(); + _state.chunk->writeRelativePtr<Fossil::RelativePtrOffset>(val.chunk); + break; + + case ValInfo::Kind::ContentsOfChunk: + { + if (!_state.chunk) + { + _state.chunk = val.chunk; + } + else + { + _state.chunk->addContentsOf(val.chunk); + } + } + break; + + default: + SLANG_UNEXPECTED("unknown Fossil::SerialWriter::ValInfo::Kind"); + break; + } +} + +bool SerialWriter::_shouldEmitWithPointerIndirection(FossilizedValKind kind) +{ + switch (kind) + { + default: + return false; + + case FossilizedValKind::Optional: + return true; + + case FossilizedValKind::Array: + case FossilizedValKind::Dictionary: + case FossilizedValKind::String: + case FossilizedValKind::Variant: + break; + } + + switch (_state.layout->getKind()) + { + default: + return true; + + case FossilizedValKind::Optional: + case FossilizedValKind::Ptr: + return false; + } +} + +SerialWriter::LayoutObj*& SerialWriter::_reserveDestinationForWrite() +{ + switch (_state.layout->getKind()) + { + case FossilizedValKind::Struct: + case FossilizedValKind::Tuple: + { + auto recordLayout = (RecordLayoutObj*)_state.layout; + auto elementIndex = _state.elementCount; + auto& elementLayout = _getOrAddField(recordLayout, elementIndex).layout; + return elementLayout; + } + break; + + case FossilizedValKind::Ptr: + case FossilizedValKind::Optional: + case FossilizedValKind::Array: + case FossilizedValKind::Dictionary: + case FossilizedValKind::Variant: + { + auto containerLayout = (ContainerLayoutObj*)_state.layout; + auto& elementLayout = containerLayout->baseLayout; + return elementLayout; + } + break; + + default: + SLANG_UNEXPECTED("unhandled case"); + break; + } +} + +SerialWriter::LayoutObj* SerialWriter::_reserveDestinationForWrite(FossilizedValKind srcKind) +{ + return _mergeLayout(_reserveDestinationForWrite(), srcKind); +} + +SerialWriter::LayoutObj* SerialWriter::_reserveDestinationForWrite(LayoutObj* srcLayout) +{ + SLANG_ASSERT(srcLayout != nullptr); + _mergeLayout(_reserveDestinationForWrite(), srcLayout); + return srcLayout; +} + +void SerialWriter::_commitWrite(ValInfo const& val) +{ + auto outerKind = _state.layout->getKind(); + switch (outerKind) + { + case FossilizedValKind::Struct: + case FossilizedValKind::Tuple: + { + auto recordLayout = (RecordLayoutObj*)_state.layout; + auto elementIndex = _state.elementCount++; + auto& fieldInfo = _getOrAddField(recordLayout, elementIndex); + + Size fieldOffset = 0; + if (elementIndex != 0) + { + auto chunk = _state.chunk; + chunk->writePaddingToAlignTo(val.getAlignment()); + + fieldOffset = chunk->getContentSize(); + } + fieldInfo.offset = fieldOffset; + + _writeValueRaw(val); + } + break; + + case FossilizedValKind::Optional: + case FossilizedValKind::Ptr: + case FossilizedValKind::Array: + case FossilizedValKind::Dictionary: + case FossilizedValKind::Variant: + { + auto elementIndex = _state.elementCount++; + + switch (outerKind) + { + case FossilizedValKind::Optional: + case FossilizedValKind::Ptr: + if (elementIndex > 0) + { + SLANG_UNEXPECTED( + "error during serialization: optional with more than one value inside!!"); + } + break; + + default: + break; + } + + _writeValueRaw(val); + } + break; + + default: + SLANG_UNEXPECTED("unhandled case"); + break; + } +} + +void SerialWriter::_writeSimpleValue( + FossilizedValKind kind, + void const* data, + size_t size, + size_t alignment) +{ + auto layout = _reserveDestinationForWrite(kind); + SLANG_ASSERT(layout->size == size); + SLANG_ASSERT(layout->alignment = alignment); + _commitWrite(ValInfo::rawData(data, size, alignment)); +} + +void SerialWriter::_writeNull() +{ + RelativePtrOffset offset = 0; + _writeSimpleValue(FossilizedValKind::Ptr, offset); +} + +void SerialWriter::_flush() +{ + while (_writtenObjectDefinitionCount < _fossilizedObjects.getCount()) + { + auto objectIndex = _writtenObjectDefinitionCount++; + auto fossilizedObject = _fossilizedObjects[objectIndex]; + + SLANG_ASSERT(fossilizedObject->liveObjectPtr); + + _state = State(fossilizedObject->ptrLayout, fossilizedObject->chunk); + + fossilizedObject->callback(&fossilizedObject->liveObjectPtr, fossilizedObject->userData); + } + + // Once we've written out all the payload data, we can start to work on + // serializing layout information for all the variant values that were + // written. + // + for (auto variantInfo : _variants) + { + auto layoutChunk = _getOrCreateChunkForLayout(variantInfo.layout); + variantInfo.chunk->addPrefixRelativePtr<Fossil::RelativePtrOffset>(layoutChunk); + } +} + +ChunkBuilder* SerialWriter::_getOrCreateChunkForLayout(LayoutObj* layout) +{ + if (!layout) + return nullptr; + + // We start by looking for an existing chunk for `layout`, + // which would be cached on the object itself. + // + if (auto existingChunk = layout->chunk) + return existingChunk; + + // Next we look for an existing chunk that matches the + // structure of `layout`. + // + LayoutObjKey key = {layout}; + if (auto found = _mapLayoutObjToChunk.tryGetValue(key)) + { + auto existingChunk = *found; + layout->chunk = existingChunk; + return existingChunk; + } + + // If no existing layout has been written to a chunk, + // then we'll create one. + // + auto chunk = _blobBuilder->addChunk(); + layout->chunk = chunk; + _mapLayoutObjToChunk.add(key, chunk); + + auto kind = layout->getKind(); + auto rawKind = UInt32(kind); + chunk->writeData(&rawKind, sizeof(rawKind)); + + switch (kind) + { + default: + break; + + case FossilizedValKind::Ptr: + case FossilizedValKind::Optional: + { + auto containerLayout = (ContainerLayoutObj*)layout; + auto elementLayout = containerLayout->baseLayout; + auto elementLayoutChunk = _getOrCreateChunkForLayout(elementLayout); + chunk->writeRelativePtr<Fossil::RelativePtrOffset>(elementLayoutChunk); + } + break; + + case FossilizedValKind::Array: + case FossilizedValKind::Dictionary: + { + auto containerLayout = (ContainerLayoutObj*)layout; + auto elementLayout = containerLayout->baseLayout; + auto elementLayoutChunk = _getOrCreateChunkForLayout(elementLayout); + chunk->writeRelativePtr<Fossil::RelativePtrOffset>(elementLayoutChunk); + + UInt32 elementStride = 0; + if (elementLayout) + { + elementStride = + UInt32(roundUpToAlignment(elementLayout->size, elementLayout->alignment)); + SLANG_ASSERT(elementStride != 0); + } + chunk->writeData(&elementStride, sizeof(elementStride)); + } + break; + + case FossilizedValKind::Struct: + case FossilizedValKind::Tuple: + { + auto recordLayout = (RecordLayoutObj*)layout; + + auto fieldCount = UInt32(recordLayout->fieldCount); + chunk->writeData(&fieldCount, sizeof(fieldCount)); + + for (Index i = 0; i < fieldCount; ++i) + { + auto& field = recordLayout->fields[i]; + auto fieldLayoutChunk = _getOrCreateChunkForLayout(field.layout); + chunk->writeRelativePtr<Fossil::RelativePtrOffset>(fieldLayoutChunk); + + auto fieldOffset = UInt32(field.offset); + chunk->writeData(&fieldOffset, sizeof(fieldOffset)); + + if (i != 0) + { + // Make sure that all but the first field have + // a non-zero offset, to validate that offsets + // are being comptued at all. + // + SLANG_ASSERT(fieldOffset != 0); + } + } + } + break; + } + + return chunk; +} + +bool SerialWriter::LayoutObjKey::operator==(LayoutObjKey const& that) const +{ + if (obj == that.obj) + return true; + + if (!obj || !that.obj) + return false; + + SLANG_ASSERT(obj && that.obj); + + if (obj->kind != that.obj->kind) + return false; + + switch (obj->kind) + { + default: + break; + + case FossilizedValKind::Array: + case FossilizedValKind::Dictionary: + case FossilizedValKind::Optional: + case FossilizedValKind::Ptr: + { + auto thisContainer = (ContainerLayoutObj*)obj; + auto thatContainer = (ContainerLayoutObj*)that.obj; + + LayoutObjKey thisElement = thisContainer->baseLayout; + LayoutObjKey thatElement = thatContainer->baseLayout; + + if (thisElement != thatElement) + return false; + } + break; + + case FossilizedValKind::Tuple: + case FossilizedValKind::Struct: + { + auto thisRecord = (RecordLayoutObj*)obj; + auto thatRecord = (RecordLayoutObj*)that.obj; + + if (thisRecord->fieldCount != thatRecord->fieldCount) + return false; + + auto fieldCount = thisRecord->fieldCount; + for (Index i = 0; i < fieldCount; ++i) + { + auto thisField = thisRecord->fields[i]; + auto thatField = thatRecord->fields[i]; + + if (thisField.offset != thatField.offset) + return false; + + LayoutObjKey thisFieldLayout = thisField.layout; + LayoutObjKey thatFieldLayout = thatField.layout; + + if (thisFieldLayout != thatFieldLayout) + return false; + } + } + break; + } + + return true; +} + +bool SerialWriter::LayoutObjKey::operator!=(LayoutObjKey const& that) const +{ + return !(*this == that); +} + +HashCode64 SerialWriter::LayoutObjKey::getHashCode() const +{ + Hasher hasher; + hashInto(hasher); + return hasher.getResult(); +} + +void SerialWriter::LayoutObjKey::hashInto(Hasher& hasher) const +{ + if (!obj) + { + hasher.hashValue(obj); + return; + } + + hasher.hashValue(obj->kind); + + switch (obj->kind) + { + default: + break; + + case FossilizedValKind::Array: + case FossilizedValKind::Dictionary: + case FossilizedValKind::Optional: + case FossilizedValKind::Ptr: + { + auto container = (ContainerLayoutObj*)obj; + + LayoutObjKey(container->baseLayout).hashInto(hasher); + } + break; + + case FossilizedValKind::Tuple: + case FossilizedValKind::Struct: + { + auto record = (RecordLayoutObj*)obj; + + auto fieldCount = record->fieldCount; + hasher.hashValue(record->fieldCount); + + for (Index i = 0; i < fieldCount; ++i) + { + auto& field = record->fields[i]; + hasher.hashValue(field.offset); + LayoutObjKey(field.layout).hashInto(hasher); + } + } + break; + } +} + + +// +// SerialReader +// + +SerialReader::SerialReader(FossilizedValRef valRef) +{ + _state.type = State::Type::Root; + _state.baseValue = valRef; + _state.elementIndex = 0; + _state.elementCount = 1; +} + +SerialReader::~SerialReader() +{ + _flush(); +} + +SerializationMode SerialReader::getMode() +{ + return SerializationMode::Read; +} + +void SerialReader::handleBool(bool& value) +{ + auto valRef = _readValRef(); + value = as<FossilizedBoolVal>(valRef)->getValue(); +} + +void SerialReader::handleInt8(int8_t& value) +{ + auto valRef = _readValRef(); + value = as<FossilizedInt8Val>(valRef)->getValue(); +} + +void SerialReader::handleInt16(int16_t& value) +{ + auto valRef = _readValRef(); + value = as<FossilizedInt16Val>(valRef)->getValue(); +} + +void SerialReader::handleInt32(Int32& value) +{ + auto valRef = _readValRef(); + value = as<FossilizedInt32Val>(valRef)->getValue(); +} + +void SerialReader::handleInt64(Int64& value) +{ + auto valRef = _readValRef(); + value = as<FossilizedInt64Val>(valRef)->getValue(); +} + +void SerialReader::handleUInt8(uint8_t& value) +{ + auto valRef = _readValRef(); + value = as<FossilizedUInt8Val>(valRef)->getValue(); +} + +void SerialReader::handleUInt16(uint16_t& value) +{ + auto valRef = _readValRef(); + value = as<FossilizedUInt16Val>(valRef)->getValue(); +} + +void SerialReader::handleUInt32(UInt32& value) +{ + auto valRef = _readValRef(); + value = as<FossilizedUInt32Val>(valRef)->getValue(); +} + +void SerialReader::handleUInt64(UInt64& value) +{ + auto valRef = _readValRef(); + value = as<FossilizedUInt64Val>(valRef)->getValue(); +} + +void SerialReader::handleFloat32(float& value) +{ + auto valRef = _readValRef(); + value = as<FossilizedFloat32Val>(valRef)->getValue(); +} + +void SerialReader::handleFloat64(double& value) +{ + auto valRef = _readValRef(); + value = as<FossilizedFloat64Val>(valRef)->getValue(); +} + +void SerialReader::handleString(String& value) +{ + auto valRef = _readPotentiallyIndirectValRef(); + if (!valRef) + { + value = String(); + } + else + { + value = as<FossilizedStringObj>(valRef)->getValue(); + } +} + +void SerialReader::beginArray() +{ + auto valRef = _readPotentiallyIndirectValRef(); + auto arrayRef = as<FossilizedContainerObj>(valRef); + + _pushState(); + + _state.type = State::Type::Array; + _state.baseValue = valRef; + _state.elementIndex = 0; + _state.elementCount = getElementCount(arrayRef); +} + +void SerialReader::endArray() +{ + _popState(); +} + +void SerialReader::beginDictionary() +{ + auto valRef = _readPotentiallyIndirectValRef(); + auto dictionaryRef = as<FossilizedContainerObj>(valRef); + + _pushState(); + + _state.type = State::Type::Dictionary; + _state.baseValue = valRef; + _state.elementIndex = 0; + _state.elementCount = getElementCount(dictionaryRef); +} + +void SerialReader::endDictionary() +{ + _popState(); +} + +bool SerialReader::hasElements() +{ + return _state.elementIndex < _state.elementCount; +} + +void SerialReader::beginStruct() +{ + auto valRef = _readValRef(); + auto recordRef = as<FossilizedRecordVal>(valRef); + + _pushState(); + + _state.type = State::Type::Struct; + _state.baseValue = valRef; + _state.elementIndex = 0; + _state.elementCount = getFieldCount(recordRef); +} + +void SerialReader::endStruct() +{ + _popState(); +} + +void SerialReader::beginVariant() +{ + auto valRef = _readPotentiallyIndirectValRef(); + auto variantRef = as<FossilizedVariantObj>(valRef); + + auto contentValRef = getVariantContent(variantRef); + auto contentRecordRef = as<FossilizedRecordVal>(contentValRef); + + _pushState(); + + _state.type = State::Type::Struct; + _state.baseValue = contentValRef; + _state.elementIndex = 0; + _state.elementCount = getFieldCount(contentRecordRef); +} + +void SerialReader::endVariant() +{ + _popState(); +} + +void SerialReader::handleFieldKey(char const* name, Int index) +{ + // For now we are ignoring field keys, and treating + // structs as basically equivalent to tuples. + SLANG_UNUSED(name); + SLANG_UNUSED(index); +} + +void SerialReader::beginTuple() +{ + auto valRef = _readValRef(); + auto recordRef = as<FossilizedRecordVal>(valRef); + + _pushState(); + + _state.type = State::Type::Tuple; + _state.baseValue = valRef; + _state.elementIndex = 0; + _state.elementCount = getFieldCount(recordRef); +} + +void SerialReader::endTuple() +{ + _popState(); +} + +void SerialReader::beginOptional() +{ + auto valRef = _readIndirectValRef(); + auto optionalRef = as<FossilizedOptionalObj>(valRef); + + _pushState(); + + _state.type = State::Type::Optional; + _state.baseValue = valRef; + _state.elementIndex = 0; + _state.elementCount = Count(hasValue(optionalRef)); +} + +void SerialReader::endOptional() +{ + _popState(); +} + +void SerialReader::handleSharedPtr(void*& value, Callback callback, void* userData) +{ + // The fossilized value at our cursor must be a pointer, + // and we can resolve what it is pointing to easily enough. + // + auto valRef = _readValRef(); + auto ptrRef = as<FossilizedPtrVal>(valRef); + auto targetValRef = getPtrTarget(ptrRef); + + // The logic here largely mirrors what appears in + // `SerialWriter::handleSharedPtr`. + // + // We first check for an explicitly written null pointer. + // If we find one our work is very easy. + // + if (!targetValRef) + { + value = nullptr; + return; + } + + // Now we need to check if we've previously read in + // a reference to the same object. + // + if (auto found = _mapFossilizedObjectPtrToObjectInfo.tryGetValue(targetValRef.getData())) + { + auto objectInfo = *found; + + // We've seen this object before, although it + // is still possible that we are in the middle + // of reading it as part of an invocation + // of `handleSharedPtr()` further up the call + // stack. + // + // If a non-nullpointer value has already been + // written into the `objectInfo`, then that means + // the callback that was run for the prior (or + // in-flight) read operation has already allocated + // or found an object and written it out. + // In that case we will trust the value. + // + if (objectInfo->resurrectedObjectPtr == nullptr) + { + // It is possible that the pointer is null because + // the callback that was invoked explicitly *chose* + // to yield a null pointer (perhaps the application + // is choosing not to deserialize some optional + // piece of state). + // + // However, if there is still a callback in-flight + // to read this object, and the pointer is null, + // then we have reached a circular reference, + // and need to signal an error. + // + if (objectInfo->state == ObjectState::ReadingInProgress) + { + SLANG_UNEXPECTED("circularity detected in fossil deserialization"); + } + } + value = objectInfo->resurrectedObjectPtr; + return; + } + + // At this point we are reading a reference to an + // object index that has not yet been read at all. + // + auto objectInfo = RefPtr(new ObjectInfo()); + _mapFossilizedObjectPtrToObjectInfo.add(targetValRef.getData(), objectInfo); + + objectInfo->fossilizedObjectRef = targetValRef; + + // We cannot return from this function until we have + // stored a pointer into `value`, to represent the + // deserialized object. + // + // Thus we will set ourselves up to start reading + // from the relevant object definition, and invoke + // the callback that was passed in. + // + // Calling into user-defined serialization logic from + // within this function creates the possibility of + // unbounded/infinite recursion, so it is vital that + // the user is properly using `deferSerializeObjectContents()` + // to delay reading data that isn't immediately + // necessary. + // + // We will still set the `objectInfo.state` to reflect + // this in-flight operation so that we can detect + // a cirularity if one occurs at runtime. + // + objectInfo->state = ObjectState::ReadingInProgress; + + // We save/restore the current cursor around + // the callback, because we need to be able + // to return to the current state to continue + // reading whatever comes after the pointer + // we were invoked to read. + // + _pushState(); + _state.type = State::Type::Object; + _state.baseValue = objectInfo->fossilizedObjectRef; + _state.elementIndex = 0; + _state.elementCount = 1; + + // Note that we are passing the address of `objectInfo.ptr`, + // and `objectInfo` is a reference to an element of the + // `_objects` array. Thus whenever the `callback` stores + // a pointer into that output parameter, the value it writes + // will automatically be visible to any subsequent calls + // to `handleSharedPtr()`, even if they occur before + // `callback` returns. + // + // Thus a "true" circularity can only occur if the callback + // recursively reads a reference to the same object again + // *before* it allocates the in-memory representation of + // that objects and stores a pointer to it into the output + // parameter. + // + callback(&objectInfo->resurrectedObjectPtr, userData); + + _popState(); + + objectInfo->state = ObjectState::ReadingComplete; + + value = objectInfo->resurrectedObjectPtr; +} + +void SerialReader::handleUniquePtr(void*& value, Callback callback, void* userData) +{ + // We treat all pointers as shared pointers, because there isn't really + // an optimized representation we would want to use for the unique case. + // + handleSharedPtr(value, callback, userData); +} + +void SerialReader::handleDeferredObjectContents(void* valuePtr, Callback callback, void* userData) +{ + // Unlike the case in `SerialWriter::handleDeferredObjectContents()`, + // we very much *do* want to delay invoking the callback until later. + // + // There is a kind of symmetry going on, where the writer delays the + // callback passed to `handleSharedPtr()`, but *not* the callback + // passed to `handleDeferredObjectContents()`, while the reader + // does the opposite: immediately calls the callback in `handleSharedPtr()` + // but delays calling it here. + + // We make sure to save the current `_cursor` value along with + // the arguments that will be passed into the callback, so that + // we can restore the reader to this state before invoking + // the callbak in `_flush()`. + + DeferredAction deferredAction; + deferredAction.savedState = _state; + deferredAction.resurrectedObjectPtr = valuePtr; + deferredAction.callback = callback; + deferredAction.userData = userData; + + _deferredActions.add(deferredAction); +} + +void SerialReader::_flush() +{ + // We need to flush any actions that were deferred + // and are still pending. + // + while (_deferredActions.getCount() != 0) + { + // TODO: For simplicity we are using the `_deferredActions` + // array as a stack (LIFO), but it would be good to + // check whether there is a menaingful difference in how + // large the array would need to grow for a FIFO vs. LIFO, + // and pick the better option. + // + auto deferredAction = _deferredActions.getLast(); + _deferredActions.removeLast(); + + _state = deferredAction.savedState; + deferredAction.callback(deferredAction.resurrectedObjectPtr, deferredAction.userData); + } +} + +FossilizedValRef SerialReader::_readValRef() +{ + switch (_state.type) + { + case State::Type::Root: + case State::Type::Object: + SLANG_ASSERT(_state.elementCount == 1); + SLANG_ASSERT(_state.elementIndex == 0); + _state.elementIndex++; + return _state.baseValue; + + case State::Type::Struct: + case State::Type::Tuple: + { + SLANG_ASSERT(_state.elementIndex < _state.elementCount); + auto index = _state.elementIndex++; + + auto recordRef = as<FossilizedRecordVal>(_state.baseValue); + return getField(recordRef, index); + } + + case State::Type::Optional: + { + SLANG_ASSERT(_state.elementCount == 1); + SLANG_ASSERT(_state.elementIndex == 0); + + auto optionalRef = as<FossilizedOptionalObj>(_state.baseValue); + return getValue(optionalRef); + } + + case State::Type::Array: + case State::Type::Dictionary: + { + SLANG_ASSERT(_state.elementIndex < _state.elementCount); + auto index = _state.elementIndex++; + + auto containerRef = as<FossilizedContainerObj>(_state.baseValue); + return getElement(containerRef, index); + } + + default: + SLANG_UNEXPECTED("unhandled case"); + break; + } +} + +FossilizedValRef SerialReader::_readIndirectValRef() +{ + auto ptrValRef = _readValRef(); + auto ptrRef = as<FossilizedPtrVal>(ptrValRef); + + auto valRef = getPtrTarget(ptrRef); + return valRef; +} + + +FossilizedValRef SerialReader::_readPotentiallyIndirectValRef() +{ + auto valRef = _readValRef(); + if (auto ptrRef = as<FossilizedPtrVal>(valRef)) + { + return getPtrTarget(ptrRef); + } + return valRef; +} + +void SerialReader::_pushState() +{ + _stack.add(_state); +} + +void SerialReader::_popState() +{ + SLANG_ASSERT(_stack.getCount() != 0); + _state = _stack.getLast(); + _stack.removeLast(); +} + +} // namespace Fossil +} // namespace Slang diff --git a/source/slang/slang-serialize-fossil.h b/source/slang/slang-serialize-fossil.h new file mode 100644 index 000000000..0393c5784 --- /dev/null +++ b/source/slang/slang-serialize-fossil.h @@ -0,0 +1,741 @@ +// slang-serialize-fossil.h +#ifndef SLANG_SERIALIZE_FOSSIL_H +#define SLANG_SERIALIZE_FOSSIL_H + +// +// This file provides implementations of `ISerializerImpl` that +// serialize hierarchical data in the "memory-mappable" binary +// format defined in `slang-fossil.h`. +// + +#include "../core/slang-blob-builder.h" +#include "../core/slang-internally-linked-list.h" +#include "../core/slang-io.h" +#include "../core/slang-memory-arena.h" +#include "../core/slang-relative-ptr.h" +#include "slang-fossil.h" +#include "slang-serialize.h" + +namespace Slang +{ +namespace Fossil +{ + +/// Serializer implementation for writing objects to a fossil-format blob. +struct SerialWriter : ISerializerImpl +{ +public: + SerialWriter(ChunkBuilder* chunk); + SerialWriter(BlobBuilder& blobBuilder); + + ~SerialWriter(); + +private: + SerialWriter() = delete; + + void _initialize(ChunkBuilder* chunk); + + // The fossil format stores layout information, but that + // information is kept separate from the values themselves. + // + // The nature of the `ISerializer` interface means that we + // can only discover the layout as it is first being written, + // so we need an intermediate representation of layouts + // that we compute during the serialization process, before + // we can write those layouts out as their own bytes. + // + // Two related issues make this task a little intricate: + // + // * We don't want to redundantly serialize many copies of + // the same layout (since the whole point of keeping the + // layout information separate from the content is to + // save on space), and ideally we don't want to *create* + // a large number of intermediate layouts that will end + // up getting deduplicated out of existence. + // + // * If the same C++ type is getting serialized multiple times + // (e.g., in a loop serializing an array) we both want to + // re-use the layout from the first element for subsequent + // elements *and* we want to handle the case where parts of + // the layout get expanded on subsequent iterations (e.g., + // the first element in an array might have contained a null + // pointer, so there is no layout info for what it points to, + // but a later element might fill in that gap). + // + // The `_mergeLayouts()` operation is central to how these + // issues are handled, allowing code to attach new information + // to an existing layout as it goes. + + /// Representation of a layout for data that has been serialized. + class LayoutObj + { + public: + LayoutObj(FossilizedValKind kind, Size size = 0, Size alignment = 1) + : kind(kind), size(size), alignment(alignment) + { + } + + virtual ~LayoutObj() {} + + FossilizedValKind getKind() const { return kind; } + + Size getSize() const { return size; } + Size getAlignment() const { return alignment; } + + FossilizedValKind kind; + Size size = 0; + Size alignment = 1; + + /// If this layout is getting serialized out, then this + /// is a pointer to the chunk that will store the `FossilizedValLayout`. + /// + ChunkBuilder* chunk = nullptr; + }; + + /// Create a layout of the given `kind`. + /// + /// If `kind` is one of the simple layout kinds, then this will + /// return a singleton layout. + /// + LayoutObj* _createLayout(FossilizedValKind kind); + + LayoutObj* _createSimpleLayout(FossilizedValKind kind); + Dictionary<FossilizedValKind, LayoutObj*> _simpleLayouts; + + // Rather than try to do detailed memory management for + // layouts, we simply allocate them from an arena. + + MemoryArena _arena; + + /// Merge the `dst` layout object with the given `kind`. + /// + /// This more or less ensures that the layout *exists* + /// and has the right kind. + /// + /// If `dst` is null, it will be initialized via `_createLayout`. + /// + /// If `dst` is non-null, it will be checked against `kind`. + /// + LayoutObj* _mergeLayout(LayoutObj*& dst, FossilizedValKind kind); + + /// Merge the `src` layout into the `dst` layout. + /// + /// If `dst` is null, sets it to `src`. + /// + /// If `dst` is non-null, validates that `dst` and + /// `src` have the same kind, and then may recursively + /// merge their contents (e.g., if both are arrays, + /// it will merge the element layouts). + /// + void _mergeLayout(LayoutObj*& dst, LayoutObj* src); + + /// Layout for simple types (integers, strings, etc.) + class SimpleLayoutObj : public LayoutObj + { + public: + SimpleLayoutObj(FossilizedValKind kind, Size size) + : LayoutObj(kind, size, size) + { + } + + SimpleLayoutObj(FossilizedValKind kind) + : LayoutObj(kind) + { + } + }; + + /// Layouts for objects that have one conceptual type parameter. + /// + /// The obvious cases include pointers, arrays, and optionals. + /// + /// This is also used for dictionaries (the element type is + /// a pair). + /// + /// This is also used for variants (the element type is the type + /// of data that a *particular* variant used, whether or not + /// it matches any others). + /// + class ContainerLayoutObj : public LayoutObj + { + public: + ContainerLayoutObj( + FossilizedValKind kind, + LayoutObj* baseLayout, + Size size = 0, + Size alignment = 1) + : LayoutObj(kind, size, alignment), baseLayout(baseLayout) + { + } + + LayoutObj* baseLayout = nullptr; + }; + + /// Layouts for tuples and structs. + /// + class RecordLayoutObj : public LayoutObj + { + public: + RecordLayoutObj(FossilizedValKind kind) + : LayoutObj(kind) + { + } + + struct FieldInfo + { + LayoutObj* layout = nullptr; + Size offset = 0; + }; + + Count fieldCount = 0; + Count fieldCapacity = 0; + FieldInfo* fields = nullptr; + }; + + /// Get or add a field to the given `recordLayout` at the given `index`. + /// + /// If there is not already a field at `index`, then `index` must be + /// equal to the number of existing fields. + /// + RecordLayoutObj::FieldInfo& _getOrAddField(RecordLayoutObj* layout, Index index); + + // The serialized representation only references layouts as part of + // its encoding of variants, with each variant having a prefix field + // that is a relative pointer to its serialized layout. + // + // Because we want to deduplicate layouts, we keep track of all of + // the variant values we have serialized (each of which should be its + // own chunk), and use that array to come back later and write out + // their final layouts (after deduplication). + + struct VariantInfo + { + LayoutObj* layout = nullptr; + ChunkBuilder* chunk = nullptr; + }; + List<VariantInfo> _variants; + + /// Create a chunk to represent `layout`, or return a pre-existing one. + ChunkBuilder* _getOrCreateChunkForLayout(LayoutObj* layout); + + /// Key for deduplication of `LayoutObj`s. + struct LayoutObjKey + { + LayoutObjKey() {} + + LayoutObjKey(LayoutObj* obj) + : obj(obj) + { + } + + LayoutObj* obj = nullptr; + + bool operator==(LayoutObjKey const& that) const; + bool operator!=(LayoutObjKey const& that) const; + + HashCode64 getHashCode() const; + void hashInto(Hasher& hasher) const; + }; + Dictionary<LayoutObjKey, ChunkBuilder*> _mapLayoutObjToChunk; + + // We also go ahead and deduplicate strings as part of serialization, + // since it is easy to do so. + + Dictionary<String, ChunkBuilder*> _mapStringToChunk; + + // Like almost any implementation of `ISerializer`, we need to track + // information on the objects that have been encountered on the other + // side of pointers, so that we can delay serializing their contents + // until an appropriate time. + + struct FossilizedObjectInfo + { + /// Pointer to the "live" object. + void* liveObjectPtr = nullptr; + + /// Chunk that will store the bytes of the fossilized object. + ChunkBuilder* chunk = nullptr; + + /// Layout for a pointer to the fossilized `chunk`. + LayoutObj* ptrLayout = nullptr; + + /// Callback information used by the ISerializer interface. + Callback callback = nullptr; + void* userData = nullptr; + }; + + List<FossilizedObjectInfo*> _fossilizedObjects; + Dictionary<void*, FossilizedObjectInfo*> _mapLiveObjectPtrToFossilizedObject; + Index _writtenObjectDefinitionCount = 0; + + /// Flush all pending operations. + /// + /// This function ensures that all of the to-be-writen objects have + /// been written out, and that all of the variants that need a pointer + /// to a serialized layout get one. + /// + void _flush(); + + // + // As the user makes various begin/end calls on this `SerialWriter`, + // we need to push/pop state information so that we don't lose it. + // + + struct State + { + /// The layout for the value being composed. + LayoutObj* layout = nullptr; + + /// The number of elements/fields or other sub-values written so var. + Count elementCount = 0; + + /// The chunk that holds the data for the value. + /// + /// Can be null if nothing has been written yet, in which + /// case it may be allocated on teh first write. + /// + ChunkBuilder* chunk = nullptr; + + State() {} + + State(LayoutObj* layout, ChunkBuilder* chunk = nullptr) + : layout(layout), chunk(chunk) + { + } + }; + + /// The current state. + State _state; + + /// Stack of suspended states. + List<State> _stack; + + /// The underlying blob builder that we are writing to. + BlobBuilder* _blobBuilder = nullptr; + + // + // Depending on the kind of value being written, it may + // require a different representation. The `Val + + /// Represents a conceptual value to be written. + /// + /// Depending on the kind of value being written, it may + /// require a different representation. The `ValInfo` type + /// abstracts over these differences. + /// + /// Simple values that just consist of bytes can use the + /// `RawData` case. + /// + /// Values that are encoded as a relative pointer use the + /// `RelativePtr` case (unsurprisingly). + /// + /// The `ContentsOfChunk` case is used when the conceptual + /// value is some kind of aggregate that is stored inline + /// rather than indirectly. + /// + struct ValInfo + { + public: + enum class Kind + { + RawData, + RelativePtr, + ContentsOfChunk, + }; + + static ValInfo rawData(void const* data, Size size, Size alignment); + static ValInfo relativePtrTo(ChunkBuilder* targetChunk); + static ValInfo contentsOf(ChunkBuilder* chunk); + + Size getAlignment() const; + + Kind kind; + union + { + struct + { + void const* ptr; + Size size; + Size alignment; + } data; + ChunkBuilder* chunk; + }; + + private: + ValInfo() = default; + ValInfo(const ValInfo&) = default; + ValInfo(ValInfo&&) = default; + ValInfo(Kind kind) + : kind(kind) + { + } + }; + + // In order to allow building up layout information as values are + // being written, the process of writing a value is broken into + // two parts: + // + // * First, the code conceptually "reserves" a destination for the + // value it will write, passing in what it knows about the expected + // layout for the value. The reserve operation returns a layout + // to use (which may be a pre-existing one). + // + // * Second, once the value is ready as a `ValInfo`, the code "commits" + // the write and puts actual data in a chunk somewhere. + // + // For simple values these operations occur on after the other in + // the same function. For complex things that need a begin/end pair, + // the reserve usually happens in a `begin*()` or `push*()` function, + // while the commit happens in an `end*()` or `pop*()` function. + + LayoutObj*& _reserveDestinationForWrite(); + LayoutObj* _reserveDestinationForWrite(FossilizedValKind srcKind); + LayoutObj* _reserveDestinationForWrite(LayoutObj* srcLayout); + + void _commitWrite(ValInfo const& val); + + /// Write a value without doing any of the checks that `_commitWrite` does. + /// + /// (Usually this is called because `_commitWrite()` has already been called) + void _writeValueRaw(ValInfo const& val); + + /// Ensure that the current `State` has a non-null chunk that data + /// can be written to. + /// + void _ensureChunkExists(); + + // There are various different categories of values that each + // need slightly different handling, so each gets its own + // operations that the various `ISerializer::begin()/end()` + // functions will delegate to. + // + // The easiest case is simple values that consist of nothing + // but plain data and have a layout that can be fully summarized + // by the kind. + + void _writeSimpleValue(FossilizedValKind kind, void const* data, size_t size, size_t alignment); + + template<typename T> + void _writeSimpleValue(FossilizedValKind kind, T const& value) + { + _writeSimpleValue(kind, &value, sizeof(value), sizeof(value)); + } + + /// Write a null (relative) pointer. + /// + /// Use this case when there is no more refined type information + /// available about what the layout of the pointed-to data *would* + /// be if the pointer were non-null. + /// + void _writeNull(); + + // + // "Inline" values are aggregates like tuple and structs that + // are always stored by-value in their parent. + // + + void _pushInlineValueScope(FossilizedValKind kind); + void _popInlineValueScope(); + + // + // "Indirect" values are those like optionals that are + // stored as a pointer to an (optional) out-of-line value. + // + + void _pushIndirectValueScope(FossilizedValKind kind); + void _popIndirectValueScope(); + + // + // Many cases of values are *potentially* indirect, in that + // they should be stored via pointer indirection *unless* + // their immediate parent is something that already introduced + // an indirection. + // + // A simple example is a string. A string will by default + // be stored as a (relative) pointer to its content. However, + // if there happens to be an *optional* string, then there is + // no need for a second indirection. + // + // Arrays, dictionaries, strings, and variants are all + // potentially-indirect values. + // + // TODO: This is one aspect of the current design that may need + // to be revisited, if it proves to add too much complexity. + // + + void _pushPotentiallyIndirectValueScope(FossilizedValKind kind); + ChunkBuilder* _popPotentiallyIndirectValueScope(); + + /// Determine if a potentially-indirect value of `kind` should be + /// emitted indirectly, in the current state. + /// + bool _shouldEmitWithPointerIndirection(FossilizedValKind kind); + + /// Helper function to share details between `_popIndirectValueScope` + /// and `_popPotentiallyIndirectValueScope`. + /// + ChunkBuilder* _writeKnownIndirectValueSharedLogic(ChunkBuilder* valueChunk); + + // + // Containers like arrays and dictionaries are potentially-indirect + // values where the chunk that stores their content needs to + // be given a prefix with the element count. + // + + void _pushContainerScope(FossilizedValKind kind); + void _popContainerScope(); + + // + // A variant is a potentially-indirect value where the chunk + // that stores its content needs to be given a prefix with + // the layout of the content. + // + + void _pushVariantScope(); + void _popVariantScope(); + + // + // All of the above operations ultimately bottleneck through + // `_pushState()`/`_popState()`. + // + + void _pushState(LayoutObj* layout); + void _popState(); + +private: + // + // The following declarations are the requirements + // of the `ISerializerImpl` interface: + // + + virtual SerializationMode getMode() override; + + virtual void handleBool(bool& value) override; + + virtual void handleInt8(int8_t& value) override; + virtual void handleInt16(int16_t& value) override; + virtual void handleInt32(Int32& value) override; + virtual void handleInt64(Int64& value) override; + + virtual void handleUInt8(uint8_t& value) override; + virtual void handleUInt16(uint16_t& value) override; + virtual void handleUInt32(UInt32& value) override; + virtual void handleUInt64(UInt64& value) override; + + virtual void handleFloat32(float& value) override; + virtual void handleFloat64(double& value) override; + + virtual void handleString(String& value) override; + + virtual void beginArray() override; + virtual void endArray() override; + + virtual void beginOptional() override; + virtual void endOptional() override; + + virtual void beginDictionary() override; + virtual void endDictionary() override; + + virtual bool hasElements() override; + + virtual void beginTuple() override; + virtual void endTuple() override; + + virtual void beginStruct() override; + virtual void endStruct() override; + + virtual void beginVariant() override; + virtual void endVariant() override; + + virtual void handleFieldKey(char const* name, Int index) override; + + virtual void handleSharedPtr(void*& value, Callback callback, void* userData) override; + virtual void handleUniquePtr(void*& value, Callback callback, void* userData) override; + + virtual void handleDeferredObjectContents(void* valuePtr, Callback callback, void* userData) + override; +}; + +/// Serializer implementation for reading objects from a fossil-format blob. +struct SerialReader : ISerializerImpl +{ +public: + SerialReader(FossilizedValRef valRef); + ~SerialReader(); + +private: + /// A state that the reader can be in. + struct State + { + /// Type of state; related to the kind of value being read from. + /// + enum class Type + { + Root, + Array, + Dictionary, + Optional, + Tuple, + Struct, + Object, + }; + + /// The type of state. + Type type = Type::Root; + + /// The fossilized value (data and layout) that is being read from. + /// + /// Depending on the `type` of state, this might either be the next value + /// that will be read (e.g., for the `Root` case), or it might be + /// a container that is a parent of the next value to be read. + /// + FossilizedValRef baseValue; + + /// Index of next element to read. + /// + /// This is used in the case where `baseValue` is some kind of + /// container or record. + /// + Index elementIndex = 0; + + /// Total number of values that can be read. + /// + /// If `baseValue` is a container, this is the element count. + /// If `baseValue` is a tuple/struct, this is the field count. + /// If `baseValue` is an optional, this is either zero or one. + /// If this state is a singleton case like `Root`, will be one. + /// + Count elementCount = 0; + }; + + /// The current state. + State _state; + + /// Stack of saved states. + List<State> _stack; + + void _pushState(); + void _popState(); + + // + // Like other `ISerializerImpl`s for reading, we track objects + // that are in the process of being read in, to avoid possible + // unbounded recursion (and detect circularities when they + // occur). + // + + enum class ObjectState + { + Unread, + ReadingInProgress, + ReadingComplete, + }; + struct ObjectInfo : public RefObject + { + ObjectState state = ObjectState::Unread; + + void* resurrectedObjectPtr = nullptr; + FossilizedValRef fossilizedObjectRef; + }; + Dictionary<void*, RefPtr<ObjectInfo>> _mapFossilizedObjectPtrToObjectInfo; + + // + // Again, like other `ISerializerImpl`s for reading, we + // maintain a list of deferred serialization actions that + // need to be performed to finish reading the state of + // in-memory objects. + // + + struct DeferredAction + { + void* resurrectedObjectPtr; + + State savedState; + + Callback callback; + void* userData; + }; + List<DeferredAction> _deferredActions; + + /// Execute all deferred actions that are still pending. + void _flush(); + + /// Read a simple/inline value. + /// + /// This is the case for scalars, tuples, and structs. + /// + FossilizedValRef _readValRef(); + + /// Read an indirect value. + /// + /// This is the case for things like optionals, that are + /// always encoded as a pointer. + /// + FossilizedValRef _readIndirectValRef(); + + /// Read a potentially-indirect value. + /// + /// If the value that gets read is a pointer, then this + /// function will return a reference to whatever it points to. + /// + /// Otherwise, this will return a reference to the value itself. + /// + FossilizedValRef _readPotentiallyIndirectValRef(); + +private: + // + // The following declarations are the requirements + // of the `ISerializerImpl` interface: + // + + virtual SerializationMode getMode() override; + + virtual void handleBool(bool& value) override; + + virtual void handleInt8(int8_t& value) override; + virtual void handleInt16(int16_t& value) override; + virtual void handleInt32(Int32& value) override; + virtual void handleInt64(Int64& value) override; + + virtual void handleUInt8(uint8_t& value) override; + virtual void handleUInt16(uint16_t& value) override; + virtual void handleUInt32(UInt32& value) override; + virtual void handleUInt64(UInt64& value) override; + + virtual void handleFloat32(float& value) override; + virtual void handleFloat64(double& value) override; + + virtual void handleString(String& value) override; + + virtual void beginArray() override; + virtual void endArray() override; + + virtual void beginDictionary() override; + virtual void endDictionary() override; + + virtual bool hasElements() override; + + virtual void beginStruct() override; + virtual void endStruct() override; + + virtual void beginVariant() override; + virtual void endVariant() override; + + virtual void handleFieldKey(char const* name, Int index) override; + + virtual void beginTuple() override; + virtual void endTuple() override; + + virtual void beginOptional() override; + virtual void endOptional() override; + + virtual void handleSharedPtr(void*& value, Callback callback, void* userData) override; + virtual void handleUniquePtr(void*& value, Callback callback, void* userData) override; + + virtual void handleDeferredObjectContents(void* valuePtr, Callback callback, void* userData) + override; +}; + +} // namespace Fossil +} // namespace Slang + +#endif diff --git a/source/slang/slang-serialize-riff.cpp b/source/slang/slang-serialize-riff.cpp index 01b39e825..469803c2a 100644 --- a/source/slang/slang-serialize-riff.cpp +++ b/source/slang/slang-serialize-riff.cpp @@ -174,6 +174,21 @@ void RIFFSerialWriter::beginStruct() _cursor.beginListChunk(RIFFSerial::kStructFourCC); } +void RIFFSerialWriter::endStruct() +{ + _cursor.endChunk(); +} + +void RIFFSerialWriter::beginVariant() +{ + beginStruct(); +} + +void RIFFSerialWriter::endVariant() +{ + endStruct(); +} + void RIFFSerialWriter::handleFieldKey(char const* name, Int index) { // For now we are ignoring field keys, and treating @@ -182,11 +197,6 @@ void RIFFSerialWriter::handleFieldKey(char const* name, Int index) SLANG_UNUSED(index); } -void RIFFSerialWriter::endStruct() -{ - _cursor.endChunk(); -} - void RIFFSerialWriter::beginTuple() { _cursor.beginListChunk(RIFFSerial::kTupleFourCC); @@ -506,6 +516,21 @@ void RIFFSerialReader::beginStruct() _beginListChunk(RIFFSerial::kStructFourCC); } +void RIFFSerialReader::endStruct() +{ + _endListChunk(); +} + +void RIFFSerialReader::beginVariant() +{ + beginStruct(); +} + +void RIFFSerialReader::endVariant() +{ + endStruct(); +} + void RIFFSerialReader::handleFieldKey(char const* name, Int index) { // For now we are ignoring field keys, and treating @@ -514,11 +539,6 @@ void RIFFSerialReader::handleFieldKey(char const* name, Int index) SLANG_UNUSED(index); } -void RIFFSerialReader::endStruct() -{ - _endListChunk(); -} - void RIFFSerialReader::beginTuple() { _beginListChunk(RIFFSerial::kTupleFourCC); diff --git a/source/slang/slang-serialize-riff.h b/source/slang/slang-serialize-riff.h index a464a5ded..87f83f3f0 100644 --- a/source/slang/slang-serialize-riff.h +++ b/source/slang/slang-serialize-riff.h @@ -224,9 +224,13 @@ private: virtual bool hasElements() override; virtual void beginStruct() override; - virtual void handleFieldKey(char const* name, Int index) override; virtual void endStruct() override; + virtual void beginVariant() override; + virtual void endVariant() override; + + virtual void handleFieldKey(char const* name, Int index) override; + virtual void beginTuple() override; virtual void endTuple() override; @@ -410,9 +414,13 @@ private: virtual bool hasElements() override; virtual void beginStruct() override; - virtual void handleFieldKey(char const* name, Int index) override; virtual void endStruct() override; + virtual void beginVariant() override; + virtual void endVariant() override; + + virtual void handleFieldKey(char const* name, Int index) override; + virtual void beginTuple() override; virtual void endTuple() override; diff --git a/source/slang/slang-serialize.h b/source/slang/slang-serialize.h index d76ab8338..b962ee2b7 100644 --- a/source/slang/slang-serialize.h +++ b/source/slang/slang-serialize.h @@ -335,6 +335,30 @@ struct ISerializerImpl /// End serializing a struct value. virtual void endStruct() = 0; + /// Begin serializing a variant value. + /// + /// A variant should be used to serialize any type + /// that behaves like a "tagged union," where different + /// instances may have different sequences of members, + /// of different types. + /// + /// User code reading from a variant must be able to + /// use the members read so far to determine what + /// members it should read next (e.g., by serializing + /// a tag enumerant first, followed by the tag-dependent + /// members). + /// + /// A variant is otherwise like a struct. Some serializer + /// implementations may treat variants just like structs, + /// while others may rely on any type serialized as a + /// struct always including the same members in the same + /// order. + /// + virtual void beginVariant() = 0; + + /// End serializing a variant value. + virtual void endVariant() = 0; + /// Set the key for the next struct field to be serialized. /// /// If no name is available for the field, `name` may be `nullptr`. @@ -623,6 +647,21 @@ private: Serializer _serializer; }; +struct ScopedSerializerVariant +{ +public: + ScopedSerializerVariant(Serializer const& serializer) + : _serializer(serializer) + { + serializer->beginVariant(); + } + + ~ScopedSerializerVariant() { _serializer->endVariant(); } + +private: + Serializer _serializer; +}; + struct ScopedSerializerTuple { public: @@ -667,8 +706,8 @@ private: #define SLANG_SCOPED_SERIALIZER_STRUCT(SERIALIZER) \ ::Slang::ScopedSerializerStruct SLANG_CONCAT(_scopedSerializerStruct, __LINE__)(SERIALIZER) -#define SLANG_SCOPED_SERIALIZER_TAGGED_UNION(SERIALIZER) \ - ::Slang::ScopedSerializerStruct SLANG_CONCAT(_scopedSerializerStruct, __LINE__)(SERIALIZER) +#define SLANG_SCOPED_SERIALIZER_VARIANT(SERIALIZER) \ + ::Slang::ScopedSerializerVariant SLANG_CONCAT(_scopedSerializerVariant, __LINE__)(SERIALIZER) #define SLANG_SCOPED_SERIALIZER_TUPLE(SERIALIZER) \ ::Slang::ScopedSerializerTuple SLANG_CONCAT(_scopedSerializerTuple, __LINE__)(SERIALIZER) |