diff options
40 files changed, 2684 insertions, 1981 deletions
diff --git a/docs/user-guide/a3-02-reference-capability-atoms.md b/docs/user-guide/a3-02-reference-capability-atoms.md index f0345679c..4f4a1a12b 100644 --- a/docs/user-guide/a3-02-reference-capability-atoms.md +++ b/docs/user-guide/a3-02-reference-capability-atoms.md @@ -152,6 +152,9 @@ Versions `hlsl_nvapi` > Represents HLSL NVAPI support. +`hlsl_2018` +> Represet HLSL compatibility support. + `dxil_lib` > Represents capabilities required for DXIL Library compilation. diff --git a/examples/ray-tracing/main.cpp b/examples/ray-tracing/main.cpp index 6a0abf8b4..fffe5e75f 100644 --- a/examples/ray-tracing/main.cpp +++ b/examples/ray-tracing/main.cpp @@ -423,13 +423,13 @@ struct RayTracing : public WindowedAppBase IBufferResource::Desc asBufferDesc; asBufferDesc.type = IResource::Type::Buffer; asBufferDesc.defaultState = ResourceState::AccelerationStructure; - asBufferDesc.sizeInBytes = (Size)compactedSize; + asBufferDesc.sizeInBytes = (gfx::Size)compactedSize; gBLASBuffer = gDevice->createBufferResource(asBufferDesc); IAccelerationStructure::CreateDesc createDesc; createDesc.buffer = gBLASBuffer; createDesc.kind = IAccelerationStructure::Kind::BottomLevel; createDesc.offset = 0; - createDesc.size = (Size)compactedSize; + createDesc.size = (gfx::Size)compactedSize; gDevice->createAccelerationStructure(createDesc, gBLAS.writeRef()); commandBuffer = gTransientHeaps[0]->createCommandBuffer(); diff --git a/source/core/slang-char-encode.h b/source/core/slang-char-encode.h index 74968a6ab..2ba8ba992 100644 --- a/source/core/slang-char-encode.h +++ b/source/core/slang-char-encode.h @@ -21,7 +21,7 @@ template<typename ReadByteFunc> Char32 getUnicodePointFromUTF8(const ReadByteFunc& readByte) { Char32 codePoint = 0; - uint32_t leading = Byte(readByte()); + uint32_t leading = uint32_t(readByte()); uint32_t mask = 0x80; Index count = 0; while (leading & mask) diff --git a/source/core/slang-common.h b/source/core/slang-common.h index 2aec6f0ce..9248e250f 100644 --- a/source/core/slang-common.h +++ b/source/core/slang-common.h @@ -4,6 +4,7 @@ #include "slang.h" #include <assert.h> +#include <cstddef> #include <stdint.h> #define VARIADIC_TEMPLATE @@ -11,35 +12,150 @@ namespace Slang { +/// Signed 32-bit integer. +/// +/// This type should be used when the exact size +/// in bits is important (e.g., when dealing with +/// explicit binary file formats, etc.). Otherwise +/// prefer the plain `Int` type or a semantically +/// richer type like `Count` or `Index`. +/// typedef int32_t Int32; + +/// Unsigned 32-bit integer. +/// +/// This type should be used when the exact size +/// in bits is important (e.g., when dealing with +/// explicit binary file formats, etc.). Otherwise +/// prefer the plain `Int` type or a semantically +/// richer type like `Count` or `Index`. +/// typedef uint32_t UInt32; +/// Signed 64-bit integer. +/// +/// This type should be used when the exact size +/// in bits is important (e.g., when dealing with +/// explicit binary file formats, etc.). Otherwise +/// prefer the plain `Int` type or a semantically +/// richer type like `Count` or `Index`. +/// typedef int64_t Int64; + +/// Unsigned 64-bit integer. +/// +/// This type should be used when the exact size +/// in bits is important (e.g., when dealing with +/// explicit binary file formats, etc.). Otherwise +/// prefer the plain `Int` type or a semantically +/// richer type like `Count` or `Index`. +/// typedef uint64_t UInt64; -// Define -typedef SlangUInt UInt; +/// "Default" integer type for the Slang codebase. +/// +/// When there is not a clear reason to another +/// integer type, use this one. +/// +/// Note that this type is currently defined to be +/// the same as the `SlangInt` type exposed through +/// the public Slang API, but this may not be the +/// case forever. +/// typedef SlangInt Int; +/// "Default" unsigned integer type for the Slang codebase. +/// +/// Only use this type when you explicitly need +/// an unsigned type that's the same size as `Int`. +/// Otherwise you should probably just be using `Int`. +/// +/// Note that this type is currently defined to be +/// the same as the `SlangUInt` type exposed through +/// the public Slang API, but this may not be the +/// case forever. +/// +typedef SlangUInt UInt; + static const UInt kMaxUInt = ~UInt(0); static const Int kMaxInt = Int(kMaxUInt >> 1); -// typedef unsigned short Word; - typedef intptr_t PtrInt; -// TODO(JS): It looks like Index is actually 64 bit on 64 bit targets(!) -// Previous discussions landed on Index being int32_t. - -// Type used for indexing, in arrays/views etc. Signed. +/// Default type for indices. +/// +/// This is (and should always be) an alias for `Int`. +/// +/// Use this type to document the intention that an +/// integer parameter/variable/etc. represents an +/// index into some kind of sequence, or any other +/// kind of ordinal number. +/// typedef Int Index; + +static const Index kMaxIndex = kMaxInt; + +/// Unsigned equivalent of `Index`. +/// +/// Please don't use this unless you have a good reason. +/// typedef UInt UIndex; + +/// Default type for counts. +/// +/// This is (and should always be) an alias for `Int`. +/// +/// Use this type to document the intention that an +/// integer parameter/variable/etc. represents a +/// count of the number of elements in some container, +/// or any other kind of cardinal number. +/// typedef Int Count; + +/// Unsigned equivalent of `Count`. +/// +/// Please don't use this unless you have a good reason. +/// typedef UInt UCount; -static const Index kMaxIndex = kMaxInt; -typedef uint8_t Byte; +/// Explicit type for when manipulating bytes. +/// +/// Use this type to document the intention that a +/// parameter/variable/etc. represents an 8-bit byte, with +/// no particular interpretation of that byte as +/// any higher-level type. +/// +/// Note that the `char` types have special semantics +/// when it comes to "type punning" that are not shared +/// with other types like `uint8_t`. Using a variation +/// of `char` here helps avoid the possibility of undefined +/// behavior when code reads other types to/from arrays +/// of `Byte`s. +/// +/// We are not using `std::byte` here because that is +/// defined as an `enum class` and does not support +/// mathematical or bitwise operations, which a lot +/// of the Slang codebase does on `Byte`s. +/// +typedef unsigned char Byte; + +/// Preferred integer type for sizes measured in bytes. +/// +/// Use this type to document the intention that an +/// integer parameter/variable/etc. represents the +/// size of something, in bytes, rather than being +/// some other kind of integer. +/// +/// Note that this type is unsigned, despite the stated +/// default in the Slang codebase being signed integer +/// types. The reason for this is that variables +/// holding sizes are often compared against the +/// result of the `sizeof` operator, which yields +/// a `size_t`. Our hands are, to some extent, tied +/// on this matter. +/// +using Size = size_t; // TODO(JS): // Perhaps these should be named Utf8, Utf16 and UnicodePoint/Rune/etc? For now, just keep it simple diff --git a/source/core/slang-riff-file-system.cpp b/source/core/slang-riff-file-system.cpp index 7da076974..fc377cd4c 100644 --- a/source/core/slang-riff-file-system.cpp +++ b/source/core/slang-riff-file-system.cpp @@ -117,14 +117,10 @@ SlangResult RiffFileSystem::saveFileBlob(const char* path, ISlangBlob* dataBlob) SlangResult RiffFileSystem::loadArchive(const void* archive, size_t archiveSizeInBytes) { // Load the riff - RiffContainer container; + auto rootList = RIFF::RootChunk::getFromBlob(archive, archiveSizeInBytes); - MemoryStreamBase stream(FileAccess::Read, archive, archiveSizeInBytes); - SLANG_RETURN_ON_FAIL(RiffUtil::read(&stream, container)); - - RiffContainer::ListChunk* rootList = container.getRoot(); // Make sure it's the right type - if (rootList == nullptr || rootList->m_fourCC != RiffFileSystemBinary::kContainerFourCC) + if (rootList == nullptr || rootList->getType() != RiffFileSystemBinary::kContainerFourCC) { return SLANG_FAIL; } @@ -133,10 +129,11 @@ SlangResult RiffFileSystem::loadArchive(const void* archive, size_t archiveSizeI _clear(); // Find the header - const auto header = rootList->findContainedData<RiffFileSystemBinary::Header>( - RiffFileSystemBinary::kHeaderFourCC); + auto headerChunk = rootList->findDataChunk(RiffFileSystemBinary::kHeaderFourCC); + + const auto header = headerChunk->readPayloadAs<RiffFileSystemBinary::Header>(); - CompressionSystemType compressionType = CompressionSystemType(header->compressionSystemType); + CompressionSystemType compressionType = CompressionSystemType(header.compressionSystemType); switch (compressionType) { case CompressionSystemType::None: @@ -162,52 +159,56 @@ SlangResult RiffFileSystem::loadArchive(const void* archive, size_t archiveSizeI // Read all of the contained data { - List<RiffContainer::DataChunk*> srcEntries; - rootList->findContained(RiffFileSystemBinary::kEntryFourCC, srcEntries); - - for (auto chunk : srcEntries) + for (auto chunk : rootList->getChildren()) { - auto data = chunk->getSingleData(); + auto dataChunk = as<RIFF::DataChunk>(chunk); + if (!dataChunk) + continue; + + if (dataChunk->getType() != RiffFileSystemBinary::kEntryFourCC) + continue; - const uint8_t* srcData = (const uint8_t*)data->getPayload(); - const size_t dataSize = data->getSize(); + auto payloadData = (const uint8_t*)dataChunk->getPayload(); + auto payloadSize = dataChunk->getPayloadSize(); - if (dataSize < sizeof(RiffFileSystemBinary::Entry)) + if (payloadSize < sizeof(RiffFileSystemBinary::Entry)) { return SLANG_FAIL; } - auto srcEntry = (const RiffFileSystemBinary::Entry*)srcData; - srcData += sizeof(*srcEntry); + MemoryReader reader(payloadData, payloadSize); + + RiffFileSystemBinary::Entry srcEntry; + reader.read(srcEntry); // Check if seems plausible - if (sizeof(RiffFileSystemBinary::Entry) + srcEntry->compressedSize + - srcEntry->pathSize != - dataSize) + if (sizeof(RiffFileSystemBinary::Entry) + srcEntry.compressedSize + srcEntry.pathSize != + payloadSize) { return SLANG_FAIL; } Entry dstEntry; - const char* path = (const char*)srcData; - srcData += srcEntry->pathSize; + const char* path = (const char*)reader.getRemainingData(); + reader.skip(srcEntry.pathSize); - dstEntry.m_canonicalPath = UnownedStringSlice(path, srcEntry->pathSize - 1); - dstEntry.m_type = (SlangPathType)srcEntry->pathType; - dstEntry.m_uncompressedSizeInBytes = srcEntry->uncompressedSize; + dstEntry.m_canonicalPath = UnownedStringSlice(path, srcEntry.pathSize - 1); + dstEntry.m_type = (SlangPathType)srcEntry.pathType; + dstEntry.m_uncompressedSizeInBytes = srcEntry.uncompressedSize; switch (dstEntry.m_type) { case SLANG_PATH_TYPE_FILE: { - if (srcData + srcEntry->compressedSize != data->getPayloadEnd()) + if (reader.getRemainingSize() != srcEntry.compressedSize) { return SLANG_FAIL; } // Get the compressed data - dstEntry.m_contents = RawBlob::create(srcData, srcEntry->compressedSize); + dstEntry.m_contents = + RawBlob::create(reader.getRemainingData(), srcEntry.compressedSize); break; } case SLANG_PATH_TYPE_DIRECTORY: @@ -235,11 +236,9 @@ SlangResult RiffFileSystem::storeArchive(bool blobOwnsContent, ISlangBlob** outB // All blobs are owned in this style SLANG_UNUSED(blobOwnsContent) - RiffContainer container; - RiffContainer::ScopeChunk scopeContainer( - &container, - RiffContainer::Chunk::Kind::List, - RiffFileSystemBinary::kContainerFourCC); + RIFF::Builder builder; + RIFF::BuildCursor cursor(builder); + SLANG_SCOPED_RIFF_BUILDER_LIST_CHUNK(cursor, RiffFileSystemBinary::kContainerFourCC); { RiffFileSystemBinary::Header header; @@ -247,7 +246,7 @@ SlangResult RiffFileSystem::storeArchive(bool blobOwnsContent, ISlangBlob** outB ? m_compressionSystem->getSystemType() : CompressionSystemType::None; header.compressionSystemType = uint32_t(compressionSystemType); - container.addDataChunk(RiffFileSystemBinary::kHeaderFourCC, &header, sizeof(header)); + cursor.addDataChunk(RiffFileSystemBinary::kHeaderFourCC, &header, sizeof(header)); } for (const auto& [_, srcEntry] : m_entries) @@ -258,10 +257,7 @@ SlangResult RiffFileSystem::storeArchive(bool blobOwnsContent, ISlangBlob** outB continue; } - RiffContainer::ScopeChunk scopeData( - &container, - RiffContainer::Chunk::Kind::Data, - RiffFileSystemBinary::kEntryFourCC); + SLANG_SCOPED_RIFF_BUILDER_DATA_CHUNK(cursor, RiffFileSystemBinary::kEntryFourCC); RiffFileSystemBinary::Entry dstEntry; dstEntry.uncompressedSize = 0; @@ -278,41 +274,33 @@ SlangResult RiffFileSystem::storeArchive(bool blobOwnsContent, ISlangBlob** outB } // Entry header - container.write(&dstEntry, sizeof(dstEntry)); + cursor.addData(&dstEntry, sizeof(dstEntry)); // Path - container.write( + cursor.addData( srcEntry.m_canonicalPath.getBuffer(), srcEntry.m_canonicalPath.getLength() + 1); // Add the contained data without copying if (blob) { - RiffContainer::Data* data = container.addData(); - container.setUnowned( - data, + cursor.addUnownedData( const_cast<void*>(blob->getBufferPointer()), blob->getBufferSize()); } } - OwnedMemoryStream stream(FileAccess::Write); - // We now write the RiffContainer to the stream - SLANG_RETURN_ON_FAIL(RiffUtil::write(container.getRoot(), true, &stream)); - - List<uint8_t> data; - stream.swapContents(data); - - *outBlob = ListBlob::moveCreate(data).detach(); + SLANG_RETURN_ON_FAIL(builder.writeToBlob(outBlob)); return SLANG_OK; } /* static */ bool RiffFileSystem::isArchive(const void* data, size_t sizeInBytes) { - MemoryStreamBase stream(FileAccess::Read, data, sizeInBytes); - RiffListHeader header; - return SLANG_SUCCEEDED(RiffUtil::readHeader(&stream, header)) && - header.subType == RiffFileSystemBinary::kContainerFourCC; + auto rootList = RIFF::RootChunk::getFromBlob(data, sizeInBytes); + if (!rootList) + return false; + + return rootList->getType() == RiffFileSystemBinary::kContainerFourCC; } } // namespace Slang diff --git a/source/core/slang-riff-file-system.h b/source/core/slang-riff-file-system.h index 6eda1a371..f3eab9c14 100644 --- a/source/core/slang-riff-file-system.h +++ b/source/core/slang-riff-file-system.h @@ -11,9 +11,9 @@ namespace Slang // The riff information used for RiffArchiveFileSystem struct RiffFileSystemBinary { - static const FourCC kContainerFourCC = SLANG_FOUR_CC('S', 'c', 'o', 'n'); - static const FourCC kEntryFourCC = SLANG_FOUR_CC('S', 'f', 'i', 'l'); - static const FourCC kHeaderFourCC = SLANG_FOUR_CC('S', 'h', 'e', 'a'); + static const FourCC::RawValue kContainerFourCC = SLANG_FOUR_CC('S', 'c', 'o', 'n'); + static const FourCC::RawValue kEntryFourCC = SLANG_FOUR_CC('S', 'f', 'i', 'l'); + static const FourCC::RawValue kHeaderFourCC = SLANG_FOUR_CC('S', 'h', 'e', 'a'); struct Header { diff --git a/source/core/slang-riff.cpp b/source/core/slang-riff.cpp index c1e3e81c3..be15497f3 100644 --- a/source/core/slang-riff.cpp +++ b/source/core/slang-riff.cpp @@ -1,1024 +1,878 @@ +// slang-riff.cpp #include "slang-riff.h" +#include "slang-blob.h" #include "slang-com-helper.h" -#include "slang-hex-dump-util.h" namespace Slang { +namespace RIFF +{ -/* static */ int64_t RiffUtil::calcChunkTotalSize(const RiffHeader& chunk) +Size _roundUpToChunkAlignment(Size size) { - size_t size = chunk.size + sizeof(RiffHeader); - return getPadSize(size); + auto alignmentMask = Size(Chunk::kChunkAlignment) - 1; + return (size + alignmentMask) & ~alignmentMask; } -/* static */ SlangResult RiffUtil::skip( - const RiffHeader& chunk, - Stream* stream, - int64_t* remainingBytesInOut) -{ - int64_t chunkSize = calcChunkTotalSize(chunk); - if (remainingBytesInOut) - { - *remainingBytesInOut -= chunkSize; - } +// +// RIFF::Chunk +// - // Skip the payload (we don't need to skip the Chunk because that was already read - SLANG_RETURN_ON_FAIL(stream->seek(SeekOrigin::Current, chunkSize - sizeof(RiffHeader))); - return SLANG_OK; -} +// +// RIFF::DataChunk +// -/* static */ SlangResult RiffUtil::readChunk(Stream* stream, RiffHeader& outChunk) +void DataChunk::writePayloadInto(void* outData, Size size) const { - size_t readBytes; - SLANG_RETURN_ON_FAIL(stream->read(&outChunk, sizeof(RiffHeader), readBytes)); - // TODO(JS): Could handle endianness issues here... - return (readBytes == sizeof(RiffHeader)) ? SLANG_OK : SLANG_FAIL; + SLANG_ASSERT(size <= getPayloadSize()); + ::memcpy(outData, getPayload(), size); } -/* static */ SlangResult RiffUtil::writeData( - const RiffHeader* header, - size_t headerSize, - const void* payload, - size_t payloadSize, - Stream* out) -{ - SLANG_ASSERT(uint64_t(payloadSize) <= uint64_t(0xfffffffff)); - SLANG_ASSERT(headerSize >= sizeof(RiffHeader)); - - // TODO(JS): Could handle endianness here +// +// RIFF::BoundsCheckedChunkPtr +// - RiffHeader chunk; - chunk.type = header->type; - chunk.size = uint32_t(headerSize - sizeof(RiffHeader) + payloadSize); - - // The chunk - SLANG_RETURN_ON_FAIL(out->write(&chunk, sizeof(RiffHeader))); +void BoundsCheckedChunkPtr::_set(Chunk const* chunk, Size sizeLimit) +{ + // We start by clearing out the state of this + // pointer, so that we can early-out if any + // validation checks fail, and be sure we + // have a null pointer. + // + _ptr = nullptr; + _sizeLimit = 0; - // Remainder of header - if (headerSize > sizeof(RiffHeader)) + // If there's nothing to point to, then the pointer + // should be null anyway. + // + if (!chunk || !sizeLimit) { - // The rest of the header - SLANG_RETURN_ON_FAIL(out->write(header + 1, headerSize - sizeof(RiffHeader))); + return; } - // Write the payload - SLANG_RETURN_ON_FAIL(out->write(payload, payloadSize)); + // Because this type can be used to traverse RIFF + // chunks that were loaded into memory from in-theory + // untrusted sources, we try to provide some validation + // checks to make sure that access to the chunk will + // be safe (or as safe as we can easily ensure). - // The riff spec requires all chunks are 4 byte aligned (even if size is not) - size_t padSize = getPadSize(payloadSize); - if (padSize - payloadSize) + // If the available size isn't even enough for the + // header of a RIFF chunk, then something is wrong. + // + if (sizeLimit < sizeof(Chunk::Header)) { - uint8_t end[kRiffPadSize] = {0}; - SLANG_RETURN_ON_FAIL(out->write(end, padSize - payloadSize)); + SLANG_UNEXPECTED("invalid RIFF"); + return; } - return SLANG_OK; -} - -/* static */ SlangResult RiffUtil::readPayload( - Stream* stream, - size_t size, - void* outData, - size_t& outReadSize) -{ - outReadSize = 0; - - SLANG_RETURN_ON_FAIL(stream->readExactly(outData, size)); + // Once we've checked that there is enough space + // for a valid RIFF header, we can read the + // size that the `chunk` reports itself as having. + // + auto reportedSize = chunk->getTotalSize(); - const size_t alignedSize = getPadSize(size); - // Skip to the alignment - if (alignedSize > size) + // If the reported size is too small, then something + // is wrong. + // + if (reportedSize < sizeof(Chunk::Header)) { - SLANG_RETURN_ON_FAIL(stream->seek(SeekOrigin::Current, alignedSize - size)); + SLANG_UNEXPECTED("invalid RIFF"); + return; } - outReadSize = alignedSize; - return SLANG_OK; -} -/* static */ SlangResult RiffUtil::readData( - Stream* stream, - RiffHeader* outHeader, - size_t headerSize, - List<uint8_t>& data) -{ - RiffHeader chunk; - SLANG_RETURN_ON_FAIL(readChunk(stream, chunk)); - if (chunk.size < headerSize) + // If the reported size is bigger than the size limit, + // then it must be invalid (it is reporting itself as + // bigger than the region of memory that is supposed + // to contain it). + // + if (reportedSize > sizeLimit) { - return SLANG_FAIL; + SLANG_UNEXPECTED("invalid RIFF"); + return; } - *outHeader = chunk; - - // Read the header - if (headerSize > sizeof(RiffHeader)) + // If the chunk claims to be a list chunk, then it + // must be big enough to hold the larger header + // that list chunks use. + // + if (as<ListChunk>(chunk)) { - SLANG_RETURN_ON_FAIL(stream->readExactly(outHeader + 1, headerSize - sizeof(RiffHeader))); + if (reportedSize < sizeof(ListChunk::Header)) + { + SLANG_UNEXPECTED("invalid RIFF"); + return; + } } - const size_t payloadSize = chunk.size - (headerSize - sizeof(RiffHeader)); - size_t readSize; - data.setCount(payloadSize); - return readPayload(stream, payloadSize, data.getBuffer(), readSize); + // At this point we've performed some basic validation + // telling us that the chunk header appears plausible. + // This does not mean that we've fully validated the + // hierarchy of child chunks under it (in the case of + // a list chunk), but that validation can be performed + // on-demand while descending the hierarchy. + + _ptr = chunk; + _sizeLimit = sizeLimit; } -/* static */ SlangResult RiffUtil::readHeader(Stream* stream, RiffListHeader& outHeader) +void BoundsCheckedChunkPtr::_set(Chunk const* chunk) { - // Need to read the chunk header - SLANG_RETURN_ON_FAIL(readChunk(stream, outHeader.chunk)); - outHeader.subType = 0; + // In the case where we are being set to point to a + // single chunk, we have to assume that whatever + // code derived the `chunk` pointer has validated + // that it is safe to access its header. + // + // We will simply set up a pointer that can reference + // the `chunk` itself, as well as any of its children + // (if it has any), but that cannot be used to access + // further sibling chunks under the same parent. + // + _set(chunk, chunk->getTotalSize()); +} - if (isListType(outHeader.chunk.type)) - { - // Read the sub type - SLANG_RETURN_ON_FAIL( - stream->readExactly(&outHeader.subType, sizeof(RiffListHeader) - sizeof(RiffHeader))); - } +BoundsCheckedChunkPtr BoundsCheckedChunkPtr::getNextSibling() const +{ + SLANG_ASSERT(_ptr != nullptr); + if (!_ptr) + return nullptr; - return SLANG_OK; + // The RIFF chunk reports its own size, and when navigating + // the children of a list chunk, each child chunk starts + // at the next (aligned) offset after the previous one. + // + auto chunkSize = _ptr->getTotalSize(); + + // As a simple validation check, we check for a chunk that + // reports its size as something bigger than the available + // size; that would represent an invalid input. + // + if (chunkSize > _sizeLimit) + { + SLANG_UNEXPECTED("invalid RIFF chunk size"); + UNREACHABLE_RETURN(nullptr); + } + + // The next chunk (if there is one) would start at the + // next offset after this chunk, rounded up to the minimum + // alignment for a chunk. Thus, we round up the reported + // size of this chunk to compute the offset to the next + // chunk. + // + auto offsetToNextChunk = _roundUpToChunkAlignment(chunkSize); + + // If stepping forward by the given number of bytes would + // cause us to exceed our size limit, then we have reached + // the end of the list of sibling chunks, and should + // return a null pointer. + // + if (offsetToNextChunk >= _sizeLimit) + return nullptr; + + auto nextChunk = (RIFF::Chunk const*)(offsetToNextChunk + (Byte const*)_ptr); + auto nextSizeLimit = _sizeLimit - offsetToNextChunk; + + return BoundsCheckedChunkPtr(nextChunk, nextSizeLimit); } -namespace -{ // anonymous -struct DumpVisitor : public RiffContainer::Visitor +// +// RIFF::ListChunk +// + +BoundsCheckedChunkPtr ListChunk::getFirstChild() const { - typedef RiffContainer::Chunk Chunk; - typedef RiffContainer::ListChunk ListChunk; - typedef RiffContainer::DataChunk DataChunk; + // Because this type could be used to navigate + // an untrusted RIFF that has been loaded into memory, + // we make some efforts to validate that things + // seem okay as we navigate it. + // The first child of a list chunk (if it has any) + // comes right after the list header. + // + Size firstChildOffset = sizeof(ListChunk::Header); - // Visitor - virtual SlangResult enterList(ListChunk* list) SLANG_OVERRIDE + // The size that the parent chunk reports should + // be appropriate to store the header. + // + // Note that in order to compute the reported size + // we are *accessing* the header, so if there really + // are too few bytes available, it is up to whatever + // code computed this `ListChunk*` to have done + // their own validation checks. + // + Size reportedParentSize = getTotalSize(); + if (reportedParentSize < firstChildOffset) { - _dumpIndent(); - // If it's the root it's 'riff' - _dumpRiffType(list == m_rootChunk ? RiffFourCC::kRiff : RiffFourCC::kList); - m_writer.put(" "); - _dumpRiffType(list->getSubType()); - m_writer.put("\n"); - m_indent++; - return SLANG_OK; + SLANG_UNEXPECTED("invalid RIFF"); + UNREACHABLE_RETURN(nullptr); } - virtual SlangResult handleData(DataChunk* data) SLANG_OVERRIDE - { - _dumpIndent(); - // Write out the name - _dumpRiffType(data->m_fourCC); - m_writer.put(" "); - const RiffHashCode hash = data->calcHash(); + // The total size that the childen of this chunk can + // consume is all of the reported size of the parent, + // after the `ListChunk::Header`. + // + Size availableSizeForChildren = reportedParentSize - firstChildOffset; - // We don't know in general what the contents is or means... but we can display a hash - HexDumpUtil::dump(uint32_t(hash), m_writer.getWriter()); - m_writer.put(" "); + // The available size can be zero, in the case where + // the parent chunk has no children. + // + if (availableSizeForChildren == 0) + return nullptr; - m_writer.put("\n"); - return SLANG_OK; - } - virtual SlangResult leaveList(ListChunk* list) SLANG_OVERRIDE + // If the parent chunk has a non-zero size, then it should + // have at least one child, and the available size had better + // be big enough to at least hold the *header* of that first + // child. + // + if (availableSizeForChildren < sizeof(Chunk::Header)) { - SLANG_UNUSED(list); - m_indent--; - return SLANG_OK; + SLANG_UNEXPECTED("invalid RIFF"); + UNREACHABLE_RETURN(nullptr); } - DumpVisitor(WriterHelper writer, Chunk* rootChunk) - : m_writer(writer), m_indent(0), m_rootChunk(rootChunk) - { - } + // At this point we've convinced ourselves that there is + // conceivably enough space for at least one child chunk, + // so we will form a `BoundsCheckedChunkPtr` to it, which + // will trigger further validity checks on that child chunk. + // + auto firstChild = (Chunk const*)(firstChildOffset + (Byte const*)this); + return BoundsCheckedChunkPtr(firstChild, availableSizeForChildren); +} - void _dumpIndent() - { - for (int i = 0; i < m_indent; ++i) - { - m_writer.put(" "); - } - } - void _dumpRiffType(FourCC fourCC) +DataChunk const* ListChunk::findDataChunk(Chunk::Type type) const +{ + for (auto chunk : getChildren()) { - char c[5]; - for (int i = 0; i < 4; ++i) - { - c[i] = char(fourCC); - fourCC >>= 8; - } - c[4] = 0; - m_writer.put(c); - } - - Chunk* m_rootChunk; + auto dataChunk = as<DataChunk>(chunk); + if (!dataChunk) + continue; - int m_indent; - WriterHelper m_writer; -}; + if (dataChunk->getType() != type) + continue; -} // namespace - -/* static */ void RiffUtil::dump(RiffContainer::Chunk* chunk, WriterHelper writer) -{ - DumpVisitor visitor(writer, chunk); - chunk->visit(&visitor); + return dataChunk; + } + return nullptr; } -/* static */ SlangResult RiffUtil::write( - RiffContainer::ListChunk* list, - bool isRoot, - Stream* stream) +ListChunk const* ListChunk::findListChunk(Chunk::Type type) const { - RiffListHeader listHeader; - - listHeader.chunk.type = isRoot ? RiffFourCC::kRiff : RiffFourCC::kList; - listHeader.chunk.size = uint32_t(list->m_payloadSize); - listHeader.subType = list->getSubType(); - - // Write the header - SLANG_RETURN_ON_FAIL(stream->write(&listHeader, sizeof(listHeader))); - - // Write the contained chunks - Chunk* chunk = list->m_containedChunks; - while (chunk) + for (auto chunk : getChildren()) { - switch (chunk->m_kind) - { - case Chunk::Kind::List: - { - auto listChunk = static_cast<ListChunk*>(chunk); - // It's a container - SLANG_RETURN_ON_FAIL(write(listChunk, false, stream)); - break; - } - case Chunk::Kind::Data: - { - auto dataChunk = static_cast<DataChunk*>(chunk); - - // Must be a regular chunk with data - RiffHeader chunkHeader; - chunkHeader.type = dataChunk->m_fourCC; - chunkHeader.size = uint32_t(dataChunk->m_payloadSize); - - SLANG_RETURN_ON_FAIL(stream->write(&chunkHeader, sizeof(chunkHeader))); - - RiffContainer::Data* data = dataChunk->m_dataList; - while (data) - { - SLANG_RETURN_ON_FAIL(stream->write(data->getPayload(), data->getSize())); - - // Next but of data - data = data->m_next; - } + auto listChunk = as<ListChunk>(chunk); + if (!listChunk) + continue; - // Need to write for alignment - const size_t remainingSize = - getPadSize(dataChunk->m_payloadSize) - dataChunk->m_payloadSize; + if (listChunk->getType() != type) + continue; - if (remainingSize) - { - static const uint8_t trailing[kRiffPadSize] = {0}; - SLANG_RETURN_ON_FAIL(stream->write(trailing, remainingSize)); - } - } - default: - break; - } - - // Next - chunk = chunk->m_next; + return listChunk; } - - return SLANG_OK; + return nullptr; } -/* static */ SlangResult RiffUtil::write(RiffContainer* container, Stream* stream) +ListChunk const* ListChunk::findListChunkRec(Chunk::Type type) const { - return write(container->getRoot(), true, stream); -} + // Note: The search being performed here could + // be implemented without any need for recursion + // (or a stack), by taking advantage of the way + // that RIFF chunks are laid out. If we have some + // chunk C, then the next aligned offset in memory + // after C is either at the end of the hierarchy, + // or it is the next sibling of one of C's ancestors + // (where C is being counted as its own ancestor). + // + // However, it's not really clear if there's enough + // of a benefit to justify that more subtle implementation. -/* static */ SlangResult RiffUtil::read(Stream* stream, RiffContainer& outContainer) -{ - typedef RiffContainer::ScopeChunk ScopeChunk; - outContainer.reset(); + if (getType() == type) + return this; - size_t remaining; + for (auto chunk : getChildren()) { - RiffListHeader header; + auto listChunk = as<ListChunk>(chunk); + if (!listChunk) + continue; - SLANG_RETURN_ON_FAIL(readHeader(stream, header)); - if (!isListType(header.chunk.type)) - { - return SLANG_FAIL; - } + auto found = listChunk->findListChunkRec(type); + if (!found) + continue; - remaining = getPadSize(header.chunk.size) - (sizeof(RiffListHeader) - sizeof(RiffHeader)); - outContainer.startChunk(Chunk::Kind::List, header.subType); + return found; } + return nullptr; +} - List<size_t> remainingStack; - while (true) - { - // It must be the end - if (remaining == 0) - { - // If it's a container then we pop container - outContainer.endChunk(); - if (remainingStack.getCount() <= 0) - { - break; - } - - remaining = remainingStack.getLast(); - remainingStack.removeLast(); - } - else - { - RiffListHeader header; - SLANG_RETURN_ON_FAIL(readHeader(stream, header)); - - // The amount of data can't be larger than what remains - if (header.chunk.size > remaining) - { - return SLANG_FAIL; - } - if (header.chunk.type == RiffFourCC::kList) - { - if (header.chunk.size & kRiffPadMask) - { - SLANG_ASSERT(!"A list chunk can only have divisible by 2 size"); - return SLANG_FAIL; - } +// +// RIFF::RootChunk +// - // Work out the pad size - const size_t padSize = getPadSize(header.chunk.size); +RootChunk const* RootChunk::getFromBlob(void const* data, size_t dataSize) +{ + // Our goal is to determine whether the given + // blob superficially looks like a RIFF. - // Subtract the size of this chunk from remaining of the current chunk - remaining -= sizeof(RiffHeader) + padSize; - // Push it, for when we hit the end - remainingStack.add(remaining); + // The data pointer should be non-null if there + // was any data passed in. + // + SLANG_ASSERT(data || !dataSize); - // Work out how much remains in this container - remaining = padSize - (sizeof(RiffListHeader) - sizeof(RiffHeader)); + // If there's no data, then it's obvious not usable. + // + if (!data) + return nullptr; - // Start a container - outContainer.startChunk(Chunk::Kind::List, header.subType); - } - else - { - ScopeChunk scopeChunk(&outContainer, Chunk::Kind::Data, header.chunk.type); - RiffContainer::Data* data = outContainer.addData(); + // If there isn't even enough data to store the header + // for a root, chunk, then the blob is too small. + // + if (dataSize < sizeof(RootChunk::Header)) + return nullptr; - outContainer.setPayload(data, nullptr, header.chunk.size); + // We cast the data pointer to a root chunk here, so that + // we can access the fields in the header, but we are not + // yet convinced it is actually a valid RIFF, so we may + // still return `nullptr`. + // + auto rootChunk = reinterpret_cast<RootChunk const*>(data); + + // The root chunk of a valid RIFF should have the `"RIFF"` + // tag. This acts as a kind of "magic number" to mark the + // start of a RIFF. + // + if (rootChunk->getTag() != RootChunk::kTag) + return nullptr; - size_t readSize; - SLANG_RETURN_ON_FAIL( - readPayload(stream, header.chunk.size, data->getPayload(), readSize)); + // By reading the size field from the root chunk, we can + // determine how big of a file the root chunk claims that + // we have. + // + auto reportedSize = rootChunk->getTotalSize(); + + // If the size implied by the RIFF header is larger than the + // blob, then we do not have a properly structured RIFF, + // and we would be at risk of reading past the end of the + // buffer if we attempted to use it. + // + if (reportedSize > dataSize) + return nullptr; - // All read sizes must end up aligned - SLANG_ASSERT((readSize & kRiffPadMask) == 0); + // Note: It is possible that the `reportedSize` is strictly + // *less than* the `dataSize` that was passed in, and there + // is a policy choice to be made about how to handle that case. + // + // We err on the side of leniency here, because the client who + // is calling this function might intentionally be storing + // additional data in the same blob, after the RIFF, and could + // use the RIFF's ability to report its own size as a way to + // locate that data. - // Correct remaining - remaining -= sizeof(RiffHeader) + readSize; - } - } - } + // Note: At this point we could recursively walk the hierarchy + // of the RIFF and validate that all the contained chunks appear + // valid in terms of the sizes they report, but doing so would + // take an amount of time that scales with the size of the RIFF, + // and our goal here is to be efficient. + // + // Access to the data through the `RIFF::Chunk` API will do its + // best to validate the information in chunks as they are + // accessed. The code is attempting to be able to catch + // corrupted or accidentally malformed input, but is not aspiring + // to anything like proper security. - return outContainer.isFullyConstructed() ? SLANG_OK : SLANG_FAIL; + return rootChunk; } -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! RiffContainer::Chunk !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -SlangResult RiffContainer::Chunk::visit(Visitor* visitor) +RootChunk const* RootChunk::getFromBlob(ISlangBlob* blob) { - switch (m_kind) - { - case Kind::Data: - { - return visitor->handleData(static_cast<DataChunk*>(this)); - } - case Kind::List: - { - auto list = static_cast<ListChunk*>(this); - SLANG_RETURN_ON_FAIL(visitor->enterList(list)); - - Chunk* chunk = list->m_containedChunks; - while (chunk) - { - SLANG_RETURN_ON_FAIL(chunk->visit(visitor)); - - chunk = chunk->m_next; - } - - SLANG_RETURN_ON_FAIL(visitor->leaveList(list)); - return SLANG_OK; - } - default: - return SLANG_FAIL; - } + SLANG_ASSERT(blob); + return getFromBlob(blob->getBufferPointer(), blob->getBufferSize()); } -SlangResult RiffContainer::Chunk::visitPreOrder(VisitorCallback callback, void* data) +// +// RIFF::ChunkBuilder +// + +Size ChunkBuilder::_updateCachedTotalSize() const { - switch (m_kind) + Size totalSize = 0; + if (auto dataChunk = as<DataChunkBuilder>(this)) { - case Kind::Data: - { - return callback(this, data); - } - case Kind::List: - { - auto list = static_cast<ListChunk*>(this); - // Do this containing node first - SLANG_RETURN_ON_FAIL(callback(this, data)); + // Every chunk starts with a header. + // + totalSize += sizeof(DataChunk::Header); - // Do the contents next - Chunk* chunk = list->m_containedChunks; - while (chunk) - { - SLANG_RETURN_ON_FAIL(chunk->visitPreOrder(callback, data)); - chunk = chunk->m_next; - } - return SLANG_OK; + // After the header comes the payload of + // the chunk, which for a data chunk + // will be the concatenation of all its + // shards. + // + for (auto shard : dataChunk->getShards()) + { + totalSize += shard->getPayloadSize(); } - default: - return SLANG_FAIL; } -} - -SlangResult RiffContainer::Chunk::visitPostOrder(VisitorCallback callback, void* data) -{ - switch (m_kind) + else if (auto listChunk = as<ListChunkBuilder>(this)) { - case Kind::Data: - { - return callback(this, data); - } - case Kind::List: - { - auto list = static_cast<ListChunk*>(this); + // A list chunk starts with a header, just + // like a data chunk, although its header + // is larger. + // + totalSize += sizeof(ListChunk::Header); - // Do the contents first - Chunk* chunk = list->m_containedChunks; - while (chunk) - { - SLANG_RETURN_ON_FAIL(chunk->visitPostOrder(callback, data)); - chunk = chunk->m_next; - } - // Then the list node (so a post order) - SLANG_RETURN_ON_FAIL(callback(this, data)); - return SLANG_OK; + // After the header come the child chunks, in order. + // + for (auto child : listChunk->getChildren()) + { + // We recursively poke the child chunks to + // update their cached total size, so that + // we can be sure we are getting correct + // information. + // + auto childSize = child->_updateCachedTotalSize(); + + // We cannot simply add the size of the child + // chunk directly to `totalSize`, because a + // RIFF guarantees that every chunk must start + // on a suitably aligned boundary. Thus, we + // first round `totalSize` up to the necessary + // alignment, and then add the child's size. + // + totalSize = _roundUpToChunkAlignment(totalSize); + totalSize += childSize; } - default: - return SLANG_FAIL; } -} - -size_t RiffContainer::Chunk::calcPayloadSize() -{ - switch (m_kind) + else { - case Kind::Data: - return static_cast<DataChunk*>(this)->calcPayloadSize(); - case Kind::List: - return static_cast<ListChunk*>(this)->calcPayloadSize(); - default: - return 0; + SLANG_UNREACHABLE("RIFF chunk must be data or list"); } -} -RiffContainer::Data* RiffContainer::Chunk::getSingleData() const -{ - return (m_kind == Kind::Data) ? static_cast<const DataChunk*>(this)->getSingleData() : nullptr; + _cachedTotalSize = totalSize; + return totalSize; } -// !!!!!!!!!!!!!!!!!!!!!!!!!!! RiffContainer::ListChunk !!!!!!!!!!!!!!!!!!!!!! - -size_t RiffContainer::ListChunk::calcPayloadSize() +Result ChunkBuilder::_writeTo(Stream* stream) const { - // Have to include the part of the header not taken up by the RiffHeader - size_t size = sizeof(RiffListHeader) - sizeof(RiffHeader); - Chunk* chunk = m_containedChunks; - while (chunk) - { - size_t chunkSize = chunk->m_payloadSize + sizeof(RiffHeader); - // Align the contained chunk size - size += RiffUtil::getPadSize(chunkSize); + // The size information that gets written into + // the chunk header will be based on the cached + // size information for this chunk. + // + // If nobody has called `_updateCachedTotalSize()` + // at all, then that is a problem. + // + SLANG_ASSERT(_getCachedTotalSize() >= sizeof(Chunk::Header)); - chunk = chunk->m_next; - } - return size; -} + // The size that gets written into the chunk header + // is the total size of the chunk, ecluding the chunk + // header. Note that this size will *include* the + // additional field of the list chunk header. + // + Size totalSizeExcludingChunkHeader = _getCachedTotalSize() - sizeof(Chunk::Header); -RiffContainer::Chunk* RiffContainer::ListChunk::findContained(FourCC fourCC) const -{ - Chunk* chunk = m_containedChunks; - while (chunk) - { - if (chunk->m_fourCC == fourCC) - { - return chunk; - } - chunk = chunk->m_next; - } - return nullptr; -} + // Because the size field in the header is only 32 bits, + // we want to double-check that it can actually represent + // the size of the data to be written into it. + // + UInt32 sizeToWriteInHeader = UInt32(totalSizeExcludingChunkHeader); + SLANG_ASSERT(Size(sizeToWriteInHeader) == totalSizeExcludingChunkHeader); -void RiffContainer::ListChunk::findContained(FourCC type, List<ListChunk*>& out) -{ - Chunk* chunk = m_containedChunks; - while (chunk) + if (auto dataChunk = as<DataChunkBuilder>(this)) { - if (chunk->m_fourCC == type && chunk->m_kind == Chunk::Kind::List) - { - out.add(static_cast<ListChunk*>(chunk)); - } - chunk = chunk->m_next; - } -} + // We start by writing the header. + // + DataChunk::Header header; + header.size = sizeToWriteInHeader; -void RiffContainer::ListChunk::findContained(FourCC type, List<DataChunk*>& out) -{ - Chunk* chunk = m_containedChunks; - while (chunk) - { - if (chunk->m_fourCC == type && chunk->m_kind == Chunk::Kind::Data) - { - out.add(static_cast<DataChunk*>(chunk)); - } - chunk = chunk->m_next; - } -} + // The tag of a data chunk is its type `FourCC`. + // + header.tag = dataChunk->getType(); -RiffContainer::ListChunk* RiffContainer::ListChunk::findContainedList(FourCC type) -{ - Chunk* chunk = m_containedChunks; - while (chunk) - { - if (chunk->m_fourCC == type && chunk->m_kind == Chunk::Kind::List) - { - return static_cast<ListChunk*>(chunk); - } - chunk = chunk->m_next; - } - return nullptr; -} + // Once we've filled in the header fields, we + // can write it to the output stream. + // + SLANG_RETURN_ON_FAIL(stream->write(&header, sizeof(header))); -RiffContainer::Data* RiffContainer::ListChunk::findContainedData(FourCC type) const -{ - Chunk* found = findContained(type); - if (found && found->m_kind == Kind::Data) - { - DataChunk* dataChunk = static_cast<DataChunk*>(found); - // Assumes that there is a single data chunk - - Data* data = dataChunk->m_dataList; - if (data && data->m_next == nullptr) + // Now we can simply write the payload bytes, + // which are the concatenation of the payloads + // of all the shards. + // + for (auto shard : dataChunk->getShards()) { - return data; + auto payload = shard->getPayload(); + auto payloadSize = shard->getPayloadSize(); + SLANG_RETURN_ON_FAIL(stream->write(payload, payloadSize)); } } - return nullptr; -} - -void* RiffContainer::ListChunk::findContainedData(FourCC type, size_t minSize) const -{ - Data* data = findContainedData(type); - return (data && data->m_size >= minSize) ? data->getPayload() : nullptr; -} - -static RiffContainer::ListChunk* _findListRec(RiffContainer::ListChunk* list, FourCC subType) -{ - RiffContainer::Chunk* chunk = list->m_containedChunks; - while (chunk) - { - if (auto childList = as<RiffContainer::ListChunk>(chunk)) + else if (auto listChunk = as<ListChunkBuilder>(this)) + { + // We start by writing the header. + // + ListChunk::Header header; + header.chunkHeader.size = sizeToWriteInHeader; + + // The tag of a list chunk is either `"RIFF"` + // (for a root chunk) or `"LIST"` (for any + // other list chunk). + // + header.chunkHeader.tag = + listChunk->getKind() == Chunk::Kind::Root ? RootChunk::kTag : ListChunk::kTag; + + // The type of a list chunk is stored in the + // additional header field after the base + // chunk header. + // + header.type = listChunk->getType(); + + // Once we've filled in the header fields, we + // can write it to the output stream. + // + SLANG_RETURN_ON_FAIL(stream->write(&header, sizeof(header))); + + // Now we recursively write each of the child chunks, + // keeping track of the total size so far, so that + // we can insert padding as needing to bring things + // up to alignment. + // + Size totalSize = sizeof(header); + for (auto child : listChunk->getChildren()) { - // Test if the child is the subtype, if so we are done - if (childList->getSubType() == subType) + // We note the total size written so far, + // as well as the size after rounding up + // to the required alignment. + // + auto unalignedSize = totalSize; + auto alignedSize = _roundUpToChunkAlignment(unalignedSize); + SLANG_ASSERT(alignedSize >= unalignedSize); + + // If the aligned size is greater than the + // unaligned size, then we may need to write + // some padding bytes into the output stream. + // + auto paddingSize = alignedSize - unalignedSize; + + // The amount of padding to be inserted must + // always be less than the minimum chunk alignment. + // + SLANG_ASSERT(paddingSize < Chunk::kChunkAlignment); + + // We'll write padding bytes to get things up + // to the necessary alignment. + // + auto remainingPaddingToWrite = paddingSize; + while (remainingPaddingToWrite--) { - return childList; - } - auto found = _findListRec(childList, subType); - if (found) - { - return found; + static const Byte kPadding[1] = {0}; + stream->write(kPadding, 1); } + + // Now we are at a suitably aligned offset. + // + totalSize = alignedSize; + + // With the alignment concern dealt with, + // we can simply recursively write the + // child chunk and update the total size. + // + SLANG_RETURN_ON_FAIL(child->_writeTo(stream)); + totalSize += child->_getCachedTotalSize(); } - chunk = chunk->m_next; + + // As a validation check, we expect the total number + // of bytes we've written here to match the total + // size that was cached on this chunk (and that was + // written into the chunk header). + // + SLANG_ASSERT(totalSize == _getCachedTotalSize()); } - return nullptr; + else + { + SLANG_UNREACHABLE("RIFF chunk must be data or list"); + } + return SLANG_OK; } -/* static */ RiffContainer::ListChunk* RiffContainer::ListChunk::findListRec(FourCC subType) +MemoryArena& ChunkBuilder::_getMemoryArena() const { - return (getSubType() == subType) ? this : _findListRec(this, subType); + return getRIFFBuilder()->_getMemoryArena(); } -// !!!!!!!!!!!!!!!!!!!!!!!!!!! RiffContainer::DataChunk !!!!!!!!!!!!!!!!!!!!!! +// +// RIFF::ListChunkBuilder +// -RiffContainer::Data* RiffContainer::DataChunk::getSingleData() const +DataChunkBuilder* ListChunkBuilder::addDataChunk(Chunk::Type type) { - Data* data = m_dataList; - return (data && data->m_next == nullptr) ? data : nullptr; + auto chunk = new (_getMemoryArena()) DataChunkBuilder(type, this); + _children.add(chunk); + return chunk; } -RiffReadHelper RiffContainer::DataChunk::asReadHelper() const +ListChunkBuilder* ListChunkBuilder::addListChunk(Chunk::Type type) { - Data* data = getSingleData(); - if (data) - { - return RiffReadHelper((const uint8_t*)data->getPayload(), data->getSize()); - } - return RiffReadHelper(nullptr, 0); + auto chunk = new (_getMemoryArena()) ListChunkBuilder(type, this); + _children.add(chunk); + return chunk; } -RiffHashCode RiffContainer::DataChunk::calcHash() const -{ - RiffHashCode hash = 0; - - Data* data = m_dataList; - while (data) - { - // This is a little contrived (in that we don't use the function getHashCode), but the - // reason to be careful is we want the same result however many Data blocks there are. - const char* buffer = (const char*)data->getPayload(); - const size_t size = data->getSize(); - - for (size_t i = 0; i < size; ++i) +// +// RIFF::DataChunkBuilder +// + +void DataChunkBuilder::addData(void const* data, Size size) +{ + // Adding no data should be a no-op. + // + if (size == 0) + return; + + // The most interesting implementation detail here + // is that we will try to detect cases where we + // can re-use an existing `Shard` by adding the data + // to the end of that shard's allocation. + // + // This is only possible because of the way that + // we are using a single `MemoryArena` to allocate + // everything, which makes it possible that the + // next address the arena would return for an allocation + // of `size` bytes is the same as the ending address + // of the payload for the last shard of this chunk. + // + auto& arena = _getMemoryArena(); + + // We start by checking if this chunk already has + // a last shard that we could consider appending to. + // + auto lastShard = _shards.getLast(); + if (lastShard) + { + // If there is a last shard, then we can compute + // the end address of its payload, and see if + // it is the same as the cursor of the arena + // we are allocating from. + // + auto payload = lastShard->getPayload(); + auto payloadSize = lastShard->getPayloadSize(); + auto payloadEnd = (Byte*)payload + payloadSize; + if (payloadEnd == arena.getCursor()) { - hash = RiffHashCode(buffer[i]) + (hash << 6) + (hash << 16) - hash; + // Now that we've confirmed that the shard's + // payload ends at an address the arena could + // conceivably allocate from, we need to ask + // the arena to allocate `size` bytes from + // the current block it is using, and see if + // doing so succeeds. + // + if (arena.allocateCurrentUnaligned(size)) + { + // At this point, we've confirmed that we + // are in our special case, and the relevant + // bytes have been allocated from the arena. + // + // Now we can simply write the new data at + // what used to be the end address for the + // shard's payload, and adjust its state + // to account for the new allocation. + // + ::memcpy(payloadEnd, data, size); + lastShard->setPayload(payload, payloadSize + size); + return; + } } - - data = data->m_next; } - return hash; + // If we didn't land in our special case, we + // will simply allocate a new shard to hold + // the data. + // + // Note that the order of allocation here is + // intentional, and supports the optimized special + // case that we checked for above. We make the + // allocation for the payload *last*, so that + // it is possible that the arena's next allocation + // could come right after the payload allocation + // in memory. + // + // If we allocated the payload first and the + // `Shard` second, then there would already be + // another allocation after the payload, and + // the optimized case would never trigger. + // + auto shard = _addShard(); + auto payload = arena.allocateUnaligned(size); + ::memcpy(payload, data, size); + shard->setPayload(payload, size); } -size_t RiffContainer::DataChunk::calcPayloadSize() const +void DataChunkBuilder::addUnownedData(void const* data, size_t size) { - size_t size = 0; - Data* data = m_dataList; - while (data) - { - size += data->getSize(); - data = data->m_next; - } - return size; + // Unowned data will always have to be added as its own shard. + // + auto shard = _addShard(); + shard->setPayload(data, size); } -void RiffContainer::DataChunk::getPayload(void* inDst) const +DataChunkBuilder::Shard* DataChunkBuilder::_addShard() { - uint8_t* dst = (uint8_t*)inDst; + auto shard = new (_getMemoryArena()) Shard(); + _shards.add(shard); + return shard; +} - Data* data = m_dataList; - while (data) - { - const size_t size = data->getSize(); - ::memcpy(dst, data->getPayload(), size); +// +// RIFF::Builder +// - dst += size; - data = data->m_next; - } +Builder::Builder() + : _arena(4096) +{ } -bool RiffContainer::DataChunk::isEqual(const void* inData, size_t count) const +Result Builder::writeTo(Stream* stream) { - const uint8_t* src = (const uint8_t*)inData; - - Data* data = m_dataList; - while (data) - { - const size_t size = data->getSize(); - // Can't have more content than remaining - // Contents must match - if (size > count || ::memcmp(src, data->getPayload(), size) != 0) - { - return false; - } - - src += size; - count -= size; - - // Next data block - data = data->m_next; - } + // If there's no root chunk, then this isn't + // a well-formed RIFF. + // + if (!_rootChunk) + return SLANG_FAIL; - // If match must be at the end - return count == 0; + // The `ChunkBuilder::_writeTo()` method requires size + // information for each of the chunks in the hierarchy. + // Rather than try to keep size information up-to-date + // during the building process, we simply compute it + // all at once, right before writing the output. + // + _rootChunk->_updateCachedTotalSize(); + _rootChunk->_writeTo(stream); + return SLANG_OK; } -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! RiffContainer !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -RiffContainer::RiffContainer() - : m_arena(4096) +Result Builder::writeToBlob(ISlangBlob** outBlob) { - m_rootList = nullptr; - m_listChunk = nullptr; - m_dataChunk = nullptr; -} + OwnedMemoryStream stream(FileAccess::Write); + SLANG_RETURN_ON_FAIL(writeTo(&stream)); -void RiffContainer::reset() -{ - m_arena.reset(); + List<uint8_t> data; + stream.swapContents(data); - m_rootList = nullptr; - m_listChunk = nullptr; - m_dataChunk = nullptr; + *outBlob = ListBlob::moveCreate(data).detach(); + return SLANG_OK; } -RiffContainer::ListChunk* RiffContainer::_newListChunk(FourCC subType) +ListChunkBuilder* Builder::addRootChunk(Chunk::Type type) { - SLANG_ASSERT(!RiffUtil::isListType(subType)); + // There must not already be a root chunk set. + SLANG_ASSERT(getRootChunk() == nullptr); - ListChunk* chunk = (ListChunk*)m_arena.allocate(sizeof(ListChunk)); - chunk->init(subType); + auto chunk = new (_getMemoryArena()) ListChunkBuilder(type, this); + _rootChunk = chunk; return chunk; } -RiffContainer::DataChunk* RiffContainer::_newDataChunk(FourCC type) -{ - SLANG_ASSERT(!RiffUtil::isListType(type)); - DataChunk* chunk = (DataChunk*)m_arena.allocate(sizeof(DataChunk)); - chunk->init(type); - return chunk; -} +// +// RIFF::BuildCursor +// -void RiffContainer::_addChunk(Chunk* chunk) +BuildCursor::BuildCursor() {} + +BuildCursor::BuildCursor(Builder& builder) + : _riffBuilder(&builder) { - if (m_listChunk) - { - chunk->m_parent = m_listChunk; - Chunk*& next = m_listChunk->m_endChunk ? m_listChunk->m_endChunk->m_next - : m_listChunk->m_containedChunks; - SLANG_ASSERT(next == nullptr); - next = chunk; - m_listChunk->m_endChunk = chunk; - } } -void RiffContainer::setCurrentChunk(Chunk* chunk) +BuildCursor::BuildCursor(ChunkBuilder* chunk) { - SLANG_ASSERT(chunk); - - switch (chunk->m_kind) - { - case Chunk::Kind::Data: - m_listChunk = nullptr; - m_dataChunk = static_cast<RiffContainer::DataChunk*>(chunk); - break; - - case Chunk::Kind::List: - m_dataChunk = nullptr; - m_listChunk = static_cast<RiffContainer::ListChunk*>(chunk); - break; - } + setCurrentChunk(chunk); } -void RiffContainer::startChunk(Chunk::Kind kind, FourCC fourCC) +void BuildCursor::setCurrentChunk(ChunkBuilder* chunk) { - SLANG_ASSERT(m_listChunk || m_rootList == nullptr); - - switch (kind) - { - case Chunk::Kind::Data: - { - // We can only start a data chunk if we are in a container, and we can't already be in - // data chunk - SLANG_ASSERT(m_listChunk && m_dataChunk == nullptr); - - DataChunk* chunk = _newDataChunk(fourCC); - _addChunk(chunk); - m_dataChunk = chunk; - break; - } - case Chunk::Kind::List: - { - // We can't be in a data chunk - SLANG_ASSERT(m_dataChunk == nullptr); - - ListChunk* list = _newListChunk(fourCC); - - // If this is the first, make it the root - if (!m_rootList) - { - m_rootList = list; - } - - _addChunk(list); - - m_listChunk = list; - break; - } - } + _currentChunk = chunk; + _riffBuilder = chunk ? chunk->getRIFFBuilder() : nullptr; } -void RiffContainer::endChunk() +DataChunkBuilder* BuildCursor::addDataChunk(Chunk::Type type) { - size_t chunkPayloadSize; - - // The chunk we are popping - // Only keep track of this in debug builds - [[maybe_unused]] Chunk* chunk = nullptr; + // The current chunk must be a list chunk, so that + // we can add children to it. + // + auto parentChunk = as<ListChunkBuilder>(getCurrentChunk()); + SLANG_ASSERT(parentChunk); - ListChunk* parent; - if (m_dataChunk) - { - chunk = m_dataChunk; - - parent = m_dataChunk->m_parent; - chunkPayloadSize = m_dataChunk->m_payloadSize; - - m_dataChunk = nullptr; - } - else - { - chunk = m_listChunk; - - SLANG_ASSERT(m_listChunk && m_dataChunk == nullptr); - parent = m_listChunk->m_parent; - chunkPayloadSize = m_listChunk->m_payloadSize; - } - - m_listChunk = parent; - - if (parent) - { - // Fix the size taking into account padding bytes requirement - chunkPayloadSize = RiffUtil::getPadSize(chunkPayloadSize); - // Update the parents size - parent->m_payloadSize += sizeof(RiffHeader) + chunkPayloadSize; - } - - // Check it's size seems ok - SLANG_ASSERT(isChunkOk(chunk)); + return parentChunk->addDataChunk(type); } -void RiffContainer::addDataChunk(FourCC dataFourCC, const void* data, size_t dataSizeInBytes) +void BuildCursor::addDataChunk(Chunk::Type type, void const* data, size_t size) { - startChunk(Chunk::Kind::Data, dataFourCC); - write(data, dataSizeInBytes); + beginDataChunk(type); + addData(data, size); endChunk(); } -void RiffContainer::setPayload(Data* data, const void* payload, size_t size) +ListChunkBuilder* BuildCursor::addListChunk(Chunk::Type type) { - // We must be in a data chunk - SLANG_ASSERT(m_dataChunk); - // The data shouldn't be set up - SLANG_ASSERT(data->m_ownership == Ownership::Uninitialized); - - // Add current chunks data - m_dataChunk->m_payloadSize += size; - - data->m_ownership = Ownership::Arena; - data->m_size = size; - - if (size) - { - data->m_payload = m_arena.allocateAligned(size, kPayloadMinAlignment); - } - - if (payload) + // If there is no current chunk being written into, + // then an attempt to add a new chunk should set + // the root chunk of the entire RIFF. + // + auto currentChunk = getCurrentChunk(); + if (!currentChunk) { - ::memcpy(data->m_payload, payload, size); + SLANG_ASSERT(getRIFFBuilder()); + return _riffBuilder->addRootChunk(type); } -} - -void RiffContainer::moveOwned(Data* data, void* payload, size_t size) -{ - // We must be in a data chunk - SLANG_ASSERT(m_dataChunk); - // The data shouldn't be set up - SLANG_ASSERT(data->m_ownership == Ownership::Uninitialized); - - // Add current chunks data - m_dataChunk->m_payloadSize += size; - data->m_ownership = Ownership::Owned; - data->m_size = size; + // Otherwise, the current chunk must be a list + // chunk, and we add a new child to it. + // + auto parentChunk = as<ListChunkBuilder>(currentChunk); + SLANG_ASSERT(parentChunk); - // The area will manage this block - m_arena.addExternalBlock(payload, size); - data->m_payload = payload; + return parentChunk->addListChunk(type); } -void RiffContainer::setUnowned(Data* data, void* payload, size_t size) +void BuildCursor::beginDataChunk(Chunk::Type type) { - // We must be in a data chunk - SLANG_ASSERT(m_dataChunk); - // The data shouldn't be set up - SLANG_ASSERT(data->m_ownership == Ownership::Uninitialized); - // Add current chunks data - m_dataChunk->m_payloadSize += size; - - data->m_ownership = Ownership::NotOwned; - data->m_size = size; - data->m_payload = payload; -} - -RiffContainer::Data* RiffContainer::addData() -{ - // We must be in a chunk - SLANG_ASSERT(m_dataChunk); - - Data* data = (Data*)m_arena.allocate(sizeof(Data)); - data->init(); - - Data*& next = m_dataChunk->m_endData ? m_dataChunk->m_endData->m_next : m_dataChunk->m_dataList; - SLANG_ASSERT(next == nullptr); - - // Add to linked list - next = data; - // Make this the new end - m_dataChunk->m_endData = data; - return data; + auto chunk = addDataChunk(type); + setCurrentChunk(chunk); } -RiffContainer::Data* RiffContainer::makeSingleData(DataChunk* dataChunk) +void BuildCursor::beginListChunk(Chunk::Type type) { - // There is no data - if (dataChunk->m_dataList == nullptr) - { - return nullptr; - } - - if (dataChunk->m_dataList->m_next == nullptr) - { - return dataChunk->m_dataList; - } - - { - Data* data = dataChunk->m_dataList; - - // Okay lets combine all into one block - const size_t payloadSize = dataChunk->calcPayloadSize(); - - void* dst = m_arena.allocateAligned(payloadSize, kPayloadMinAlignment); - dataChunk->getPayload(dst); - - // Remove other datas - data->m_next = nullptr; - // Make this the end - dataChunk->m_endData = data; - - // Point to the block with all of the data - data->m_ownership = Ownership::Arena; - data->m_payload = dst; - data->m_size = payloadSize; - - return data; - } + auto chunk = addListChunk(type); + setCurrentChunk(chunk); } -void RiffContainer::write(const void* inData, size_t size) +void BuildCursor::endChunk() { - // We must be in a chunk - SLANG_ASSERT(m_dataChunk); - // Get the last data chunk - Data* endData = m_dataChunk->m_endData; - if (endData) - { - uint8_t* end = ((uint8_t*)endData->m_payload) + endData->m_size; - // See if can just add to end of current data - if (end == m_arena.getCursor() && m_arena.allocateCurrentUnaligned(size)) - { - ::memcpy(end, inData, size); - endData->m_size += size; + SLANG_ASSERT(getCurrentChunk() != nullptr); - // Add current chunks data - m_dataChunk->m_payloadSize += size; - return; - } - } - - auto data = addData(); - setPayload(data, inData, size); + auto chunk = getCurrentChunk(); + setCurrentChunk(chunk->getParent()); } -static SlangResult _isChunkOk(RiffContainer::Chunk* chunk, void* data) +void BuildCursor::addData(void const* data, Size size) { - SLANG_UNUSED(data); - return chunk->calcPayloadSize() == chunk->m_payloadSize ? SLANG_OK : SLANG_FAIL; -} + // The current chunk must be a data chunk, so that + // we can add data to it. + // + auto dataChunk = as<DataChunkBuilder>(getCurrentChunk()); + SLANG_ASSERT(dataChunk); -/* static */ bool RiffContainer::isChunkOk(Chunk* chunk) -{ - return SLANG_SUCCEEDED(chunk->visitPostOrder(&_isChunkOk, nullptr)); + dataChunk->addData(data, size); } -static SlangResult _calcAndSetSize(RiffContainer::Chunk* chunk, void* data) +void BuildCursor::addUnownedData(void const* data, Size size) { - SLANG_UNUSED(data); - chunk->m_payloadSize = chunk->calcPayloadSize(); - return SLANG_OK; -} + // The current chunk must be a data chunk, so that + // we can add data to it. + // + auto dataChunk = as<DataChunkBuilder>(getCurrentChunk()); + SLANG_ASSERT(dataChunk); -/* static */ void RiffContainer::calcAndSetSize(Chunk* chunk) -{ - chunk->visitPostOrder(&_calcAndSetSize, nullptr); + dataChunk->addUnownedData(data, size); } +} // namespace RIFF } // namespace Slang diff --git a/source/core/slang-riff.h b/source/core/slang-riff.h index c858158e6..9c533aeb8 100644 --- a/source/core/slang-riff.h +++ b/source/core/slang-riff.h @@ -1,527 +1,1095 @@ +// slang-riff.h #ifndef SLANG_RIFF_H #define SLANG_RIFF_H +// This file defines an API for reading and writing files in the +// RIFF file format. +// +// Some references on the RIFF format include: +// +// * http://fileformats.archiveteam.org/wiki/RIFF +// * http://www.fileformat.info/format/riff/egff.htm +// +// RIFF files, and formats inspired by it, are commonly used as +// binary interchange formats in cases where ad hoc extensibility +// is needed. +// + #include "slang-basic.h" #include "slang-memory-arena.h" -#include "slang-semantic-version.h" #include "slang-stream.h" #include "slang-writer.h" namespace Slang { -// http://fileformats.archiveteam.org/wiki/RIFF -// http://www.fileformat.info/format/riff/egff.htm +// +// An important concept in the RIFF format, as well as +// many derived formats, is a *four-character code*, usually +// referred to as a "FourCC" or "FOURCC". +// +/// A 32-bit value that comprises four ASCII characters. +/// +/// A `FourCC` can be used as a kind of "extensible `enum`" in situations +/// where different developers or groups may want to independently add +/// cases, while minimizing the chances of accidental collisions. +/// +/// A `FourCC` can be efficienctly compared, or used in `switch` +/// statements, which can be an advantage compared to alternative +/// extensible formats like strings or UUIDs. +/// +/// In memory, the characters of a `FourCC` come in the +/// usual order that they would for an array of four `char`s; +/// that is, the first character occupies the byte with the +/// lowest address, and so forth. When that same memory +/// is read as a 32-bit integer, the integer value read will +/// depend on the endianness of the architecture. +/// +struct FourCC +{ +public: + /// The value of a `FourCC`, represented as single integer. + using RawValue = UInt32; -typedef uint32_t FourCC; + FourCC() { _rawValue = 0; } -/* Use of macros to construct and extract from FourCC means the FourCC ordering can be fixed for - * endian differences. */ + FourCC(RawValue rawValue) { _rawValue = rawValue; } + + void operator=(RawValue rawValue) { _rawValue = rawValue; } + + RawValue getRawValue() const { return _rawValue; } + + operator RawValue() const { return _rawValue; } + +private: + // + // The storage for a `FourCC` is defined in a + // way that makes the textual form more visible + // when debugging. + // + union + { + char _text[4]; + RawValue _rawValue; + }; +}; + +// +// Because the integer representation of a `FourCC` depends +// on the endianness of the architecture, we define a macro +// to turn a sequence of four independent characters into +// a single `FourCC::RawValue`, based on the target +// architecture. +// #if SLANG_LITTLE_ENDIAN -#define SLANG_FOUR_CC(c0, c1, c2, c3) \ - ((FourCC(c0) << 0) | (FourCC(c1) << 8) | (FourCC(c2) << 16) | (FourCC(c3) << 24)) +#define SLANG_FOUR_CC(c0, c1, c2, c3) \ + ((FourCC::RawValue(c0) << 0) | (FourCC::RawValue(c1) << 8) | (FourCC::RawValue(c2) << 16) | \ + (FourCC::RawValue(c3) << 24)) #else -#define SLANG_FOUR_CC(c0, c1, c2, c3) \ - ((FourCC(c0) << 24) | (FourCC(c1) << 16) | (FourCC(c2) << 8) | (FourCC(c3) << 0)) - +#define SLANG_FOUR_CC(c0, c1, c2, c3) \ + ((FourCC::RawValue(c0) << 24) | (FourCC::RawValue(c1) << 16) | (FourCC::RawValue(c2) << 8) | \ + (FourCC::RawValue(c3) << 0)) #endif -enum + +namespace RIFF { - kRiffPadSize = 2, ///< We only align to 2 bytes - kRiffPadMask = kRiffPadSize - 1, -}; -// Uses it's own version of a hash -typedef int RiffHashCode; +struct Chunk; +struct DataChunk; +struct ListChunk; +struct RootChunk; +class ChunkBuilder; +class ListChunkBuilder; +class DataChunkBuilder; +struct Builder; -struct RiffHeader -{ - FourCC type; ///< The FourCC code that identifies this chunk - uint32_t size; ///< Size does *NOT* include the riff chunk size. The size can be byte sized, but - ///< on storage it will always be treated as aligned up by 4. -}; +// +// A RIFF file is organized as a tree of *chunks*. +// -struct RiffListHeader +/// A chunk in a RIFF file. +/// +struct Chunk { - RiffHeader chunk; - FourCC subType; - // This is then followed by the contained subchunk/s -}; +public: + // + // The starting offset of a chunk in a RIFF file + // is only guaranteed to be 2-byte aligned. + // Code that reads from a chunk must be cautious about + // the possibility of performing unaligned loads. + // + + /// Required alignment for the starting offset of a chunk. + static const UInt32 kChunkAlignment = 2; + + // + // Every chunk starts with a *header*, which includes + // a *tag* used to identify the kind/type of chunk + // as well a representation of the size of the chunk + // in bytes. + // + struct Header + { + // + // Note that when loading a RIFF file into memory, + // chunks may not start on 4-byte-aligned boundaries, + // so code should not directly read the following + // fields unless preconditions exist to guarantee + // higher alignment. + // + + /// Tag for this chunk. + /// + /// * For a data chunk, this will be its type. + /// * For a list chunk this will be `"LIST"`. + /// * For a root chunk this will be `"RIFF"`. + /// + FourCC tag; + + /// Size in bytes of this chunk, not including this header. + UInt32 size; + }; -struct RiffFourCC -{ - /// A 'riff' is the high level file container. It is followed by a subtype and then the - /// contained chunks. - static const FourCC kRiff = SLANG_FOUR_CC('R', 'I', 'F', 'F'); - /// A list is the same as a 'riff' except can be placed anywhere in hierarchy. - static const FourCC kList = SLANG_FOUR_CC('L', 'I', 'S', 'T'); + /// Get the header for this chunk. + Header const* getHeader() const { return (Header const*)this; } + + /// Get the tag from the header of this chunk. + FourCC getTag() const { return _readTagFromHeader(); } + + /// Get the total size of this chunk, in bytes. + /// + /// This size includes the chunk header. + /// + UInt32 getTotalSize() const { return sizeof(RIFF::Chunk) + _readSizeFromHeader(); } + + // + // There are three *kinds* of chunks that can appear in a RIFF: + // + // * *data chunks* contain zero or more bytes of data. + // + // * *list chunks* contain a sequence of other chunks + // + // * a *root chunk* is a special case of list chunk that + // is used as the root of the chunk hierarchy in a RIFF file. + // + // List chunks are identified by having a tag of `"LIST"` + // in their header, while root chunks have a tag of `"RIFF"` + // + + /// Kind of a chunk. + enum class Kind + { + Data, + List, + Root, + }; -private: - RiffFourCC() = delete; -}; + /// Get the kind of this chunk. + Kind getKind() const; -// Follows semantic version rules -// https://semver.org/ -// -// major.minor.patch -// Patch versions indicate a change. -// Minor means a change that is backwards compatible with previous minor versions. A step in minor -// and/or major zeros patch. Major means a non compatible change. A step in major, zeros minor and -// patch. -struct RiffSemanticVersion -{ - typedef RiffSemanticVersion ThisType; - typedef uint32_t RawType; + // + // Every chunk has a *type*, which is a `FourCC`. + // + // For data chunks, the type is stored as the tag + // of the chunk header, while for list and root + // chunks the type is stored immediately after + // the chunk header. + // - /// == - bool operator==(const ThisType& rhs) const { return m_raw == rhs.m_raw; } - bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } + /// Type of a chunk. + using Type = FourCC; - /// A patch change indices a different version but does not change the compatibility of the - /// format - int getPatch() const { return m_raw & 0xff; } - /// A minor change implies a format change that is backwards compatible - int getMinor() const { return (m_raw >> 8) & 0xff; } - /// A major change is binary incompatible by default - int getMajor() const { return (m_raw >> 16); } + /// Get the type of this chunk. + Type getType() const; - SemanticVersion asSemanticVersion() const - { - return SemanticVersion(getMajor(), getMinor(), getPatch()); - } +private: + Header _header; - static RawType makeRaw(int major, int minor, int patch) +protected: + // + // Rather than directly reading the `_tag` or `_size` + // members, code should use the following accessors, + // which account for possible alignment issues. + // + + FourCC _readTagFromHeader() const { - SLANG_ASSERT((major | minor | patch) >= 0); - SLANG_ASSERT(major < 0x10000 && minor < 0x100 && patch < 0x100); - return (RawType(major) << 16) | (RawType(minor) << 8) | RawType(patch); + auto header = getHeader(); + FourCC result; + memcpy(&result, &header->tag, sizeof(header->tag)); + return result; } - static RiffSemanticVersion makeFromRaw(RawType raw) + UInt32 _readSizeFromHeader() const { - ThisType version; - version.m_raw = raw; - return version; + auto header = getHeader(); + UInt32 result; + memcpy(&result, &header->size, sizeof(header->size)); + return result; } +}; - static RiffSemanticVersion make(int major, int minor, int patch) +/// A chunk that contains zero or more bytes of payload data. +struct DataChunk : Chunk +{ +public: + /// Get the size in bytes of the payload data of this chunk. + UInt32 getPayloadSize() const { return _readSizeFromHeader(); } + + /// Get a pointer to the payload data of this chunk. + /// + /// Note that this pointer is only guaranteed to be aligned + /// up to `RIFF::Chunk::kAlignment`, for chunks of a RIFF + /// file loaded directly into memory. + /// + void const* getPayload() const { return static_cast<void const*>(this + 1); } + + /// Write the payload data of this chunk into the given buffer. + /// + /// The payload must be at least `size` bytes. + /// If the payload is larger than `size` bytes, only the + /// first `size` bytes will be written to the buffer. + /// + void writePayloadInto(void* outData, Size size) const; + + /// Get the payload data of this chunk. + /// + template<typename T> + void writePayloadInto(T& outValue) const { - return makeFromRaw(makeRaw(major, minor, patch)); + writePayloadInto(&outValue, sizeof(outValue)); } - static RiffSemanticVersion make(const SemanticVersion& in) + + /// Get the payload data of this chunk. + /// + template<typename T> + T readPayloadAs() const { - return makeFromRaw(makeRaw(in.m_major, in.m_minor, in.m_patch)); + T result; + writePayloadInto(result); + return result; } - /// True if the read version is compatible with the current version, based on semantic rules. - static bool areCompatible(const ThisType& currentVersion, const ThisType& readVersion) + /// Get the type of this chunk. + Type getType() const { - const RawType currentRaw = currentVersion.m_raw; - const RawType readRaw = readVersion.m_raw; + // The type of a data chunk is just the tag + // from the chunk header. - // Must have same major version. - // For minor version, the read version must be less than or equal. - return ((currentRaw & 0xffff0000) == (readRaw & 0xffff0000)) && - ((currentRaw & 0xff00) >= (readRaw & 0xff00)); + return getTag(); } - RawType m_raw; + /// Determine if a chunk is an instance of this kind. + static bool _isChunkOfThisKind(Chunk const* chunk) + { + return chunk->getKind() == Chunk::Kind::Data; + } }; -/* A helper class that makes reading data from a data block simpler */ -class RiffReadHelper +/// A pointer to a `RIFF::Chunk` that is dynamically +/// checked to make sure access doesn't go past a +/// certain size bound. +/// +struct BoundsCheckedChunkPtr { public: - template<typename T> - SlangResult read(T& out) - { - if (m_cur + sizeof(T) > m_end) - { - return SLANG_FAIL; - } - // TODO: consider whether this type should enforce alignment. - // SLANG_ASSERT((size_t(m_cur) & (SLANG_ALIGN_OF(T) - 1)) == 0); - ::memcpy(&out, m_cur, sizeof(T)); - m_cur += sizeof(T); - return SLANG_OK; - } + /// Initialize a null pointer + BoundsCheckedChunkPtr() {} - /// Get the data - const uint8_t* getData() const { return m_cur; } - /// Get the remaining size - size_t getRemainingSize() const { return size_t(m_end - m_cur); } + /// Initialize a null pointer + BoundsCheckedChunkPtr(nullptr_t) {} - RiffReadHelper(const uint8_t* data, size_t size) - : m_start(data), m_end(data + size), m_cur(data) - { - } - SlangResult skip(size_t size) - { - if (m_cur + size > m_end) - { - return SLANG_FAIL; - } - m_cur += size; - return SLANG_OK; - } + /// Initialize a pointer to a chunk, with a size limit. + BoundsCheckedChunkPtr(Chunk const* chunk, Size sizeLimit) { _set(chunk, sizeLimit); } -protected: - const uint8_t* m_start; - const uint8_t* m_end; - const uint8_t* m_cur; -}; + /// Initialize a pointer to a chunk, with a limit based on its reported size. + BoundsCheckedChunkPtr(Chunk const* chunk) { _set(chunk); } + + /// Get the underlying chunk pointer. + Chunk const* get() const { return _ptr; } -/* A container for data in RIFF format. Holds the contents in memory. + operator Chunk const*() const { return get(); } + Chunk const* operator->() const { return get(); } -With the data held in memory allows for adding or removing chunks at will. + BoundsCheckedChunkPtr getNextSibling() const; + +private: + Chunk const* _ptr = nullptr; + Size _sizeLimit = 0; + + void _set(Chunk const* chunk, Size sizeLimit); + void _set(Chunk const* chunk); +}; -A future implementation does not necessarily have to be backed by memory when construction, -as data could be written to stream, and the chunk sizes written by seeking back over the file and -setting the value. -In normal usage the chunk sizes are calculated during construction. If the structure is changed, the -sizes may need to be recalculated, before serialization. -*/ -class RiffContainer +template<typename T = Chunk> +struct ChunkList { public: - // This alignment is only made for arena based allocations. - // For external blocks it's client code to have appropriate alignment. - // This is needed because when reading a RiffContainer, all allocation is arena based, and - // if the payload contains 8 byte aligned data, the overall payload needs to be 8 byte aligned. - static const size_t kPayloadMinAlignment = 8; - - enum class Ownership - { - Uninitialized, ///< Doesn't contain anything - NotOwned, ///< It's not owned by the container - Arena, ///< It's owned and allocated on the arena - Owned, ///< It's owned, but wasn't allocated on the arena - }; + ChunkList() {} + + ChunkList(BoundsCheckedChunkPtr firstChunk) + : _firstChunk(firstChunk) + { + } - struct Data + struct Iterator { - /// Get the payload - void* getPayload() { return m_payload; } - /// Get the end pointer - void* getPayloadEnd() { return (void*)((uint8_t*)m_payload + m_size); } - /// Get the size of the payload - size_t getSize() const { return m_size; } - /// Get the ownership of the data held in the payload - Ownership getOwnership() const { return m_ownership; } + public: + Iterator() {} - void init() + Iterator(BoundsCheckedChunkPtr chunk) + : _chunk(chunk) { - m_ownership = Ownership::Uninitialized; - m_size = 0; - m_next = nullptr; - m_payload = nullptr; } - Ownership m_ownership; ///< Stores the ownership of the payload - size_t m_size; ///< The size of the payload - void* m_payload; ///< The payload - Data* m_next; ///< The next Data block in the list - }; + T const* operator*() const { return static_cast<T const*>(_chunk.get()); } + + void operator++() { _chunk = _chunk.getNextSibling(); } + + bool operator!=(Iterator const& that) const { return _chunk != that._chunk; } - struct Chunk; - struct ListChunk; - struct DataChunk; + private: + BoundsCheckedChunkPtr _chunk; + }; - typedef SlangResult (*VisitorCallback)(Chunk* chunk, void* data); + Iterator begin() const { return Iterator(_firstChunk); } + Iterator end() const { return Iterator(); } - class Visitor; - struct Chunk + template<typename U> + ChunkList<U> cast() const { - enum class Kind - { - List, ///< Strictly speaking this can be a 'LIST' or a 'RIFF' as they have the same - ///< structure - Data, - }; + return ChunkList<U>(_firstChunk); + } - void init(Kind kind, FourCC fourCC) - { - m_kind = kind; - m_fourCC = fourCC; - m_payloadSize = 0; - m_next = nullptr; - m_parent = nullptr; - } + T const* getFirst() const { return *begin(); } + +private: + friend struct ListChunk; - SlangResult visit(Visitor* visitor); - SlangResult visitPostOrder(VisitorCallback callback, void* data); - SlangResult visitPreOrder(VisitorCallback callback, void* data); + BoundsCheckedChunkPtr _firstChunk; +}; - /// Returns a single data chunk - Data* getSingleData() const; +struct ListChunk : Chunk +{ +public: + // + // A (non-root) list chunk has a tag of `"LIST"` + // in its header. + // - /// Calculate the payload size - size_t calcPayloadSize(); + static const FourCC::RawValue kTag = SLANG_FOUR_CC('L', 'I', 'S', 'T'); - Kind m_kind; ///< Kind of chunk - FourCC m_fourCC; ///< The chunk type for data, or the sub type for a List (riff/list) - size_t m_payloadSize; ///< The payload size (ie does NOT include RiffChunk header). - Chunk* m_next; ///< Next chunk in this list - ListChunk* m_parent; ///< The chunk this belongs to - }; + // + // A list chunk starts with a header, as all chunks do, + // but for a list chunk the ordinary `Chunk::Header` + // is followed by an additional `FourCC`, to specify + // the type of the list chunk. + // - struct ListChunk : public Chunk + struct Header { - typedef Chunk Super; - SLANG_FORCE_INLINE static bool isType(const Chunk* chunk) - { - return chunk->m_kind == Kind::List; - } + // + // As is the case with any other chunk, code that + // wants to access these fields should be mindful + // of the way that a RIFF does not guarantee 4-byte + // alignment for chunks. + // + + /// The base chunk header. + Chunk::Header chunkHeader; + + /// The type of this list chunk. + Type type; + }; - void init(FourCC subType) - { - Super::init(Kind::List, subType); - m_containedChunks = nullptr; - m_endChunk = nullptr; + /// Get the header for this list chunk. + Header const* getHeader() const { return (Header const*)this; } - m_payloadSize = uint32_t(sizeof(RiffListHeader) - sizeof(RiffHeader)); - } + // + // The content of a list chunk comprises zero or more + // child chunks, organized as a kind of linked list. + // + // The starting offset for each successive child chunk + // is the end offset of the previous child chunk, rounded + // up to the required alignment (`RIFF::Chunk::kAlignment`). + // - /// Finds chunk (list or data) that matches type. For List/Riff, type is the subtype - Chunk* findContained(FourCC type) const; + /// List of child chunks. + using ChildList = ChunkList<>; - void* findContainedData(FourCC type, size_t minSize) const; + /// Get the list of children of this chunk. + ChildList getChildren() const { return ChildList(getFirstChild()); } - ListChunk* findContainedList(FourCC type); + /// Get the first child chunk (if any) of this chunk. + /// + /// The list of child chunks can be navigated using + /// the `BoundCheckedChunkPtr::getNextSibling` operation. + /// + BoundsCheckedChunkPtr getFirstChild() const; - /// Finds the contained data. NOTE! Assumes that there is only as single data block, and - /// will return nullptr if there is not - Data* findContainedData(FourCC type) const; + /// Find a child data chunk of the given `type`. + DataChunk const* findDataChunk(Chunk::Type type) const; - template<typename T> - T* findContainedData(FourCC type) const - { - return (T*)findContainedData(type, sizeof(T)); - } + /// Find a child list chunk of the given `type`. + ListChunk const* findListChunk(Chunk::Type type) const; - /// Find all contained that match the type - void findContained(FourCC type, List<ListChunk*>& out); + /// Recursively search for a list chunk of the given `type`. + /// + /// Will consider this chunk itself as a possible match. + /// + ListChunk const* findListChunkRec(Chunk::Type type) const; - /// Find all contained that match the type - void findContained(FourCC type, List<DataChunk*>& out); + /// Get the type of this chunk. + Type getType() const { return _readTypeFromHeader(); } - /// Find the list (including self) that matches subtype recursively - ListChunk* findListRec(FourCC subType); + /// Determine if a chunk is an instance of this kind. + static bool _isChunkOfThisKind(Chunk const* chunk) + { + // Anything that isn't a data chunk is a list. + return chunk->getKind() != Chunk::Kind::Data; + } - /// NOTE! Assumes all contained chunks have correct payload sizes - size_t calcPayloadSize(); +private: + // + // Because we are inheriting from `Chunk`, we do not + // declare a full `ListChunk::Header` here, and instead + // just declare the additional field that appears after + // the base header. + // + + Type _type; + + // + // The `_type` field is mostly just there for debugging + // purposes; when actually reading from the header, we + // make use of a cast. + // + + Type _readTypeFromHeader() const + { + auto header = getHeader(); + Type result; + memcpy(&result, &header->type, sizeof(header->type)); + return result; + } +}; - /// Get the sub type - FourCC getSubType() const { return m_fourCC; } +struct RootChunk : ListChunk +{ +public: + // + // A root chunk has a tag of `"RIFF"` in its header. + // + + static const FourCC::RawValue kTag = SLANG_FOUR_CC('R', 'I', 'F', 'F'); + + /// Get a pointer to the root chunk of a RIFF hierarchy stored in a data blob. + /// + /// Performs some minimal validity checks, and returns `nullptr` if + /// the blob provided does not superficially appear to be a valid RIFF. + /// + static RootChunk const* getFromBlob(void const* data, size_t dataSize); + + /// Get a pointer to the root chunk of a RIFF hierarchy stored in a data blob. + /// + /// Performs some minimal validity checks, and returns `nullptr` if + /// the blob provided does not superficially appear to be a valid RIFF. + /// + static RootChunk const* getFromBlob(ISlangBlob* blob); + + /// Determine if a chunk is an instance of this kind. + static bool _isChunkOfThisKind(Chunk const* chunk) + { + return chunk->getKind() == Chunk::Kind::Root; + } - /// A singly linked list of contained chunks directly contained in this chunk - Chunk* getFirstContainedChunk() const { return m_containedChunks; } +private: + static bool _isTagForThisKind(FourCC tag) { return tag == kTag; } +}; - Chunk* m_containedChunks; ///< The contained chunks - Chunk* m_endChunk; ///< The last chunk (only set when pushed, and used when popped) - }; +inline Chunk::Kind Chunk::getKind() const +{ + switch (getTag()) + { + case RootChunk::kTag: + return Chunk::Kind::Root; + case ListChunk::kTag: + return Chunk::Kind::List; + default: + return Chunk::Kind::Data; + } +} - struct DataChunk : public Chunk +inline Chunk::Type Chunk::getType() const +{ + auto tag = getTag(); + switch (tag) { - typedef Chunk Super; - SLANG_FORCE_INLINE static bool isType(const Chunk* chunk) - { - return chunk->m_kind == Kind::Data; - } + case RootChunk::kTag: + case ListChunk::kTag: + return static_cast<ListChunk const*>(this)->getType(); + + default: + return tag; + } +} - /// Calculate a hash (not necessarily very fast) - RiffHashCode calcHash() const; - /// Calculate the payload size - size_t calcPayloadSize() const; +/// Cast a `Chunk` to a sub-type of `Chunk`. +template<typename T> +T* as(Chunk* chunk) +{ + if (!chunk) + return nullptr; + if (!T::_isChunkOfThisKind(chunk)) + return nullptr; + return static_cast<T*>(chunk); +} - /// Copy the payload to dst. Dst must be at least the payload size. - void getPayload(void* dst) const; +/// Cast a `Chunk` to a sub-type of `Chunk`. +template<typename T> +T const* as(Chunk const* chunk) +{ + if (!chunk) + return nullptr; + if (!T::_isChunkOfThisKind(chunk)) + return nullptr; + return static_cast<T const*>(chunk); +} - /// True if payloads contents is equal to data - bool isEqual(const void* data, size_t count) const; +/// A linked list where the elements are themselves the nodes. +/// +template<typename T> +struct InternallyLinkedList +{ +public: + struct Node + { + public: + Node() {} - /// Get single data payload. - Data* getSingleData() const; + private: + friend struct InternallyLinkedList<T>; + T* _next = nullptr; + }; - /// Return as read helper - RiffReadHelper asReadHelper() const; + struct Iterator + { + public: + Iterator() {} - void init(FourCC fourCC) + Iterator(T* node) + : _node(node) { - Super::init(Kind::Data, fourCC); - m_dataList = nullptr; - m_endData = nullptr; } - Data* m_dataList; ///< List of 0 or more data items - Data* m_endData; ///< The last data point + T* operator*() const { return _node; } + + void operator++() { _node = static_cast<Node const*>(_node)->_next; } + + bool operator!=(Iterator const& that) const { return _node != that._node; } + + private: + T* _node = nullptr; }; - class ScopeChunk + Iterator begin() { return Iterator(_first); } + + Iterator end() { return Iterator(); } + + T* getFirst() const { return _first; } + + T* getLast() const { return _last; } + + void add(T* element) { - public: - ScopeChunk(RiffContainer* container, Chunk::Kind kind, FourCC fourCC) - : m_container(container) + if (!_last) { - container->startChunk(kind, fourCC); + SLANG_ASSERT(_first == nullptr); + + _first = element; + _last = element; } - ~ScopeChunk() { m_container->endChunk(); } + else + { + SLANG_ASSERT(_first != nullptr); - private: - RiffContainer* m_container; - }; + _last->_next = element; + _last = element; + } + } + +private: + T* _first = nullptr; + T* _last = nullptr; +}; + +/// A builder for a chunk in a RIFF. +class ChunkBuilder : public InternallyLinkedList<ChunkBuilder>::Node +{ +public: + /// Get the kind of the chunk being built. + Chunk::Kind getKind() const { return _kind; } + + /// Get the type of the chunk being built. + Chunk::Type getType() const { return _type; } + + /// Set the type of the chunk being built. + void setType(Chunk::Type type) { _type = type; } + + /// Get the parent chunk of this chunk in the RIFF hierarchy. + /// + ListChunkBuilder* getParent() const { return _parent; } - class Visitor + /// Get the RIFF builder that this chunk belongs to. + /// + RIFF::Builder* getRIFFBuilder() const { return _riffBuilder; } + +protected: + ChunkBuilder( + Chunk::Kind kind, + Chunk::Type type, + ListChunkBuilder* parent, + RIFF::Builder* riffBuilder) + : _kind(kind), _type(type), _parent(parent), _riffBuilder(riffBuilder) { - public: - virtual SlangResult enterList(ListChunk* list) = 0; - virtual SlangResult handleData(DataChunk* data) = 0; - virtual SlangResult leaveList(ListChunk* list) = 0; - }; + } + + ChunkBuilder(ChunkBuilder const&) = delete; + void operator=(ChunkBuilder const&) = delete; + MemoryArena& _getMemoryArena() const; + +private: + Chunk::Kind _kind = Chunk::Kind(-1); + Chunk::Type _type = 0; + ListChunkBuilder* _parent = nullptr; + Builder* _riffBuilder = nullptr; + + // A cached total size for this chunk. This + // is only valid after `_updateCachedTotalSize()` + // has been called, and before any subsequent + // changes to the content of this chunk or any + // of its descendents in the hierarchy. + // + mutable Size _cachedTotalSize = 0; + + Size _updateCachedTotalSize() const; + + Size _getCachedTotalSize() const { return _cachedTotalSize; } + + /// Write the binary representation of this chunk to the given `stream` + /// + /// Assumes that `_updateCachedTotalSize` has been used + /// so that the cached total size of this chunk is valid. + /// + Result _writeTo(Stream* stream) const; + + friend struct Builder; +}; - /// Add a complete data chunk - void addDataChunk(FourCC dataFourCC, const void* data, size_t dataSizeInBytes); +class ListChunkBuilder : public ChunkBuilder +{ +public: + /// A list of child chunks. + using ChildList = InternallyLinkedList<ChunkBuilder>; - /// Start a chunk - void startChunk(Chunk::Kind kind, FourCC type); + /// Get the child chunks of this list. + ChildList getChildren() const { return _children; } - /// Write data into a chunk (can only be inside a Kind::Data) - void write(const void* data, size_t size); + /// Append a new data chunk to the current list chunk. + DataChunkBuilder* addDataChunk(Chunk::Type type); - /// Adds an empty data block - Data* addData(); - /// Set the payload on a data. Payload can be passed as nullptr, if it is no memory will be - /// copied. - void setPayload(Data* data, const void* payload, size_t size); + /// Append a new data chunk to the current list chunk. + ListChunkBuilder* addListChunk(Chunk::Type type); - /// Move ownership to. - /// NOTE! The payload *must* be deallocatable via 'free' - void moveOwned(Data* data, void* payload, size_t size); - /// Move unowned. The payload scope must last longer than the RiffContainer - void setUnowned(Data* data, void* payload, size_t size); + /// Determine if a chunk is an instance of this kind. + static bool _isChunkOfThisKind(ChunkBuilder const* chunk) + { + return chunk->getKind() != Chunk::Kind::Data; + } - /// End a chunk - void endChunk(); +private: + ListChunkBuilder(Chunk::Type type, ListChunkBuilder* parent) + : ChunkBuilder(Chunk::Kind::List, type, parent, parent->getRIFFBuilder()) + { + } - /// Get the root - ListChunk* getRoot() const { return m_rootList; } + friend struct RIFF::Builder; - /// Get the current chunk - Chunk* getCurrentChunk() + ListChunkBuilder(Chunk::Type type, RIFF::Builder* riffBuilder) + : ChunkBuilder(Chunk::Kind::Root, type, nullptr, riffBuilder) { - return m_dataChunk ? static_cast<Chunk*>(m_dataChunk) : static_cast<Chunk*>(m_listChunk); } - /// Reset the container - void reset(); + ChildList _children; +}; + + +/// A builder for a data chunk in a RIFF. +class DataChunkBuilder : public ChunkBuilder +{ +public: + /// Append data to this chunk. + void addData(void const* data, Size size); - /// true if has a root container, and nothing remains open - bool isFullyConstructed() + /// Append data to this chunk. + template<typename T> + void addData(T const& value) { - return m_rootList && m_listChunk == nullptr && m_dataChunk == nullptr; + addData(&value, sizeof(value)); } - /// Makes a data chunk contain a single contiguous data block - Data* makeSingleData(DataChunk* dataChunk); + /// Append existing data to this chunk. + /// + /// The caller takes responsibility for ensuring that + /// the passed-in data pointer will remain valid for + /// the rest of the lifetime of the enclosing RIFF + /// builder. + /// + void addUnownedData(void const* data, size_t size); + + // + // While the payload of a chunk in a RIFF file is + // contiguous, the payload of a `DataChunkBuilder` + // can span multiple different allocations, which + // this implementation refers to as *shards*. + // + // Each shard has a contiguous payload, and the + // `DataChunkBuilder` owns a list of shards. The + // logical payload of the data chunk is the + // concatenation of the payloads of its shards. + // + + /// A contiguous range of bytes in a `RIFF::DataChunkBuilder` + class Shard : public InternallyLinkedList<Shard>::Node + { + public: + /// Get the payload of this shard. + void const* getPayload() const { return _payload; } - /// Get the memory arena that is backing the storage of data - MemoryArena& getMemoryArena() { return m_arena; } + /// Get the size of the payload of this shard. + Size getPayloadSize() const { return _payloadSize; } - /// The if the list and sublists appear correct - static bool isChunkOk(Chunk* chunk); + private: + friend class DataChunkBuilder; - /// Traverses over chunk hierarchy and sets the sizes - static void calcAndSetSize(Chunk* chunk); + Shard() {} - /// Ctor - RiffContainer(); + void setPayload(void const* data, Size size) + { + _payload = data; + _payloadSize = size; + } - void setCurrentChunk(Chunk* chunk); + void const* _payload = nullptr; + Size _payloadSize = 0; + }; -protected: - void _addChunk(Chunk* chunk); - ListChunk* _newListChunk(FourCC subType); - DataChunk* _newDataChunk(FourCC type); + /// List of shards in a data chunk. + using ShardList = InternallyLinkedList<Shard>; - ListChunk* m_rootList; ///< Root list + /// Get the list of shards that make up this chunk. + ShardList getShards() const { return _shards; } - ListChunk* m_listChunk; - DataChunk* m_dataChunk; + /// Determine if a chunk is an instance of this kind. + static bool _isChunkOfThisKind(ChunkBuilder const* chunk) + { + return chunk->getKind() == Chunk::Kind::Data; + } - MemoryArena m_arena; ///< Can be used to use other owned blocks +private: + friend class ListChunkBuilder; + + DataChunkBuilder(Chunk::Type type, ListChunkBuilder* parent) + : ChunkBuilder(Chunk::Kind::Data, type, parent, parent->getRIFFBuilder()) + { + } + + Shard* _addShard(); + + ShardList _shards; }; -// ----------------------------------------------------------------------------- template<typename T> -T* as(RiffContainer::Chunk* chunk) +T* as(ChunkBuilder* chunk) { - return chunk && T::isType(chunk) ? static_cast<T*>(chunk) : nullptr; + if (!chunk) + return nullptr; + if (!T::_isChunkOfThisKind(chunk)) + return nullptr; + return static_cast<T*>(chunk); } -// ----------------------------------------------------------------------------- + template<typename T> -T* as(RiffContainer::Chunk* chunk, FourCC fourCC) +T const* as(ChunkBuilder const* chunk) { - return chunk && chunk->m_fourCC == fourCC && T::isType(chunk) ? static_cast<T*>(chunk) - : nullptr; + if (!chunk) + return nullptr; + if (!T::_isChunkOfThisKind(chunk)) + return nullptr; + return static_cast<T const*>(chunk); } -struct RiffUtil +/// A builder for a RIFF-structured file. +/// +struct Builder +{ +public: + /// Initialize a builder with an empty tree of chunks. + Builder(); + + /// Write the built hierarchy out to the given `stream`. + Result writeTo(Stream* stream); + + /// Write the built hierarchy out as a blob. + Result writeToBlob(ISlangBlob** outBlob); + + /// Get the root chunk of the RIFF being built. + /// + /// If a root chunk has not yet been added, returns `nullptr`. + /// + ListChunkBuilder* getRootChunk() const { return _rootChunk; } + + /// Add a root chunk to the RIFF being built. + /// + /// There must not already be a root chunk. + /// + /// Returns the root chunk that was added. + /// + ListChunkBuilder* addRootChunk(Chunk::Type type); + + /// Get the memory arena used for allocation. + /// + /// This arena is used for allocating all of the chunk + /// builders, as well as their data. + /// + /// Note: typical use cases should never need to + /// access this; it is part of the public API + /// primarily to enable some of the unit tests. + /// + MemoryArena& _getMemoryArena() { return _arena; } + +private: + Builder(Builder const&) = delete; + void operator=(Builder const&) = delete; + + /// The root chunk of the RIFF. + ListChunkBuilder* _rootChunk = nullptr; + + /// Arena to use for all allocations. + MemoryArena _arena; +}; + +/// A stateful cursor for a RIFF::Builder. +/// +/// Represents a kind of pointer to a location in +/// the hierarchy of RIFF chunks, and allows for +/// new chunks to be added at that location. +/// +struct BuildCursor { - typedef RiffContainer::Chunk Chunk; - typedef RiffContainer::ListChunk ListChunk; - typedef RiffContainer::DataChunk DataChunk; +public: + /// Construct a cursor writing into no chunk. + BuildCursor(); + + /// Construct a cursor writing into the given `chunk`. + BuildCursor(ChunkBuilder* chunk); + + /// Construct a cursor writing at the root of the given `builder`. + /// + /// Note that this is not the same as constructing a + /// cursor for the root chunk of `builder`. Instead, adding + /// a chunk via this cursor will add/create the root chunk + /// of the entire RIFF hierarchy. + /// + BuildCursor(Builder& builder); + + /// Get the RIFF being written into, if any. + RIFF::Builder* getRIFFBuilder() const { return _riffBuilder; } + + /// Get the current chunk being written into, if any. + ChunkBuilder* getCurrentChunk() const { return _currentChunk; } + + /// Set the current chunk to write into. + void setCurrentChunk(ChunkBuilder* chunk); + + /// Append a new data chunk to the current list chunk. + DataChunkBuilder* addDataChunk(Chunk::Type type); + + /// Append a complete data chunk to the current list chunk. + void addDataChunk(Chunk::Type type, void const* data, size_t size); + + /// Append a new data chunk to the current list chunk. + ListChunkBuilder* addListChunk(Chunk::Type type); + + /// Begin a new data chunk as a child of the current list chunk. + /// + /// On return, the cursor will be set to write into the new chunk. + /// + void beginDataChunk(Chunk::Type type); + + /// Begin a new list chunk as a child of the current list chunk. + /// + /// On return, the cursor will be set to write into the new chunk. + /// + void beginListChunk(Chunk::Type type); + + /// End the current chunk. + /// + /// Sets the cursor to write to the parent of the chunk that was ended. + /// + void endChunk(); + + /// Append data onto the current data chunk. + void addData(void const* data, Size size); + + /// Write data onto the current data chunk. + template<typename T> + void addData(T const& value) + { + addData(&value, sizeof(value)); + } + + /// Append existing data to the current data chunk. + /// + /// The caller takes responsibility for ensuring that + /// the passed-in data pointer will remain valid for + /// the rest of the lifetime of the enclosing RIFF + /// builder. + /// + void addUnownedData(void const* data, Size size); + + /// Base type for RAII helpers to pair begin/end chunk calls. + struct ScopedChunk + { + protected: + ScopedChunk(BuildCursor& cursor) + : _cursor(cursor) + { + } + + ~ScopedChunk() { _cursor.endChunk(); } + + private: + BuildCursor& _cursor; + }; + + struct ScopedDataChunk : ScopedChunk + { + public: + ScopedDataChunk(BuildCursor& cursor, Chunk::Type type) + : ScopedChunk(cursor) + { + cursor.beginDataChunk(type); + } + }; + + struct ScopedListChunk : ScopedChunk + { + public: + ScopedListChunk(BuildCursor& cursor, Chunk::Type type) + : ScopedChunk(cursor) + { + cursor.beginListChunk(type); + } + }; + +private: + RIFF::Builder* _riffBuilder = nullptr; + ChunkBuilder* _currentChunk = nullptr; +}; + +#define SLANG_SCOPED_RIFF_BUILDER_DATA_CHUNK(CURSOR, TYPE) \ + ::Slang::RIFF::BuildCursor::ScopedDataChunk _scopedRIFFBuilderDataChunk(CURSOR, TYPE) - static int64_t calcChunkTotalSize(const RiffHeader& chunk); +#define SLANG_SCOPED_RIFF_BUILDER_LIST_CHUNK(CURSOR, TYPE) \ + ::Slang::RIFF::BuildCursor::ScopedListChunk _scopedRIFFBuilderListChunk(CURSOR, TYPE) - static SlangResult skip(const RiffHeader& chunk, Stream* stream, int64_t* remainingBytesInOut); +} // namespace RIFF - static SlangResult readChunk(Stream* stream, RiffHeader& outChunk); +/// A simple helper for reading from a blob. +/// +struct MemoryReader +{ + // + // TODO: This type should eventually either find + // a home somewhere that has nothing to do with + // RIFF files, or its usage in RIFF-related contexts + // should be replaced with other types. + // - static SlangResult writeData( - const RiffHeader* header, - size_t headerSize, - const void* payload, - size_t payloadSize, - Stream* out); - static SlangResult readData( - Stream* stream, - RiffHeader* outHeader, - size_t headerSize, - List<uint8_t>& data); +public: + /// Initialize a reader with no bytes remaining. + /// + MemoryReader() {} - static SlangResult readPayload(Stream* stream, size_t size, void* outData, size_t& outReadSize); + /// Initialize a reader for the given blob. + MemoryReader(void const* data, Size size) + : _cursor(static_cast<Byte const*>(data)), _remainingSize(size) + { + } - /// Read a header. Handles special case of list/riff types - static SlangResult readHeader(Stream* stream, RiffListHeader& outHeader); + /// Read data into the given buffer. + /// + /// Fails if `size` is greater than the + /// amount of data remaining. + /// + SlangResult read(void* dst, Size size) + { + if (size > getRemainingSize()) + { + return SLANG_FAIL; + } + ::memcpy(dst, _cursor, size); + _cursor += size; + _remainingSize -= size; + return SLANG_OK; + } - /// True if the type is a container type - static bool isListType(FourCC type) + /// Read data into the given value. + /// + /// Fails if `sizeof(dst)` is greater than the + /// amount of data remaining. + /// + template<typename T> + SlangResult read(T& dst) { - return type == RiffFourCC::kRiff || type == RiffFourCC::kList; + return read(&dst, sizeof(dst)); } - /// Dump the chunk structure - static void dump(Chunk* chunk, WriterHelper writer); + /// Skip over the given number of bytes. + /// + /// Fails if `size` is greater than the + /// amount of data remaining. + /// + SlangResult skip(Size size) + { + if (size > getRemainingSize()) + { + return SLANG_FAIL; + } + _cursor += size; + _remainingSize -= size; + return SLANG_OK; + } - /// Get the size taking into account padding - static size_t getPadSize(size_t in) { return (in + kRiffPadMask) & ~size_t(kRiffPadMask); } + /// Get a pointer to the data that remains to be read. + Byte const* getRemainingData() const { return _cursor; } - /// Write a chunk list and contents to a stream - static SlangResult write(ListChunk* listChunk, bool isRoot, Stream* stream); - /// Write a container to the stream - static SlangResult write(RiffContainer* container, Stream* stream); + /// Get the size of the data that remains to be read. + Size getRemainingSize() const { return _remainingSize; } - /// Read the stream into the container - static SlangResult read(Stream* stream, RiffContainer& outContainer); +private: + Byte const* _cursor = nullptr; + Size _remainingSize = 0; }; } // namespace Slang diff --git a/source/core/slang-semantic-version.cpp b/source/core/slang-semantic-version.cpp index b05eb503f..a7697ad8f 100644 --- a/source/core/slang-semantic-version.cpp +++ b/source/core/slang-semantic-version.cpp @@ -93,6 +93,28 @@ void SemanticVersion::append(StringBuilder& buf) const return bestVersion; } +bool SemanticVersion::isBackwardsCompatibleWith(const ThisType& otherVersion) const +{ + // Compatibility is not guaranteed across major revisions. + // + if (m_major != otherVersion.m_major) + return false; + + // Within a given major revision, a version with a higher + // minor revision is backwards-compatible with one that + // has a lower minor revision, but not vice-versa. + // + if (m_minor < otherVersion.m_minor) + return false; + + // If the major and minor revisions pass our check, then + // we consider it a match. Note that this intentionally + // doesn't check the path version at all. + // + return true; +} + + // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! MatchSemanticVersion !!!!!!!!!!!!!!!!!!!!!!!!!!!!! /* static */ SemanticVersion MatchSemanticVersion::findAnyBest( diff --git a/source/core/slang-semantic-version.h b/source/core/slang-semantic-version.h index 0bb467ab7..096606e08 100644 --- a/source/core/slang-semantic-version.h +++ b/source/core/slang-semantic-version.h @@ -12,7 +12,7 @@ struct SemanticVersion { typedef SemanticVersion ThisType; - typedef uint64_t IntegerType; + typedef UInt64 RawValue; SemanticVersion() : m_major(0), m_minor(0), m_patch(0) @@ -34,14 +34,22 @@ struct SemanticVersion /// All zeros means nothing is set bool isSet() const { return m_major || m_minor || m_patch; } - IntegerType toInteger() const + RawValue getRawValue() const { - return (IntegerType(m_major) << 48) | (IntegerType(m_minor) << 32) | m_patch; + return (RawValue(m_major) << 48) | (RawValue(m_minor) << 32) | m_patch; } - void setFromInteger(IntegerType v) + void setRawValue(RawValue v) { set(int(v >> 48), int((v >> 32) & 0xffff), int(v & 0xffffffff)); } + + static SemanticVersion fromRaw(RawValue rawValue) + { + SemanticVersion result; + result.setRawValue(rawValue); + return result; + } + void set(int major, int minor, int patch = 0) { SLANG_ASSERT(major >= 0 && minor >= 0 && patch >= 0); @@ -52,7 +60,7 @@ struct SemanticVersion } /// Get hash value - HashCode getHashCode() const { return Slang::getHashCode(toInteger()); } + HashCode getHashCode() const { return Slang::getHashCode(getRawValue()); } static SlangResult parse(const UnownedStringSlice& value, SemanticVersion& outVersion); static SlangResult parse( @@ -63,16 +71,29 @@ struct SemanticVersion static ThisType getEarliest(const ThisType* versions, Count count); static ThisType getLatest(const ThisType* versions, Count count); + /// Determine if this version is backwards-compatible with `otherVersion`. + /// + /// Examples of when to apply this: + /// + /// * Code for this version is asked to load data produced by `otherVersion`. + /// + /// * Code for this version is asked if it provides the API of `otherVersion`. + /// + /// This uses the rules of semantic versioning, so it should only + /// be applied to versions that obey those rules. + /// + bool isBackwardsCompatibleWith(const ThisType& otherVersion) const; + void append(StringBuilder& buf) const; - bool operator>(const ThisType& rhs) const { return toInteger() > rhs.toInteger(); } - bool operator>=(const ThisType& rhs) const { return toInteger() >= rhs.toInteger(); } + bool operator>(const ThisType& rhs) const { return getRawValue() > rhs.getRawValue(); } + bool operator>=(const ThisType& rhs) const { return getRawValue() >= rhs.getRawValue(); } - bool operator<(const ThisType& rhs) const { return toInteger() < rhs.toInteger(); } - bool operator<=(const ThisType& rhs) const { return toInteger() <= rhs.toInteger(); } + bool operator<(const ThisType& rhs) const { return getRawValue() < rhs.getRawValue(); } + bool operator<=(const ThisType& rhs) const { return getRawValue() <= rhs.getRawValue(); } - bool operator==(const ThisType& rhs) const { return toInteger() == rhs.toInteger(); } - bool operator!=(const ThisType& rhs) const { return toInteger() != rhs.toInteger(); } + bool operator==(const ThisType& rhs) const { return getRawValue() == rhs.getRawValue(); } + bool operator!=(const ThisType& rhs) const { return getRawValue() != rhs.getRawValue(); } uint16_t m_major; uint16_t m_minor; diff --git a/source/core/slang-stream.h b/source/core/slang-stream.h index 78664c17a..0a5adbde8 100644 --- a/source/core/slang-stream.h +++ b/source/core/slang-stream.h @@ -303,6 +303,12 @@ struct StreamUtil /// Appends all bytes that can be read from stream into bytes static SlangResult readAll(Stream* stream, size_t readSize, List<Byte>& ioBytes); + /// Appends all bytes that can be read from stream into bytes + static SlangResult readAll(Stream* stream, List<Byte>& ioBytes) + { + return readAll(stream, 0, ioBytes); + } + /// Read as much as can be read until a 0 sized read, or an error and append onto ioBytes /// Read size controls the size of each buffer read. Passing 0, will use the default read size. static SlangResult read(Stream* stream, size_t readSize, List<Byte>& ioBytes); diff --git a/source/core/slang-text-io.cpp b/source/core/slang-text-io.cpp index 9cc3896fd..d0f7ca551 100644 --- a/source/core/slang-text-io.cpp +++ b/source/core/slang-text-io.cpp @@ -125,7 +125,7 @@ SlangResult StreamReader::readBuffer() return SLANG_OK; } -char StreamReader::readBufferChar() +Byte StreamReader::readBufferByte() { if (m_index < m_buffer.getCount()) { diff --git a/source/core/slang-text-io.h b/source/core/slang-text-io.h index 7c13ebec8..727634c1b 100644 --- a/source/core/slang-text-io.h +++ b/source/core/slang-text-io.h @@ -74,23 +74,23 @@ protected: { case CharEncodeType::UTF8: { - codePoint = getUnicodePointFromUTF8([&]() -> Byte { return readBufferChar(); }); + codePoint = getUnicodePointFromUTF8([&]() -> Byte { return readBufferByte(); }); break; } case CharEncodeType::UTF16: { - codePoint = getUnicodePointFromUTF16([&]() -> Byte { return readBufferChar(); }); + codePoint = getUnicodePointFromUTF16([&]() -> Byte { return readBufferByte(); }); break; } case CharEncodeType::UTF16Reversed: { codePoint = - getUnicodePointFromUTF16Reversed([&]() -> Byte { return readBufferChar(); }); + getUnicodePointFromUTF16Reversed([&]() -> Byte { return readBufferByte(); }); break; } case CharEncodeType::UTF32: { - codePoint = getUnicodePointFromUTF32([&]() -> Byte { return readBufferChar(); }); + codePoint = getUnicodePointFromUTF32([&]() -> Byte { return readBufferByte(); }); break; } } @@ -99,11 +99,11 @@ protected: } private: - char readBufferChar(); + Byte readBufferByte(); SlangResult readBuffer(); RefPtr<Stream> m_stream; - List<char> m_buffer; + List<Byte> m_buffer; CharEncodeType m_encodingType = CharEncodeType::UTF8; CharEncoding* m_encoding = nullptr; diff --git a/source/core/slang-writer.h b/source/core/slang-writer.h index f85bebc21..6ec776ddf 100644 --- a/source/core/slang-writer.h +++ b/source/core/slang-writer.h @@ -188,7 +188,7 @@ public: SLANG_OVERRIDE; /// Ctor - StringWriter(StringBuilder* builder, WriterFlags flags) + StringWriter(StringBuilder* builder, WriterFlags flags = 0) : Parent(flags), m_builder(builder) { } diff --git a/source/slang/slang-compiler.h b/source/slang/slang-compiler.h index 27c368738..0fc8e69dd 100644 --- a/source/slang/slang-compiler.h +++ b/source/slang/slang-compiler.h @@ -34,7 +34,7 @@ namespace Slang struct PathInfo; struct IncludeHandler; struct SharedSemanticsContext; -struct ModuleChunkRef; +struct ModuleChunk; class ProgramLayout; class PtrType; @@ -2363,20 +2363,23 @@ public: /// Otherwise, return null. /// RefPtr<Module> findOrLoadSerializedModuleForModuleLibrary( - ModuleChunkRef moduleChunk, + ModuleChunk const* moduleChunk, + RIFF::ListChunk const* libraryChunk, DiagnosticSink* sink); RefPtr<Module> loadSerializedModule( Name* moduleName, const PathInfo& moduleFilePathInfo, - ModuleChunkRef moduleChunk, + ModuleChunk const* moduleChunk, + RIFF::ListChunk const* containerChunk, //< The outer container, if there is one. SourceLoc const& requestingLoc, DiagnosticSink* sink); SlangResult loadSerializedModuleContents( Module* module, const PathInfo& moduleFilePathInfo, - ModuleChunkRef moduleChunk, + ModuleChunk const* moduleChunk, + RIFF::ListChunk const* containerChunk, //< The outer container, if there is one. DiagnosticSink* sink); SourceFile* loadSourceFile(String pathFrom, String path); @@ -2387,8 +2390,7 @@ public: Name* name, PathInfo const& pathInfo); - bool isBinaryModuleUpToDate(String fromPath, RiffContainer* container); - bool isBinaryModuleUpToDate(String fromPath, ModuleChunkRef moduleChunk); + bool isBinaryModuleUpToDate(String fromPath, RIFF::ListChunk const* baseChunk); RefPtr<Module> findOrImportModule( Name* name, diff --git a/source/slang/slang-emit-cuda.cpp b/source/slang/slang-emit-cuda.cpp index bb6941907..74133fcf0 100644 --- a/source/slang/slang-emit-cuda.cpp +++ b/source/slang/slang-emit-cuda.cpp @@ -947,8 +947,7 @@ void CUDASourceEmitter::handleRequiredCapabilitiesImpl(IRInst* inst) { if (auto smDecoration = as<IRRequireCUDASMVersionDecoration>(decoration)) { - SemanticVersion version; - version.setFromInteger(SemanticVersion::IntegerType(smDecoration->getCUDASMVersion())); + SemanticVersion version = smDecoration->getCUDASMVersion(); m_extensionTracker->requireSMVersion(version); } } diff --git a/source/slang/slang-emit-glsl.cpp b/source/slang/slang-emit-glsl.cpp index 5e9fa7f70..850395d97 100644 --- a/source/slang/slang-emit-glsl.cpp +++ b/source/slang/slang-emit-glsl.cpp @@ -3074,11 +3074,8 @@ void GLSLSourceEmitter::handleRequiredCapabilitiesImpl(IRInst* inst) } case kIROp_RequireSPIRVVersionDecoration: { - auto intValue = - static_cast<IRRequireSPIRVVersionDecoration*>(decoration)->getSPIRVVersion(); - SemanticVersion version; - version.setFromInteger(SemanticVersion::IntegerType(intValue)); - _requireSPIRVVersion(version); + _requireSPIRVVersion( + static_cast<IRRequireSPIRVVersionDecoration*>(decoration)->getSPIRVVersion()); break; } } diff --git a/source/slang/slang-ir-insts.h b/source/slang/slang-ir-insts.h index 4bef81f47..268929fb9 100644 --- a/source/slang/slang-ir-insts.h +++ b/source/slang/slang-ir-insts.h @@ -396,7 +396,10 @@ struct IRRequireSPIRVVersionDecoration : IRDecoration IR_LEAF_ISA(RequireGLSLVersionDecoration) IRConstant* getSPIRVVersionOperand() { return cast<IRConstant>(getOperand(0)); } - IntegerLiteralValue getSPIRVVersion() { return getSPIRVVersionOperand()->value.intVal; } + SemanticVersion getSPIRVVersion() + { + return SemanticVersion::fromRaw(getSPIRVVersionOperand()->value.intVal); + } }; struct IRRequireCapabilityAtomDecoration : IRDecoration @@ -420,7 +423,10 @@ struct IRRequireCUDASMVersionDecoration : IRDecoration IR_LEAF_ISA(RequireCUDASMVersionDecoration) IRConstant* getCUDASMVersionOperand() { return cast<IRConstant>(getOperand(0)); } - IntegerLiteralValue getCUDASMVersion() { return getCUDASMVersionOperand()->value.intVal; } + SemanticVersion getCUDASMVersion() + { + return SemanticVersion::fromRaw(getCUDASMVersionOperand()->value.intVal); + } }; struct IRRequireGLSLExtensionDecoration : IRDecoration @@ -5009,13 +5015,15 @@ public: getStringValue(prelude)); } + IRInst* getSemanticVersionValue(SemanticVersion const& value) + { + SemanticVersion::RawValue rawValue = value.getRawValue(); + return getIntValue(getBasicType(BaseType::UInt64), rawValue); + } + void addRequireSPIRVVersionDecoration(IRInst* value, const SemanticVersion& version) { - SemanticVersion::IntegerType intValue = version.toInteger(); - addDecoration( - value, - kIROp_RequireSPIRVVersionDecoration, - getIntValue(getBasicType(BaseType::UInt64), intValue)); + addDecoration(value, kIROp_RequireSPIRVVersionDecoration, getSemanticVersionValue(version)); } void addSPIRVNonUniformResourceDecoration(IRInst* value) @@ -5025,11 +5033,10 @@ public: void addRequireCUDASMVersionDecoration(IRInst* value, const SemanticVersion& version) { - SemanticVersion::IntegerType intValue = version.toInteger(); addDecoration( value, kIROp_RequireCUDASMVersionDecoration, - getIntValue(getBasicType(BaseType::UInt64), intValue)); + getSemanticVersionValue(version)); } void addRequireCapabilityAtomDecoration(IRInst* value, CapabilityName atom) diff --git a/source/slang/slang-module-library.cpp b/source/slang/slang-module-library.cpp index c03d5c2dd..c3f4a1349 100644 --- a/source/slang/slang-module-library.cpp +++ b/source/slang/slang-module-library.cpp @@ -39,8 +39,8 @@ void* ModuleLibrary::castAs(const Guid& guid) } SlangResult loadModuleLibrary( - const Byte* inBytes, - size_t bytesCount, + const Byte* inData, + size_t dataSize, String path, EndToEndCompileRequest* req, ComPtr<IModuleLibrary>& outLibrary) @@ -51,32 +51,38 @@ SlangResult loadModuleLibrary( ComPtr<IModuleLibrary> scopeLibrary(library); // Load up the module - MemoryStreamBase memoryStream(FileAccess::Read, inBytes, bytesCount); - - RiffContainer riffContainer; - SLANG_RETURN_ON_FAIL(RiffUtil::read(&memoryStream, riffContainer)); + auto rootChunk = RIFF::RootChunk::getFromBlob(inData, dataSize); + if (!rootChunk) + { + return SLANG_FAIL; + } auto linkage = req->getLinkage(); auto sink = req->getSink(); auto namePool = req->getNamePool(); - auto container = ContainerChunkRef::find(&riffContainer); + auto container = ContainerChunk::find(rootChunk); + if (!container) + { + return SLANG_FAIL; + } - for (auto moduleChunk : container.getModules()) + for (auto moduleChunk : container->getModules()) { - auto loadedModule = linkage->findOrLoadSerializedModuleForModuleLibrary(moduleChunk, sink); + auto loadedModule = + linkage->findOrLoadSerializedModuleForModuleLibrary(moduleChunk, container, sink); if (!loadedModule) return SLANG_FAIL; library->m_modules.add(loadedModule); } - for (auto entryPointChunk : container.getEntryPoints()) + for (auto entryPointChunk : container->getEntryPoints()) { FrontEndCompileRequest::ExtraEntryPointInfo entryPointInfo; - entryPointInfo.mangledName = entryPointChunk.getMangledName(); - entryPointInfo.name = namePool->getName(entryPointChunk.getName()); - entryPointInfo.profile = entryPointChunk.getProfile(); + entryPointInfo.mangledName = entryPointChunk->getMangledName(); + entryPointInfo.name = namePool->getName(entryPointChunk->getName()); + entryPointInfo.profile = entryPointChunk->getProfile(); library->m_entryPoints.add(entryPointInfo); } diff --git a/source/slang/slang-repro.cpp b/source/slang/slang-repro.cpp index 9859ead52..0c4d7337d 100644 --- a/source/slang/slang-repro.cpp +++ b/source/slang/slang-repro.cpp @@ -14,10 +14,8 @@ namespace Slang { -/* static */ const RiffSemanticVersion ReproUtil::g_semanticVersion = RiffSemanticVersion::make( - ReproUtil::kMajorVersion, - ReproUtil::kMinorVersion, - ReproUtil::kPatchVersion); +/* static */ const SemanticVersion ReproUtil::g_semanticVersion = + SemanticVersion(ReproUtil::kMajorVersion, ReproUtil::kMinorVersion, ReproUtil::kPatchVersion); // We can't just use sizeof for the sizes of these types, because the hash will be dependent on the // ptr size, which isn't an issue for serialization (we turn all pointers into Offset32Ptr -> @@ -1173,17 +1171,20 @@ struct LoadContext Offset32Ptr<RequestState> requestState; SLANG_RETURN_ON_FAIL(store(request, container, requestState)); + RIFF::Builder riff; + RIFF::BuildCursor cursor(riff); + + SLANG_SCOPED_RIFF_BUILDER_LIST_CHUNK(cursor, kSlangStateFileFourCC); + SLANG_SCOPED_RIFF_BUILDER_DATA_CHUNK(cursor, kSlangStateDataFourCC); + Header header; - header.m_chunk.type = kSlangStateFourCC; header.m_semanticVersion = g_semanticVersion; header.m_typeHash = _getTypeHash(); - return RiffUtil::writeData( - &header.m_chunk, - sizeof(header), - container.getData(), - container.getDataCount(), - stream); + cursor.addData(header); + cursor.addUnownedData(container.getData(), container.getDataCount()); + + return riff.writeTo(stream); } /* static */ SlangResult ReproUtil::saveState( @@ -1210,29 +1211,63 @@ struct LoadContext /* static */ SlangResult ReproUtil::loadState( Stream* stream, DiagnosticSink* sink, - List<uint8_t>& buffer) + List<uint8_t>& outBuffer) { - Header header; - + List<Byte> streamData; { - Result res = RiffUtil::readData(stream, &header.m_chunk, sizeof(header), buffer); - if (SLANG_FAILED(res)) + auto result = StreamUtil::readAll(stream, streamData); + if (SLANG_FAILED(result)) { sink->diagnose(SourceLoc(), Diagnostics::unableToReadRiff); - return res; + return result; } } - if (header.m_chunk.type != kSlangStateFourCC) + + return loadState(streamData.getBuffer(), streamData.getCount(), sink, outBuffer); +} + +/* static */ SlangResult ReproUtil::loadState( + const uint8_t* data, + size_t dataSize, + DiagnosticSink* sink, + List<uint8_t>& outBuffer) +{ + auto rootChunk = RIFF::RootChunk::getFromBlob(data, dataSize); + if (!rootChunk) + { + sink->diagnose(SourceLoc(), Diagnostics::unableToReadRiff); + return SLANG_FAIL; + } + if (rootChunk->getType() != kSlangStateFileFourCC) + { + sink->diagnose(SourceLoc(), Diagnostics::expectingSlangRiffContainer); + return SLANG_FAIL; + } + + auto dataChunk = rootChunk->findDataChunk(kSlangStateDataFourCC); + if (!dataChunk) { sink->diagnose(SourceLoc(), Diagnostics::expectingSlangRiffContainer); return SLANG_FAIL; } - if (!RiffSemanticVersion::areCompatible(g_semanticVersion, header.m_semanticVersion)) + MemoryReader reader(dataChunk->getPayload(), dataChunk->getPayloadSize()); + + Header header; + { + auto result = reader.read(header); + if (SLANG_FAILED(result)) + { + sink->diagnose(SourceLoc(), Diagnostics::expectingSlangRiffContainer); + return result; + } + } + + if (!g_semanticVersion.isBackwardsCompatibleWith(header.m_semanticVersion)) { StringBuilder headerBuf, currentBuf; - header.m_semanticVersion.asSemanticVersion().append(headerBuf); - g_semanticVersion.asSemanticVersion().append(currentBuf); + header.m_semanticVersion.append(headerBuf); + g_semanticVersion.append(currentBuf); sink->diagnose( SourceLoc(), @@ -1248,17 +1283,19 @@ struct LoadContext return SLANG_FAIL; } - return SLANG_OK; -} + auto remainingSize = reader.getRemainingSize(); + outBuffer.setCount(remainingSize); -/* static */ SlangResult ReproUtil::loadState( - const uint8_t* data, - size_t size, - DiagnosticSink* sink, - List<uint8_t>& outBuffer) -{ - MemoryStreamBase stream(FileAccess::Read, data, size); - return loadState(&stream, sink, outBuffer); + { + auto result = reader.read(&outBuffer[0], remainingSize); + if (SLANG_FAILED(result)) + { + sink->diagnose(SourceLoc(), Diagnostics::expectingSlangRiffContainer); + return result; + } + } + + return SLANG_OK; } /* static */ ReproUtil::RequestState* ReproUtil::getRequest(const List<uint8_t>& buffer) diff --git a/source/slang/slang-repro.h b/source/slang/slang-repro.h index 0641c4ecc..b7a9a739b 100644 --- a/source/slang/slang-repro.h +++ b/source/slang/slang-repro.h @@ -29,14 +29,17 @@ struct ReproUtil kPatchVersion = 0, }; - static const uint32_t kSlangStateFourCC = - SLANG_FOUR_CC('S', 'L', 'S', 'T'); ///< Holds all the slang specific chunks - static const RiffSemanticVersion g_semanticVersion; + static const FourCC::RawValue kSlangStateFileFourCC = + SLANG_FOUR_CC('S', 'L', 'S', 'T'); ///< Root chunk for repro state file. + + static const FourCC::RawValue kSlangStateDataFourCC = + SLANG_FOUR_CC('d', 'a', 't', 'a'); ///< Holds the actual binary data. + + static const SemanticVersion g_semanticVersion; struct Header { - RiffHeader m_chunk; ///< The chunk - RiffSemanticVersion m_semanticVersion; ///< The semantic version + SemanticVersion m_semanticVersion; ///< The semantic version StableHashCode32 m_typeHash; ///< A hash based on the binary representation. If doesn't match then not ///< binary compatible (extra check over semantic versioning) diff --git a/source/slang/slang-serialize-ast.cpp b/source/slang/slang-serialize-ast.cpp index 84278650f..1ef532ad1 100644 --- a/source/slang/slang-serialize-ast.cpp +++ b/source/slang/slang-serialize-ast.cpp @@ -59,9 +59,9 @@ public: { auto containerChunk = encoder->getRIFFChunk(); - RiffContainer::Chunk* declChunk = nullptr; - RiffContainer::Chunk* importedDeclChunk = nullptr; - RiffContainer::Chunk* valChunk = nullptr; + RIFF::ChunkBuilder* declChunk = nullptr; + RIFF::ChunkBuilder* importedDeclChunk = nullptr; + RIFF::ChunkBuilder* valChunk = nullptr; { Encoder::WithArray withList(encoder); declChunk = encoder->getRIFFChunk(); @@ -102,7 +102,6 @@ public: } } while (!done); - RiffContainer::calcAndSetSize(containerChunk); encoder->setRIFFChunk(containerChunk); } @@ -345,7 +344,7 @@ public: void encodeValue(NameLoc const& value) { encode(value.name); } - void encodeValue(SemanticVersion value) { encoder->encode(value.toInteger()); } + void encodeValue(SemanticVersion value) { encoder->encode(value.getRawValue()); } void encodeValue(CapabilitySet const& value) { @@ -668,13 +667,13 @@ public: Linkage* linkage, ASTBuilder* astBuilder, DiagnosticSink* sink, - RiffContainer::Chunk* rootChunk, + RIFF::Chunk const* baseChunk, SerialSourceLocReader* sourceLocReader, SourceLoc requestingSourceLoc) : _linkage(linkage) , _astBuilder(astBuilder) , _sink(sink) - , _rootChunk(static_cast<RiffContainer::ListChunk*>(rootChunk)) + , _baseChunk(as<RIFF::ListChunk>(baseChunk)) , _sourceLocReader(sourceLocReader) , _requestingSourceLoc(requestingSourceLoc) { @@ -687,7 +686,7 @@ public: SlangResult decodeAll() { - auto cursor = _rootChunk->getFirstContainedChunk(); + auto cursor = _baseChunk->getChildren().begin(); // There are a few different top-level chunks that // hold different arrays that we need in order @@ -705,23 +704,23 @@ public: // the `ModuleDecl` itself, which should be the // first entry in the list. // - auto declChunk = cursor; - cursor = cursor->m_next; + auto declChunk = *cursor; + ++cursor; // Next there is a list of all the declarations // referenced inside of the module that need to // be imported in from outside. // - auto importedDeclChunk = cursor; - cursor = cursor->m_next; + auto importedDeclChunk = *cursor; + ++cursor; // Then there are all the `Val`-derived nodes that // are needed by the module, which will need to be // deduplicated so that they are unique within the // current compilation context. // - auto valChunk = cursor; - cursor = cursor->m_next; + auto valChunk = *cursor; + ++cursor; // The process of decoding the module is then spread // over a number of steps. @@ -792,7 +791,7 @@ private: }; ASTBuilder* _astBuilder = nullptr; - RiffContainer::ListChunk* _rootChunk = nullptr; + RIFF::ListChunk const* _baseChunk = nullptr; List<Decl*> _decls; List<Decl*> _importedDecls; @@ -801,7 +800,7 @@ private: typedef Int ValID; Val* getValByID(ValID id) { return _vals[id]; } - SlangResult decodeImportedDecls(RiffContainer::Chunk* importedDeclChunk) + SlangResult decodeImportedDecls(RIFF::Chunk const* importedDeclChunk) { Decoder decoder(importedDeclChunk); @@ -849,7 +848,7 @@ private: return module->getModuleDecl(); } - SlangResult decodeVals(RiffContainer::Chunk* valChunk) + SlangResult decodeVals(RIFF::Chunk const* valChunk) { Decoder decoder(valChunk); @@ -862,7 +861,7 @@ private: return SLANG_OK; } - SlangResult createEmptyShells(RiffContainer::Chunk* declChunk) + SlangResult createEmptyShells(RIFF::Chunk const* declChunk) { Decoder decoder(declChunk); @@ -925,7 +924,7 @@ private: return SyntaxClass<NodeBase>(nodeType).createInstance(_astBuilder); } - SlangResult fillEmptyShells(RiffContainer::Chunk* declChunk) + SlangResult fillEmptyShells(RIFF::Chunk const* declChunk) { Index declIndex = 0; @@ -1190,8 +1189,8 @@ private: void decodeValue(SemanticVersion& value, Decoder& decoder) { - SemanticVersion::IntegerType rawValue = decoder.decode<SemanticVersion::IntegerType>(); - value.setFromInteger(rawValue); + SemanticVersion::RawValue rawValue = decoder.decode<SemanticVersion::RawValue>(); + value.setRawValue(rawValue); } void decodeValue(CapabilitySet& value, Decoder& decoder) @@ -1541,7 +1540,7 @@ ModuleDecl* readSerializedModuleAST( Linkage* linkage, ASTBuilder* astBuilder, DiagnosticSink* sink, - RiffContainer::Chunk* chunk, + RIFF::Chunk const* chunk, SerialSourceLocReader* sourceLocReader, SourceLoc requestingSourceLoc) { diff --git a/source/slang/slang-serialize-ast.h b/source/slang/slang-serialize-ast.h index 6adeae8dd..45c799e9c 100644 --- a/source/slang/slang-serialize-ast.h +++ b/source/slang/slang-serialize-ast.h @@ -20,7 +20,7 @@ ModuleDecl* readSerializedModuleAST( Linkage* linkage, ASTBuilder* astBuilder, DiagnosticSink* sink, - RiffContainer::Chunk* chunk, + RIFF::Chunk const* chunk, SerialSourceLocReader* sourceLocReader, SourceLoc requestingSourceLoc); diff --git a/source/slang/slang-serialize-container.cpp b/source/slang/slang-serialize-container.cpp index c2253ed45..665775dc5 100644 --- a/source/slang/slang-serialize-container.cpp +++ b/source/slang/slang-serialize-container.cpp @@ -17,20 +17,34 @@ namespace Slang { struct ModuleEncodingContext { +private: + SerialContainerUtil::WriteOptions const& _options; + Stream* _stream = nullptr; + + // The string pool used across the whole of the container + StringSlicePool _containerStringPool; + RefPtr<SerialSourceLocWriter> _sourceLocWriter; + + RIFF::Builder _riff; + Encoder _encoder; + public: ModuleEncodingContext(SerialContainerUtil::WriteOptions const& options, Stream* stream) - : options(options), encoder(stream), containerStringPool(StringSlicePool::Style::Default) + : _options(options), _stream(stream), _containerStringPool(StringSlicePool::Style::Default) { if (options.optionFlags & SerialOptionFlag::SourceLocation) { - sourceLocWriter = new SerialSourceLocWriter(options.sourceManager); + _sourceLocWriter = new SerialSourceLocWriter(options.sourceManager); } + + _encoder = Encoder(_riff); } ~ModuleEncodingContext() { - encoder.setRIFFChunk(encoder.getRIFF()->getRoot()); + _encoder = Encoder(_riff.getRootChunk()); encodeFinalPieces(); + _riff.writeTo(_stream); } SlangResult encodeModuleList(FrontEndCompileRequest* frontEndReq) @@ -39,7 +53,7 @@ public: // is simply a matter of encoding the module for each // of the translation units that got compiled. // - Encoder::WithKeyValuePair withArray(&encoder, SerialBinary::kModuleListFourCc); + Encoder::WithKeyValuePair withArray(&_encoder, SerialBinary::kModuleListFourCc); for (TranslationUnitRequest* translationUnit : frontEndReq->translationUnits) { SLANG_RETURN_ON_FAIL(encode(translationUnit->module)); @@ -49,14 +63,14 @@ public: SlangResult encode(FrontEndCompileRequest* frontEndReq) { - Encoder::WithObject withObject(&encoder, SerialBinary::kContainerFourCc); + Encoder::WithObject withObject(&_encoder, SerialBinary::kContainerFourCc); SLANG_RETURN_ON_FAIL(encodeModuleList(frontEndReq)); return SLANG_OK; } SlangResult encode(EndToEndCompileRequest* request) { - Encoder::WithObject withObject(&encoder, SerialBinary::kContainerFourCc); + Encoder::WithObject withObject(&_encoder, SerialBinary::kContainerFourCc); // Encoding an end-to-end compile request starts with the same // work as for a front-end request: we encode each of @@ -85,7 +99,7 @@ public: auto sink = request->getSink(); auto program = request->getSpecializedGlobalAndEntryPointsComponentType(); { - Encoder::WithArray withArray(&encoder); // kContainerFourCc + Encoder::WithArray withArray(&_encoder); for (auto target : linkage->targets) { @@ -98,7 +112,7 @@ public: // and we need to encode information about each of them. // { - Encoder::WithArray withArray(&encoder, SerialBinary::kEntryPointListFourCc); + Encoder::WithArray withArray(&_encoder, SerialBinary::kEntryPointListFourCc); auto entryPointCount = program->getEntryPointCount(); for (Index ii = 0; ii < entryPointCount; ++ii) @@ -127,34 +141,34 @@ public: IRSerialWriter writer; SLANG_RETURN_ON_FAIL( - writer.write(irModule, sourceLocWriter, options.optionFlags, &serialData)); - SLANG_RETURN_ON_FAIL(IRSerialWriter::writeContainer(serialData, encoder.getRIFF())); + writer.write(irModule, _sourceLocWriter, _options.optionFlags, &serialData)); + SLANG_RETURN_ON_FAIL(IRSerialWriter::writeTo(serialData, _encoder)); return SLANG_OK; } - void encode(Name* name) { encoder.encode(name->text); } + void encode(Name* name) { _encoder.encode(name->text); } - void encode(String const& value) { encoder.encode(value); } + void encode(String const& value) { _encoder.encode(value); } - void encode(uint32_t value) { encoder.encode(UInt(value)); } + void encode(uint32_t value) { _encoder.encode(UInt(value)); } - void encodeData(void const* data, size_t size) { encoder.encodeData(data, size); } + void encodeData(void const* data, size_t size) { _encoder.encodeData(data, size); } SlangResult encode(EntryPoint* entryPoint, String const& entryPointMangledName) { - Encoder::WithObject withObject(&encoder, SerialBinary::kEntryPointFourCc); + Encoder::WithObject withObject(&_encoder, SerialBinary::kEntryPointFourCc); { - Encoder::WithObject withProperty(&encoder, SerialBinary::kNameFourCC); + Encoder::WithObject withProperty(&_encoder, SerialBinary::kNameFourCC); encode(entryPoint->getName()); } { - Encoder::WithObject withProperty(&encoder, SerialBinary::kProfileFourCC); + Encoder::WithObject withProperty(&_encoder, SerialBinary::kProfileFourCC); encode(entryPoint->getProfile().raw); } { - Encoder::WithObject withProperty(&encoder, SerialBinary::kMangledNameFourCC); + Encoder::WithObject withProperty(&_encoder, SerialBinary::kMangledNameFourCC); encode(entryPointMangledName); } @@ -164,10 +178,10 @@ public: SlangResult encode(Module* module) { - if (!(options.optionFlags & (SerialOptionFlag::IRModule | SerialOptionFlag::ASTModule))) + if (!(_options.optionFlags & (SerialOptionFlag::IRModule | SerialOptionFlag::ASTModule))) return SLANG_OK; - Encoder::WithObject withModule(&encoder, SerialBinary::kModuleFourCC); + Encoder::WithObject withModule(&_encoder, SerialBinary::kModuleFourCC); // The first piece that we write for a module is its header. // The header is intended to provide information that can be @@ -180,15 +194,15 @@ public: // sense to serialize it separately from all the rest. // { - Encoder::WithObject withProperty(&encoder, SerialBinary::kNameFourCC); - encoder.encodeString(module->getNameObj()->text); + Encoder::WithObject withProperty(&_encoder, SerialBinary::kNameFourCC); + _encoder.encodeString(module->getNameObj()->text); } // The header includes a digest of all the compile options and // the files that the compiled result depended on. // auto digest = module->computeDigest(); - encoder.encodeData(PropertyKeys<Module>::Digest, digest.data, sizeof(digest.data)); + _encoder.encodeData(PropertyKeys<Module>::Digest, digest.data, sizeof(digest.data)); // The header includes an array of the paths of all of the // files that the compiled result depended on. @@ -199,30 +213,28 @@ public: // If serialization of Slang IR modules is enabled, and there // is IR available for this module, then we we encode it. // - if ((options.optionFlags & SerialOptionFlag::IRModule)) + if ((_options.optionFlags & SerialOptionFlag::IRModule)) { if (auto irModule = module->getIRModule()) { - Encoder::WithKeyValuePair withKey(&encoder, PropertyKeys<Module>::IRModule); - IRSerialData serialData; IRSerialWriter writer; SLANG_RETURN_ON_FAIL( - writer.write(irModule, sourceLocWriter, options.optionFlags, &serialData)); - SLANG_RETURN_ON_FAIL(IRSerialWriter::writeContainer(serialData, encoder.getRIFF())); + writer.write(irModule, _sourceLocWriter, _options.optionFlags, &serialData)); + SLANG_RETURN_ON_FAIL(IRSerialWriter::writeTo(serialData, _encoder)); } } // If serialization of AST information is enabled, and we have AST // information available, then we serialize it here. // - if (options.optionFlags & SerialOptionFlag::ASTModule) + if (_options.optionFlags & SerialOptionFlag::ASTModule) { if (auto moduleDecl = module->getModuleDecl()) { - Encoder::WithKeyValuePair withKey(&encoder, PropertyKeys<Module>::ASTModule); + Encoder::WithKeyValuePair withKey(&_encoder, PropertyKeys<Module>::ASTModule); - writeSerializedModuleAST(&encoder, moduleDecl, sourceLocWriter); + writeSerializedModuleAST(&_encoder, moduleDecl, _sourceLocWriter); } } @@ -231,7 +243,7 @@ public: SlangResult encodeModuleDependencyPaths(Module* module) { - Encoder::WithObject withProperty(&encoder, PropertyKeys<Module>::FileDependencies); + Encoder::WithObject withProperty(&_encoder, PropertyKeys<Module>::FileDependencies); // TODO(tfoley): This is some of the most complicated logic // in the encoding system, because it tries to translate @@ -305,7 +317,7 @@ public: } Path::getCanonical(linkageRoot, linkageRoot); - Encoder::WithArray withArray(&encoder); + Encoder::WithArray withArray(&_encoder); for (auto file : fileDependencies) { if (file->getPathInfo().hasFoundPath()) @@ -322,26 +334,26 @@ public: auto relativeModulePath = Path::getRelativePath(linkageRoot, canonicalModulePath); - encoder.encodeString(relativeModulePath); + _encoder.encodeString(relativeModulePath); } else { // For all other dependnet files, store them as relative paths with respect // to the module's path. canonicalFilePath = Path::getRelativePath(moduleDir, canonicalFilePath); - encoder.encodeString(canonicalFilePath); + _encoder.encodeString(canonicalFilePath); } } else { // If the module is coming from string instead of an actual file, store it as // is. - encoder.encodeString(canonicalModulePath); + _encoder.encodeString(canonicalModulePath); } } else { - encoder.encodeString(file->getPathInfo().getMostUniqueIdentity()); + _encoder.encodeString(file->getPathInfo().getMostUniqueIdentity()); } } @@ -351,38 +363,28 @@ public: SlangResult encodeFinalPieces() { // We can now output the debug information. This is for all IR and AST - if (sourceLocWriter) + if (_sourceLocWriter) { // Write out the debug info SerialSourceLocData debugData; - sourceLocWriter->write(&debugData); + _sourceLocWriter->write(&debugData); - debugData.writeContainer(encoder.getRIFF()); + debugData.writeTo(_encoder); } // Write the container string table - if (containerStringPool.getAdded().getCount() > 0) + if (_containerStringPool.getAdded().getCount() > 0) { - Encoder::WithKeyValuePair withKey(&encoder, SerialBinary::kStringTableFourCc); + Encoder::WithKeyValuePair withKey(&_encoder, SerialBinary::kStringTableFourCc); List<char> encodedTable; - SerialStringTableUtil::encodeStringTable(containerStringPool, encodedTable); + SerialStringTableUtil::encodeStringTable(_containerStringPool, encodedTable); - encoder.encodeData(encodedTable.getBuffer(), encodedTable.getCount()); + _encoder.encodeData(encodedTable.getBuffer(), encodedTable.getCount()); } return SLANG_OK; } - - -private: - SerialContainerUtil::WriteOptions const& options; - RefPtr<SerialSourceLocWriter> sourceLocWriter; - - // The string pool used across the whole of the container - StringSlicePool containerStringPool; - - Encoder encoder; }; // @@ -421,123 +423,119 @@ private: return SLANG_OK; } -String StringChunkRef::getValue() +String StringChunk::getValue() const { - return Decoder(ptr()).decodeString(); + return Decoder(this).decodeString(); } -ChunkRefList<StringChunkRef> ModuleChunkRef::getFileDependencies() +RIFF::ChunkList<StringChunk> ModuleChunk::getFileDependencies() const { - Decoder decoder(ptr()); + Decoder decoder(this); Decoder::WithProperty withProperty(decoder, PropertyKeys<Module>::FileDependencies); - return ChunkRefList<StringChunkRef>(as<RiffContainer::ListChunk>(decoder.getCursor())); + return as<RIFF::ListChunk>(decoder.getCurrentChunk())->getChildren().cast<StringChunk>(); } -ModuleChunkRef ModuleChunkRef::find(RiffContainer* container) +ModuleChunk const* ModuleChunk::find(RIFF::ListChunk const* baseChunk) { - auto found = container->getRoot()->findListRec(SerialBinary::kModuleFourCC); - return ModuleChunkRef(found); + auto found = baseChunk->findListChunkRec(SerialBinary::kModuleFourCC); + return static_cast<ModuleChunk const*>(found); } -SHA1::Digest ModuleChunkRef::getDigest() +SHA1::Digest ModuleChunk::getDigest() const { - auto foundChunk = - static_cast<RiffContainer::DataChunk*>(ptr()->findContained(PropertyKeys<Module>::Digest)); + auto foundChunk = findDataChunk(PropertyKeys<Module>::Digest); if (!foundChunk) { SLANG_UNEXPECTED("module chunk had no digest"); } - if (foundChunk->calcPayloadSize() != sizeof(SHA1::Digest)) - { - SLANG_UNEXPECTED("module digest chunk had wrong size"); - } - - SHA1::Digest digest; - foundChunk->getPayload(&digest); - return digest; + return foundChunk->readPayloadAs<SHA1::Digest>(); } -String ModuleChunkRef::getName() +String ModuleChunk::getName() const { // TODO(tfoley): This kind of logic needs a way // to be greatly simplified, so that we don't // have to express such complicated logic for // simply extracting a single string property... // - Decoder decoder(ptr()); + Decoder decoder(this); Decoder::WithProperty withProperty(decoder, SerialBinary::kNameFourCC); return decoder.decodeString(); } -IRModuleChunkRef ModuleChunkRef::findIR() +IRModuleChunk const* ModuleChunk::findIR() const { - auto foundProperty = ptr()->findContainedList(PropertyKeys<Module>::IRModule); - if (!foundProperty) - return IRModuleChunkRef(nullptr); - return IRModuleChunkRef( - static_cast<RiffContainer::ListChunk*>(foundProperty->getFirstContainedChunk())); + auto foundChunk = findListChunk(IRSerialBinary::kIRModuleFourCc); + if (!foundChunk) + return nullptr; + + return static_cast<IRModuleChunk const*>(foundChunk); } -ASTModuleChunkRef ModuleChunkRef::findAST() +ASTModuleChunk const* ModuleChunk::findAST() const { - auto foundProperty = ptr()->findContainedList(PropertyKeys<Module>::ASTModule); + auto foundProperty = findListChunk(PropertyKeys<Module>::ASTModule); if (!foundProperty) - return ASTModuleChunkRef(nullptr); - return ASTModuleChunkRef( - static_cast<RiffContainer::ListChunk*>(foundProperty->getFirstContainedChunk())); + return nullptr; + + return static_cast<ASTModuleChunk const*>(foundProperty->getFirstChild().get()); } -ContainerChunkRef ContainerChunkRef::find(RiffContainer* container) +ContainerChunk const* ContainerChunk::find(RIFF::ListChunk const* baseChunk) { - auto found = container->getRoot()->findListRec(SerialBinary::kContainerFourCc); - return ContainerChunkRef(found); + auto found = baseChunk->findListChunkRec(SerialBinary::kContainerFourCc); + return static_cast<ContainerChunk const*>(found); } -ChunkRefList<ModuleChunkRef> ContainerChunkRef::getModules() +RIFF::ChunkList<ModuleChunk> ContainerChunk::getModules() const { - auto found = ptr()->findContainedList(SerialBinary::kModuleListFourCc); - return ChunkRefList<ModuleChunkRef>(found); + auto found = findListChunk(SerialBinary::kModuleListFourCc); + if (!found) + return RIFF::ChunkList<ModuleChunk>(); + return found->getChildren().cast<ModuleChunk>(); } -ChunkRefList<EntryPointChunkRef> ContainerChunkRef::getEntryPoints() +RIFF::ChunkList<EntryPointChunk> ContainerChunk::getEntryPoints() const { - auto found = ptr()->findContainedList(SerialBinary::kEntryPointListFourCc); - return ChunkRefList<EntryPointChunkRef>(found); + auto found = findListChunk(SerialBinary::kEntryPointListFourCc); + if (!found) + return RIFF::ChunkList<EntryPointChunk>(); + return found->getChildren().cast<EntryPointChunk>(); } -String EntryPointChunkRef::getMangledName() const +String EntryPointChunk::getMangledName() const { // TODO(tfoley): This kind of logic needs a way // to be greatly simplified, so that we don't // have to express such complicated logic for // simply extracting a single string property... // - Decoder decoder(ptr()); + Decoder decoder(this); Decoder::WithProperty withProperty(decoder, SerialBinary::kMangledNameFourCC); return decoder.decodeString(); } -String EntryPointChunkRef::getName() const +String EntryPointChunk::getName() const { // TODO(tfoley): This kind of logic needs a way // to be greatly simplified, so that we don't // have to express such complicated logic for // simply extracting a single string property... // - Decoder decoder(ptr()); + Decoder decoder(this); Decoder::WithProperty withProperty(decoder, SerialBinary::kNameFourCC); return decoder.decodeString(); } -Profile EntryPointChunkRef::getProfile() const +Profile EntryPointChunk::getProfile() const { // TODO(tfoley): This kind of logic needs a way // to be greatly simplified, so that we don't // have to express such complicated logic for // simply extracting a single string property... // - Decoder decoder(ptr()); + Decoder decoder(this); Decoder::WithProperty withProperty(decoder, SerialBinary::kProfileFourCC); Profile::RawVal rawVal; @@ -547,28 +545,25 @@ Profile EntryPointChunkRef::getProfile() const } -RiffContainer::ListChunk* findDebugChunk(RiffContainer::Chunk* startingChunk) +DebugChunk const* DebugChunk::find(RIFF::ListChunk const* baseChunk) { - if (!startingChunk) - return nullptr; - - RiffContainer::ListChunk* container = as<RiffContainer::ListChunk>(startingChunk); - if (!container) - container = startingChunk->m_parent; - - for (; container; container = container->m_parent) - { - if (auto debugChunk = container->findContainedList(SerialSourceLocData::kDebugFourCc)) - { - return debugChunk; - } - } + auto found = baseChunk->findListChunkRec(SerialSourceLocData::kDebugFourCc); + return static_cast<DebugChunk const*>(found); +} +DebugChunk const* DebugChunk::find( + RIFF::ListChunk const* baseChunk, + RIFF::ListChunk const* containerChunk) +{ + if (auto found = find(baseChunk)) + return found; + if (containerChunk) + return find(containerChunk); return nullptr; } SlangResult readSourceLocationsFromDebugChunk( - RiffContainer::ListChunk* debugChunk, + DebugChunk const* debugChunk, SourceManager* sourceManager, RefPtr<SerialSourceLocReader>& outReader) { @@ -585,7 +580,7 @@ SlangResult readSourceLocationsFromDebugChunk( // from the RIFF into the intermediate structure. // SerialSourceLocData intermediateData; - SLANG_RETURN_ON_FAIL(intermediateData.readContainer(debugChunk)); + SLANG_RETURN_ON_FAIL(intermediateData.readFrom(debugChunk)); // After reading the data into the intermediate representation, // we turn it into a `SerialSourceLocReader`, which vends source @@ -601,7 +596,7 @@ SlangResult readSourceLocationsFromDebugChunk( SlangResult decodeModuleIR( RefPtr<IRModule>& outIRModule, - RiffContainer::Chunk* chunk, + IRModuleChunk const* chunk, Session* session, SerialSourceLocReader* sourceLocReader) { @@ -617,11 +612,8 @@ SlangResult decodeModuleIR( // are deserializing IR nodes directly from the format written // into the RIFF. // - auto listChunk = as<RiffContainer::ListChunk>(chunk); - if (!listChunk) - return SLANG_FAIL; IRSerialData serialData; - SLANG_RETURN_ON_FAIL(IRSerialReader::readContainer(listChunk, &serialData)); + SLANG_RETURN_ON_FAIL(IRSerialReader::readFrom(chunk, &serialData)); // Next we read the actual IR representation out from the // `serialData`. This is the step that may pull source-location @@ -648,13 +640,11 @@ SlangResult decodeModuleIR( OwnedMemoryStream memoryStream(FileAccess::ReadWrite); { - RiffContainer riffContainer; + RIFF::Builder builder; + RIFF::BuildCursor cursor(builder); - // Need to put all of this in a container - RiffContainer::ScopeChunk containerScope( - &riffContainer, - RiffContainer::Chunk::Kind::List, - SerialBinary::kContainerFourCc); + // Need to put all of this in a module chunk + SLANG_SCOPED_RIFF_BUILDER_LIST_CHUNK(cursor, SerialBinary::kModuleFourCC); RefPtr<SerialSourceLocWriter> sourceLocWriter; @@ -669,7 +659,7 @@ SlangResult decodeModuleIR( SLANG_RETURN_ON_FAIL( writer.write(module, sourceLocWriter, options.optionFlags, &irData)); } - SLANG_RETURN_ON_FAIL(IRSerialWriter::writeContainer(irData, &riffContainer)); + SLANG_RETURN_ON_FAIL(IRSerialWriter::writeTo(irData, cursor)); // Write the debug info Riff container if (sourceLocWriter) @@ -677,10 +667,10 @@ SlangResult decodeModuleIR( SerialSourceLocData serialData; sourceLocWriter->write(&serialData); - SLANG_RETURN_ON_FAIL(serialData.writeContainer(&riffContainer)); + SLANG_RETURN_ON_FAIL(serialData.writeTo(cursor)); } - SLANG_RETURN_ON_FAIL(RiffUtil::write(&riffContainer, &memoryStream)); + SLANG_RETURN_ON_FAIL(builder.writeTo(&memoryStream)); } // Reset stream @@ -692,33 +682,41 @@ SlangResult decodeModuleIR( // The read ir module RefPtr<IRModule> irReadModule; { - RiffContainer riffContainer; - SLANG_RETURN_ON_FAIL(RiffUtil::read(&memoryStream, riffContainer)); + auto streamContents = memoryStream.getContents(); - RiffContainer::ListChunk* rootList = riffContainer.getRoot(); + auto rootChunk = + RIFF::RootChunk::getFromBlob(streamContents.getBuffer(), streamContents.getCount()); + if (!rootChunk) + { + return SLANG_FAIL; + } + + auto moduleChunk = ModuleChunk::find(rootChunk); + if (!moduleChunk) + { + return SLANG_FAIL; + } RefPtr<SerialSourceLocReader> sourceLocReader; // If we have debug info then find and read it if (options.optionFlags & SerialOptionFlag::SourceLocation) { - RiffContainer::ListChunk* debugList = - rootList->findContainedList(SerialSourceLocData::kDebugFourCc); - if (!debugList) + auto debugChunk = DebugChunk::find(moduleChunk); + if (!debugChunk) { return SLANG_FAIL; } SerialSourceLocData sourceLocData; - SLANG_RETURN_ON_FAIL(sourceLocData.readContainer(debugList)); + SLANG_RETURN_ON_FAIL(sourceLocData.readFrom(debugChunk)); sourceLocReader = new SerialSourceLocReader; SLANG_RETURN_ON_FAIL(sourceLocReader->read(&sourceLocData, &workSourceManager)); } { - RiffContainer::ListChunk* irList = - rootList->findContainedList(IRSerialBinary::kIRModuleFourCc); - if (!irList) + auto irChunk = moduleChunk->findIR(); + if (!irChunk) { return SLANG_FAIL; } @@ -726,7 +724,7 @@ SlangResult decodeModuleIR( { IRSerialData irReadData; IRSerialReader reader; - SLANG_RETURN_ON_FAIL(reader.readContainer(irList, &irReadData)); + SLANG_RETURN_ON_FAIL(reader.readFrom(irChunk, &irReadData)); // Check the stream read data is the same if (irData != irReadData) diff --git a/source/slang/slang-serialize-container.h b/source/slang/slang-serialize-container.h index 4c1053a6d..81d03251f 100644 --- a/source/slang/slang-serialize-container.h +++ b/source/slang/slang-serialize-container.h @@ -56,189 +56,76 @@ struct SerialContainerUtil static SlangResult write(Module* module, const WriteOptions& options, Stream* stream); }; - -struct ChunkRef +struct StringChunk : RIFF::DataChunk { public: - ChunkRef(RiffContainer::Chunk* chunk) - : _chunk(chunk) - { - } - - RiffContainer::Chunk* ptr() const { return _chunk; } - -protected: - RiffContainer::Chunk* _chunk = nullptr; + String getValue() const; }; -struct DataChunkRef : ChunkRef -{ -public: - DataChunkRef(RiffContainer::DataChunk* chunk) - : ChunkRef(chunk) - { - } - - RiffContainer::DataChunk* ptr() const { return static_cast<RiffContainer::DataChunk*>(_chunk); } +struct IRModuleChunk; - operator RiffContainer::DataChunk*() const { return ptr(); } -}; - - -template<typename T> -struct ChunkRefList +struct ASTModuleChunk : RIFF::ListChunk { -public: - struct Iterator - { - public: - Iterator(RiffContainer::Chunk* chunk) - : _chunk(chunk) - { - } - - bool operator!=(Iterator const& other) const { return _chunk != other._chunk; } - - void operator++() { _chunk = _chunk->m_next; } - - T operator*() - { - ChunkRef ref(_chunk); - return *(T*)&ref; - } - - private: - RiffContainer::Chunk* _chunk = nullptr; - }; - - Iterator begin() const { return _list ? _list->getFirstContainedChunk() : nullptr; } - Iterator end() const { return Iterator(nullptr); } - - Count getCount() - { - Count count = 0; - for (auto i : *this) - count++; - return count; - } - - T getFirst() { return *begin(); } - - ChunkRefList() {} - - ChunkRefList(RiffContainer::ListChunk* list) - : _list(list) - { - } - - operator RiffContainer::ListChunk*() const { return _list; } - -private: - RiffContainer::ListChunk* _list = nullptr; }; -struct ListChunkRef : ChunkRef +struct ModuleChunk : RIFF::ListChunk { public: - ListChunkRef(RiffContainer::Chunk* chunk) - : ChunkRef(chunk) - { - } - - RiffContainer::ListChunk* ptr() const { return static_cast<RiffContainer::ListChunk*>(_chunk); } + static ModuleChunk const* find(RIFF::ListChunk const* baseChunk); - operator RiffContainer::ListChunk*() const { return ptr(); } -}; - - -struct StringChunkRef : DataChunkRef -{ -public: - String getValue(); -}; - -struct IRModuleChunkRef : ListChunkRef -{ -public: - explicit IRModuleChunkRef(RiffContainer::ListChunk* chunk) - : ListChunkRef(chunk) - { - } -}; - -struct ASTModuleChunkRef : ListChunkRef -{ -public: - explicit ASTModuleChunkRef(RiffContainer::ListChunk* chunk) - : ListChunkRef(chunk) - { - } -}; - -struct ModuleChunkRef : ListChunkRef -{ -public: - static ModuleChunkRef find(RiffContainer* container); - - String getName(); - - IRModuleChunkRef findIR(); - ASTModuleChunkRef findAST(); + String getName() const; - SHA1::Digest getDigest(); + IRModuleChunk const* findIR() const; + ASTModuleChunk const* findAST() const; - ChunkRefList<StringChunkRef> getFileDependencies(); + SHA1::Digest getDigest() const; -protected: - ModuleChunkRef(RiffContainer::Chunk* chunk) - : ListChunkRef(chunk) - { - } + RIFF::ChunkList<StringChunk> getFileDependencies() const; }; -struct EntryPointChunkRef : ListChunkRef +struct EntryPointChunk : RIFF::ListChunk { public: String getMangledName() const; String getName() const; Profile getProfile() const; - -protected: - EntryPointChunkRef(RiffContainer::Chunk* chunk) - : ListChunkRef(chunk) - { - } }; -struct ContainerChunkRef : ListChunkRef +struct ContainerChunk : RIFF::ListChunk { public: - static ContainerChunkRef find(RiffContainer* container); + static ContainerChunk const* find(RIFF::ListChunk const* baseChunk); - ChunkRefList<ModuleChunkRef> getModules(); + RIFF::ChunkList<ModuleChunk> getModules() const; - ChunkRefList<EntryPointChunkRef> getEntryPoints(); - -protected: - ContainerChunkRef(RiffContainer::Chunk* chunk) - : ListChunkRef(chunk) - { - } + RIFF::ChunkList<EntryPointChunk> getEntryPoints() const; }; -/// Attempt to find a debug-info chunk relative to -/// the given `startingChunk`. -/// -RiffContainer::ListChunk* findDebugChunk(RiffContainer::Chunk* startingChunk); +struct DebugChunk : RIFF::ListChunk +{ +public: + /// Search for a debug information chunk. + static DebugChunk const* find(RIFF::ListChunk const* baseChunk); + + /// Search for a debug information chunk. + /// + /// The search will initially look in `baseChunk`, but + /// if that fails, it will fall back to the `containerChunk` + /// if that is non-null. + /// + static DebugChunk const* find( + RIFF::ListChunk const* baseChunk, + RIFF::ListChunk const* containerChunk); +}; SlangResult readSourceLocationsFromDebugChunk( - RiffContainer::ListChunk* debugChunk, + DebugChunk const* debugChunk, SourceManager* sourceManager, RefPtr<SerialSourceLocReader>& outReader); SlangResult decodeModuleIR( RefPtr<IRModule>& outIRModule, - RiffContainer::Chunk* chunk, + IRModuleChunk const* chunk, Session* session, SerialSourceLocReader* sourceLocReader); diff --git a/source/slang/slang-serialize-ir-types.h b/source/slang/slang-serialize-ir-types.h index b3daf1772..d4c9dd9d7 100644 --- a/source/slang/slang-serialize-ir-types.h +++ b/source/slang/slang-serialize-ir-types.h @@ -20,20 +20,20 @@ class Name; struct IRSerialBinary { /// IR module list - static const FourCC kIRModuleFourCc = SLANG_FOUR_CC('S', 'i', 'm', 'd'); + static const FourCC::RawValue kIRModuleFourCc = SLANG_FOUR_CC('S', 'i', 'm', 'd'); /* NOTE! All FourCC that can be compressed must start with capital 'S', because compressed version is the same FourCC with the 'S' replaced with 's' */ - static const FourCC kInstFourCc = SLANG_FOUR_CC('S', 'L', 'i', 'n'); - static const FourCC kChildRunFourCc = SLANG_FOUR_CC('S', 'L', 'c', 'r'); - static const FourCC kExternalOperandsFourCc = SLANG_FOUR_CC('S', 'L', 'e', 'o'); + static const FourCC::RawValue kInstFourCc = SLANG_FOUR_CC('S', 'L', 'i', 'n'); + static const FourCC::RawValue kChildRunFourCc = SLANG_FOUR_CC('S', 'L', 'c', 'r'); + static const FourCC::RawValue kExternalOperandsFourCc = SLANG_FOUR_CC('S', 'L', 'e', 'o'); - static const FourCC kUInt32RawSourceLocFourCc = SLANG_FOUR_CC('S', 'r', 's', '4'); + static const FourCC::RawValue kUInt32RawSourceLocFourCc = SLANG_FOUR_CC('S', 'r', 's', '4'); /// Debug information is held elsewhere, but if this optional section exists, it maps /// instructions to locs - static const FourCC kDebugSourceLocRunFourCc = SLANG_FOUR_CC('S', 'd', 's', 'r'); + static const FourCC::RawValue kDebugSourceLocRunFourCc = SLANG_FOUR_CC('S', 'd', 's', 'r'); }; struct IRSerialData diff --git a/source/slang/slang-serialize-ir.cpp b/source/slang/slang-serialize-ir.cpp index bdea2faaa..f03f46da8 100644 --- a/source/slang/slang-serialize-ir.cpp +++ b/source/slang/slang-serialize-ir.cpp @@ -334,51 +334,43 @@ Result IRSerialWriter::write( Result _writeInstArrayChunk( FourCC chunkId, const List<IRSerialData::Inst>& array, - RiffContainer* container) + RIFF::BuildCursor& cursor) { - typedef RiffContainer::Chunk Chunk; - typedef RiffContainer::ScopeChunk ScopeChunk; - if (array.getCount() == 0) { return SLANG_OK; } - return SerialRiffUtil::writeArrayChunk(chunkId, array, container); + return SerialRiffUtil::writeArrayChunk(chunkId, array, cursor); } -/* static */ Result IRSerialWriter::writeContainer( - const IRSerialData& data, - RiffContainer* container) +/* static */ Result IRSerialWriter::writeTo(const IRSerialData& data, RIFF::BuildCursor& cursor) { - typedef RiffContainer::Chunk Chunk; - typedef RiffContainer::ScopeChunk ScopeChunk; - - ScopeChunk scopeModule(container, Chunk::Kind::List, Bin::kIRModuleFourCc); + SLANG_SCOPED_RIFF_BUILDER_LIST_CHUNK(cursor, Bin::kIRModuleFourCc); - SLANG_RETURN_ON_FAIL(_writeInstArrayChunk(Bin::kInstFourCc, data.m_insts, container)); + SLANG_RETURN_ON_FAIL(_writeInstArrayChunk(Bin::kInstFourCc, data.m_insts, cursor)); SLANG_RETURN_ON_FAIL( - SerialRiffUtil::writeArrayChunk(Bin::kChildRunFourCc, data.m_childRuns, container)); + SerialRiffUtil::writeArrayChunk(Bin::kChildRunFourCc, data.m_childRuns, cursor)); SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayChunk( Bin::kExternalOperandsFourCc, data.m_externalOperands, - container)); + cursor)); SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayChunk( SerialBinary::kStringTableFourCc, data.m_stringTable, - container)); + cursor)); SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayChunk( Bin::kUInt32RawSourceLocFourCc, data.m_rawSourceLocs, - container)); + cursor)); if (data.m_debugSourceLocRuns.getCount()) { SerialRiffUtil::writeArrayChunk( Bin::kDebugSourceLocRunFourCc, data.m_debugSourceLocRuns, - container); + cursor); } return SLANG_OK; @@ -417,31 +409,29 @@ Result _writeInstArrayChunk( // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! IRSerialReader !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -static Result _readInstArrayChunk( - RiffContainer::DataChunk* chunk, - List<IRSerialData::Inst>& arrayOut) +static Result _readInstArrayChunk(RIFF::DataChunk const* chunk, List<IRSerialData::Inst>& arrayOut) { SerialRiffUtil::ListResizerForType<IRSerialData::Inst> resizer(arrayOut); return SerialRiffUtil::readArrayChunk(chunk, resizer); } -/* static */ Result IRSerialReader::readContainer( - RiffContainer::ListChunk* module, +/* static */ Result IRSerialReader::readFrom( + IRModuleChunk const* irModuleChunk, IRSerialData* outData) { typedef IRSerialBinary Bin; outData->clear(); - for (RiffContainer::Chunk* chunk = module->m_containedChunks; chunk; chunk = chunk->m_next) + for (auto chunk : irModuleChunk->getChildren()) { - RiffContainer::DataChunk* dataChunk = as<RiffContainer::DataChunk>(chunk); + auto dataChunk = as<RIFF::DataChunk>(chunk); if (!dataChunk) { continue; } - switch (dataChunk->m_fourCC) + switch (dataChunk->getType()) { case Bin::kInstFourCc: { diff --git a/source/slang/slang-serialize-ir.h b/source/slang/slang-serialize-ir.h index 96121516b..c9fb16c27 100644 --- a/source/slang/slang-serialize-ir.h +++ b/source/slang/slang-serialize-ir.h @@ -13,6 +13,9 @@ namespace Slang { +struct IRModuleChunk : RIFF::ListChunk +{ +}; struct IRSerialWriter { @@ -26,7 +29,7 @@ struct IRSerialWriter IRSerialData* serialData); /// Write to a container - static Result writeContainer(const IRSerialData& data, RiffContainer* container); + static Result writeTo(const IRSerialData& data, RIFF::BuildCursor& cursor); /// Get an instruction index from an instruction Ser::InstIndex getInstIndex(IRInst* inst) const @@ -93,7 +96,7 @@ struct IRSerialReader typedef IRSerialData Ser; /// Read a stream to fill in dataOut IRSerialData - static Result readContainer(RiffContainer::ListChunk* module, IRSerialData* outData); + static Result readFrom(IRModuleChunk const* irModuleChunk, IRSerialData* outData); /// Read a module from serial data Result read( diff --git a/source/slang/slang-serialize-source-loc.cpp b/source/slang/slang-serialize-source-loc.cpp index b24324048..f612a6fb3 100644 --- a/source/slang/slang-serialize-source-loc.cpp +++ b/source/slang/slang-serialize-source-loc.cpp @@ -385,47 +385,44 @@ SlangResult SerialSourceLocReader::read( /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DebugSerialData !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ -/* static */ Result SerialSourceLocData::writeContainer(RiffContainer* container) +/* static */ Result SerialSourceLocData::writeTo(RIFF::BuildCursor& cursor) { - RiffContainer::ScopeChunk debugChunkScope( - container, - RiffContainer::Chunk::Kind::List, - SerialSourceLocData::kDebugFourCc); + SLANG_SCOPED_RIFF_BUILDER_LIST_CHUNK(cursor, SerialSourceLocData::kDebugFourCc); SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayChunk( SerialSourceLocData::kDebugStringFourCc, m_stringTable, - container)); + cursor)); SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayChunk( SerialSourceLocData::kDebugLineInfoFourCc, m_lineInfos, - container)); + cursor)); SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayChunk( SerialSourceLocData::kDebugAdjustedLineInfoFourCc, m_adjustedLineInfos, - container)); + cursor)); SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayChunk( SerialSourceLocData::kDebugSourceInfoFourCc, m_sourceInfos, - container)); + cursor)); return SLANG_OK; } -/* static */ Result SerialSourceLocData::readContainer(RiffContainer::ListChunk* listChunk) +/* static */ Result SerialSourceLocData::readFrom(RIFF::ListChunk const* listChunk) { - SLANG_ASSERT(listChunk->getSubType() == SerialSourceLocData::kDebugFourCc); + SLANG_ASSERT(listChunk->getType() == SerialSourceLocData::kDebugFourCc); clear(); - for (RiffContainer::Chunk* chunk = listChunk->m_containedChunks; chunk; chunk = chunk->m_next) + for (auto chunk : listChunk->getChildren()) { - RiffContainer::DataChunk* dataChunk = as<RiffContainer::DataChunk>(chunk); + auto dataChunk = as<RIFF::DataChunk>(chunk); if (!dataChunk) { continue; } - switch (dataChunk->m_fourCC) + switch (dataChunk->getType()) { case SerialSourceLocData::kDebugStringFourCc: { diff --git a/source/slang/slang-serialize-source-loc.h b/source/slang/slang-serialize-source-loc.h index 24e1813a4..c6ef982c1 100644 --- a/source/slang/slang-serialize-source-loc.h +++ b/source/slang/slang-serialize-source-loc.h @@ -21,12 +21,12 @@ public: typedef SerialStringData::StringIndex StringIndex; // The list that contains all the subsequent modules - static const FourCC kDebugFourCc = SLANG_FOUR_CC('S', 'd', 'e', 'b'); + static const FourCC::RawValue kDebugFourCc = SLANG_FOUR_CC('S', 'd', 'e', 'b'); - static const FourCC kDebugStringFourCc = SLANG_FOUR_CC('S', 'd', 's', 't'); - static const FourCC kDebugLineInfoFourCc = SLANG_FOUR_CC('S', 'd', 'l', 'n'); - static const FourCC kDebugAdjustedLineInfoFourCc = SLANG_FOUR_CC('S', 'd', 'a', 'l'); - static const FourCC kDebugSourceInfoFourCc = SLANG_FOUR_CC('S', 'd', 's', 'o'); + static const FourCC::RawValue kDebugStringFourCc = SLANG_FOUR_CC('S', 'd', 's', 't'); + static const FourCC::RawValue kDebugLineInfoFourCc = SLANG_FOUR_CC('S', 'd', 'l', 'n'); + static const FourCC::RawValue kDebugAdjustedLineInfoFourCc = SLANG_FOUR_CC('S', 'd', 'a', 'l'); + static const FourCC::RawValue kDebugSourceInfoFourCc = SLANG_FOUR_CC('S', 'd', 's', 'o'); struct SourceRange { @@ -135,8 +135,8 @@ public: bool operator==(const ThisType& rhs) const; - Result writeContainer(RiffContainer* container); - Result readContainer(RiffContainer::ListChunk* listChunk); + Result writeTo(RIFF::BuildCursor& cursor); + Result readFrom(RIFF::ListChunk const* chunk); List<char> m_stringTable; ///< String table for debug use only List<LineInfo> m_lineInfos; ///< Line information diff --git a/source/slang/slang-serialize-types.cpp b/source/slang/slang-serialize-types.cpp index 2e6e56bb4..ca3bde608 100644 --- a/source/slang/slang-serialize-types.cpp +++ b/source/slang/slang-serialize-types.cpp @@ -149,41 +149,38 @@ struct ByteReader const void* data, size_t numEntries, size_t typeSize, - RiffContainer* container) + RIFF::BuildCursor& cursor) { - typedef RiffContainer::Chunk Chunk; - typedef RiffContainer::ScopeChunk ScopeChunk; - if (numEntries == 0) { return SLANG_OK; } - ScopeChunk scope(container, Chunk::Kind::Data, chunkId); + SLANG_SCOPED_RIFF_BUILDER_DATA_CHUNK(cursor, chunkId); SerialBinary::ArrayHeader header; header.numEntries = uint32_t(numEntries); - container->write(&header, sizeof(header)); - container->write(data, typeSize * numEntries); + cursor.addData(&header, sizeof(header)); + cursor.addData(data, typeSize * numEntries); return SLANG_OK; } /* static */ Result SerialRiffUtil::readArrayChunk( - RiffContainer::DataChunk* dataChunk, + RIFF::DataChunk const* dataChunk, ListResizer& listOut) { typedef SerialBinary Bin; - RiffReadHelper read = dataChunk->asReadHelper(); - const size_t typeSize = listOut.getTypeSize(); + MemoryReader reader(dataChunk->getPayload(), dataChunk->getPayloadSize()); + const Size typeSize = listOut.getTypeSize(); Bin::ArrayHeader header; - SLANG_RETURN_ON_FAIL(read.read(header)); - const size_t payloadSize = header.numEntries * typeSize; - SLANG_ASSERT(payloadSize == read.getRemainingSize()); + SLANG_RETURN_ON_FAIL(reader.read(header)); + const Size payloadSize = header.numEntries * typeSize; + SLANG_ASSERT(payloadSize == reader.getRemainingSize()); void* dst = listOut.setSize(header.numEntries); - ::memcpy(dst, read.getData(), payloadSize); + ::memcpy(dst, reader.getRemainingData(), payloadSize); return SLANG_OK; } diff --git a/source/slang/slang-serialize-types.h b/source/slang/slang-serialize-types.h index cd2b4c99c..09de865db 100644 --- a/source/slang/slang-serialize-types.h +++ b/source/slang/slang-serialize-types.h @@ -123,66 +123,65 @@ struct PropertyKeys template<> struct PropertyKeys<Module> { - static const FourCC Digest = SLANG_FOUR_CC('S', 'H', 'A', '1'); - static const FourCC ASTModule = SLANG_FOUR_CC('a', 's', 't', ' '); - static const FourCC IRModule = SLANG_FOUR_CC('i', 'r', ' ', ' '); - static const FourCC FileDependencies = SLANG_FOUR_CC('f', 'd', 'e', 'p'); + static const FourCC::RawValue Digest = SLANG_FOUR_CC('S', 'H', 'A', '1'); + static const FourCC::RawValue ASTModule = SLANG_FOUR_CC('a', 's', 't', ' '); + static const FourCC::RawValue FileDependencies = SLANG_FOUR_CC('f', 'd', 'e', 'p'); }; // For types/FourCC that work for serializing in general (not just IR). struct SerialBinary { - static const FourCC kRiffFourCc = RiffFourCC::kRiff; + static const FourCC::RawValue kRiffFourCc = RIFF::RootChunk::kTag; /// Container - static const FourCC kContainerFourCc = SLANG_FOUR_CC('S', 'L', 'm', 'c'); + static const FourCC::RawValue kContainerFourCc = SLANG_FOUR_CC('S', 'L', 'm', 'c'); /// A string table - static const FourCC kStringTableFourCc = SLANG_FOUR_CC('S', 'L', 's', 't'); + static const FourCC::RawValue kStringTableFourCc = SLANG_FOUR_CC('S', 'L', 's', 't'); /// TranslationUnitList - static const FourCC kModuleListFourCc = SLANG_FOUR_CC('S', 'L', 'm', 'l'); + static const FourCC::RawValue kModuleListFourCc = SLANG_FOUR_CC('S', 'L', 'm', 'l'); /// An entry point - static const FourCC kEntryPointFourCc = SLANG_FOUR_CC('E', 'P', 'n', 't'); + static const FourCC::RawValue kEntryPointFourCc = SLANG_FOUR_CC('E', 'P', 'n', 't'); - static const FourCC kEntryPointListFourCc = SLANG_FOUR_CC('e', 'p', 't', 's'); + static const FourCC::RawValue kEntryPointListFourCc = SLANG_FOUR_CC('e', 'p', 't', 's'); // Module - static const FourCC kModuleFourCC = SLANG_FOUR_CC('s', 'm', 'o', 'd'); + static const FourCC::RawValue kModuleFourCC = SLANG_FOUR_CC('s', 'm', 'o', 'd'); // The following are "generic" codes, suitable for // use when serializing content using JSON-like structure. // - static const FourCC kObjectFourCC = SLANG_FOUR_CC('o', 'b', 'j', ' '); - static const FourCC kPairFourCC = SLANG_FOUR_CC('p', 'a', 'i', 'r'); - static const FourCC kArrayFourCC = SLANG_FOUR_CC('a', 'r', 'r', 'y'); - static const FourCC kDictionaryFourCC = SLANG_FOUR_CC('d', 'i', 'c', 't'); - static const FourCC kNullFourCC = SLANG_FOUR_CC('n', 'u', 'l', 'l'); - static const FourCC kStringFourCC = SLANG_FOUR_CC('s', 't', 'r', ' '); - static const FourCC kTrueFourCC = SLANG_FOUR_CC('t', 'r', 'u', 'e'); - static const FourCC kFalseFourCC = SLANG_FOUR_CC('f', 'a', 'l', 's'); - static const FourCC kInt32FourCC = SLANG_FOUR_CC('i', '3', '2', ' '); - static const FourCC kUInt32FourCC = SLANG_FOUR_CC('u', '3', '2', ' '); - static const FourCC kFloat32FourCC = SLANG_FOUR_CC('f', '3', '2', ' '); - static const FourCC kInt64FourCC = SLANG_FOUR_CC('i', '6', '4', ' '); - static const FourCC kUInt64FourCC = SLANG_FOUR_CC('u', '6', '4', ' '); - static const FourCC kFloat64FourCC = SLANG_FOUR_CC('f', '6', '4', ' '); + static const FourCC::RawValue kObjectFourCC = SLANG_FOUR_CC('o', 'b', 'j', ' '); + static const FourCC::RawValue kPairFourCC = SLANG_FOUR_CC('p', 'a', 'i', 'r'); + static const FourCC::RawValue kArrayFourCC = SLANG_FOUR_CC('a', 'r', 'r', 'y'); + static const FourCC::RawValue kDictionaryFourCC = SLANG_FOUR_CC('d', 'i', 'c', 't'); + static const FourCC::RawValue kNullFourCC = SLANG_FOUR_CC('n', 'u', 'l', 'l'); + static const FourCC::RawValue kStringFourCC = SLANG_FOUR_CC('s', 't', 'r', ' '); + static const FourCC::RawValue kTrueFourCC = SLANG_FOUR_CC('t', 'r', 'u', 'e'); + static const FourCC::RawValue kFalseFourCC = SLANG_FOUR_CC('f', 'a', 'l', 's'); + static const FourCC::RawValue kInt32FourCC = SLANG_FOUR_CC('i', '3', '2', ' '); + static const FourCC::RawValue kUInt32FourCC = SLANG_FOUR_CC('u', '3', '2', ' '); + static const FourCC::RawValue kFloat32FourCC = SLANG_FOUR_CC('f', '3', '2', ' '); + static const FourCC::RawValue kInt64FourCC = SLANG_FOUR_CC('i', '6', '4', ' '); + static const FourCC::RawValue kUInt64FourCC = SLANG_FOUR_CC('u', '6', '4', ' '); + static const FourCC::RawValue kFloat64FourCC = SLANG_FOUR_CC('f', '6', '4', ' '); // The following codes are suitable for use when serializing // content that represents a logical file system. // - static const FourCC kDirectoryFourCC = SLANG_FOUR_CC('d', 'i', 'r', ' '); - static const FourCC kFileFourCC = SLANG_FOUR_CC('f', 'i', 'l', 'e'); - static const FourCC kNameFourCC = SLANG_FOUR_CC('n', 'a', 'm', 'e'); - static const FourCC kPathFourCC = SLANG_FOUR_CC('p', 'a', 't', 'h'); - static const FourCC kDataFourCC = SLANG_FOUR_CC('d', 'a', 't', 'a'); + static const FourCC::RawValue kDirectoryFourCC = SLANG_FOUR_CC('d', 'i', 'r', ' '); + static const FourCC::RawValue kFileFourCC = SLANG_FOUR_CC('f', 'i', 'l', 'e'); + static const FourCC::RawValue kNameFourCC = SLANG_FOUR_CC('n', 'a', 'm', 'e'); + static const FourCC::RawValue kPathFourCC = SLANG_FOUR_CC('p', 'a', 't', 'h'); + static const FourCC::RawValue kDataFourCC = SLANG_FOUR_CC('d', 'a', 't', 'a'); // TODO(tfoley): Figure out where to put all of these so that // they can be more usefully addressed. // - static const FourCC kMangledNameFourCC = SLANG_FOUR_CC('m', 'g', 'n', 'm'); - static const FourCC kProfileFourCC = SLANG_FOUR_CC('p', 'r', 'o', 'f'); + static const FourCC::RawValue kMangledNameFourCC = SLANG_FOUR_CC('m', 'g', 'n', 'm'); + static const FourCC::RawValue kProfileFourCC = SLANG_FOUR_CC('p', 'r', 'o', 'f'); struct ArrayHeader @@ -233,23 +232,18 @@ struct SerialRiffUtil const void* data, size_t numEntries, size_t typeSize, - RiffContainer* container); + RIFF::BuildCursor& cursor); template<typename T> - static Result writeArrayChunk(FourCC chunkId, const List<T>& array, RiffContainer* container) + static Result writeArrayChunk(FourCC chunkId, const List<T>& array, RIFF::BuildCursor& cursor) { - return writeArrayChunk( - chunkId, - array.begin(), - size_t(array.getCount()), - sizeof(T), - container); + return writeArrayChunk(chunkId, array.begin(), size_t(array.getCount()), sizeof(T), cursor); } - static Result readArrayChunk(RiffContainer::DataChunk* dataChunk, ListResizer& listOut); + static Result readArrayChunk(RIFF::DataChunk const* dataChunk, ListResizer& listOut); template<typename T> - static Result readArrayChunk(RiffContainer::DataChunk* dataChunk, List<T>& arrayOut) + static Result readArrayChunk(RIFF::DataChunk const* dataChunk, List<T>& arrayOut) { ListResizerForType<T> resizer(arrayOut); return readArrayChunk(dataChunk, resizer); diff --git a/source/slang/slang-serialize.h b/source/slang/slang-serialize.h index b694850ea..4b864fc94 100644 --- a/source/slang/slang-serialize.h +++ b/source/slang/slang-serialize.h @@ -30,52 +30,39 @@ struct ValNodeDesc; struct Encoder { public: - Encoder(Stream* stream) - : _stream(stream) + Encoder() {} + + Encoder(RIFF::Builder& riff) + : _cursor(riff) { } - ~Encoder() { RiffUtil::write(&_riff, _stream); } - - void beginArray(FourCC typeCode) + Encoder(RIFF::ListChunkBuilder* chunk) + : _cursor(chunk) { - _riff.startChunk(RiffContainer::Chunk::Kind::List, typeCode); } + void beginArray(FourCC typeCode) { _cursor.beginListChunk(typeCode); } + void beginArray() { beginArray(SerialBinary::kArrayFourCC); } - void endArray() - { - _riff.endChunk(); - // TODO: maybe end key... - } + void endArray() { _cursor.endChunk(); } - void beginObject(FourCC typeCode) - { - _riff.startChunk(RiffContainer::Chunk::Kind::List, typeCode); - } + void beginObject(FourCC typeCode) { _cursor.beginListChunk(typeCode); } void beginObject() { beginObject(SerialBinary::kObjectFourCC); } - void endObject() { _riff.endChunk(); } + void endObject() { _cursor.endChunk(); } - void beginKeyValuePair() - { - _riff.startChunk(RiffContainer::Chunk::Kind::List, SerialBinary::kPairFourCC); - } + void beginKeyValuePair(FourCC keyCode) { _cursor.beginListChunk(keyCode); } - void endKeyValuePair() { _riff.endChunk(); } + void beginKeyValuePair() { beginKeyValuePair(SerialBinary::kPairFourCC); } - void beginKeyValuePair(FourCC keyCode) - { - _riff.startChunk(RiffContainer::Chunk::Kind::List, keyCode); - } + void endKeyValuePair() { _cursor.endChunk(); } void encodeData(FourCC typeCode, void const* data, size_t size) { - _riff.startChunk(RiffContainer::Chunk::Kind::Data, typeCode); - _riff.write(data, size); - _riff.endChunk(); + _cursor.addDataChunk(typeCode, data, size); } void encodeData(void const* data, size_t size) @@ -200,24 +187,21 @@ public: }; private: - Stream* _stream = nullptr; - - // Implementation details below... - RiffContainer _riff; + RIFF::BuildCursor _cursor; public: - RiffContainer* getRIFF() { return &_riff; } + operator RIFF::BuildCursor&() { return _cursor; } - RiffContainer::Chunk* getRIFFChunk() { return _riff.getCurrentChunk(); } + RIFF::ChunkBuilder* getRIFFChunk() { return _cursor.getCurrentChunk(); } - void setRIFFChunk(RiffContainer::Chunk* chunk) { _riff.setCurrentChunk(chunk); } + void setRIFFChunk(RIFF::ChunkBuilder* chunk) { _cursor.setCurrentChunk(chunk); } }; struct Decoder { public: - Decoder(RiffContainer::Chunk* chunk) - : _chunk(chunk) + Decoder(RIFF::Chunk const* chunk) + : _cursor(chunk) { } @@ -226,10 +210,10 @@ public: switch (getTag()) { case SerialBinary::kTrueFourCC: - _chunk = _chunk->m_next; + _advanceCursor(); return true; case SerialBinary::kFalseFourCC: - _chunk = _chunk->m_next; + _advanceCursor(); return false; default: @@ -246,20 +230,20 @@ public: UNREACHABLE_RETURN(""); } - auto dataChunk = as<RiffContainer::DataChunk>(_chunk); + auto dataChunk = as<RIFF::DataChunk>(_cursor); if (!dataChunk) { SLANG_UNEXPECTED("invalid format in RIFF"); UNREACHABLE_RETURN(""); } - auto size = dataChunk->calcPayloadSize(); + auto size = dataChunk->getPayloadSize(); String value; value.appendRepeatedChar(' ', size); - dataChunk->getPayload((char*)value.getBuffer()); + dataChunk->writePayloadInto((char*)value.getBuffer(), size); - _chunk = _chunk->m_next; + _advanceCursor(); return value; } @@ -267,13 +251,14 @@ public: { if (getTag() == typeTag) { - auto dataChunk = as<RiffContainer::DataChunk>(_chunk); + auto dataChunk = as<RIFF::DataChunk>(_cursor); if (dataChunk) { - if (dataChunk->calcPayloadSize() >= dataSize) + auto payloadSize = dataChunk->getPayloadSize(); + if (payloadSize >= dataSize) { - dataChunk->getPayload(outData); - _chunk = _chunk->m_next; + dataChunk->writePayloadInto(outData, dataSize); + _advanceCursor(); return; } } @@ -368,7 +353,7 @@ public: float decodeFloat32() { return float(decodeFloat()); } double decodeFloat64() { return decodeFloat(); } - FourCC getTag() { return _chunk ? _chunk->m_fourCC : 0; } + FourCC getTag() { return _cursor ? _cursor->getType() : FourCC(0); } Int32 _decodeImpl(Int32*) { return decodeInt32(); } UInt32 _decodeImpl(UInt32*) { return decodeUInt32(); } @@ -393,74 +378,74 @@ public: void beginArray(FourCC typeCode = SerialBinary::kArrayFourCC) { - auto listChunk = as<RiffContainer::ListChunk>(_chunk); + auto listChunk = as<RIFF::ListChunk>(_cursor); if (!listChunk) { SLANG_UNEXPECTED("invalid format in RIFF"); } - if (listChunk->m_fourCC != typeCode) + if (listChunk->getType() != typeCode) { SLANG_UNEXPECTED("invalid format in RIFF"); } - _chunk = listChunk->getFirstContainedChunk(); + _cursor = listChunk->getFirstChild(); } void beginObject(FourCC typeCode = SerialBinary::kObjectFourCC) { - auto listChunk = as<RiffContainer::ListChunk>(_chunk); + auto listChunk = as<RIFF::ListChunk>(_cursor); if (!listChunk) { SLANG_UNEXPECTED("invalid format in RIFF"); } - if (listChunk->m_fourCC != typeCode) + if (listChunk->getType() != typeCode) { SLANG_UNEXPECTED("invalid format in RIFF"); } - _chunk = listChunk->getFirstContainedChunk(); + _cursor = listChunk->getFirstChild(); } void beginKeyValuePair(FourCC typeCode = SerialBinary::kPairFourCC) { - auto listChunk = as<RiffContainer::ListChunk>(_chunk); + auto listChunk = as<RIFF::ListChunk>(_cursor); if (!listChunk) { SLANG_UNEXPECTED("invalid format in RIFF"); } - if (listChunk->m_fourCC != typeCode) + if (listChunk->getType() != typeCode) { SLANG_UNEXPECTED("invalid format in RIFF"); } - _chunk = listChunk->getFirstContainedChunk(); + _cursor = listChunk->getFirstChild(); } void beginProperty(FourCC propertyCode) { - auto listChunk = as<RiffContainer::ListChunk>(_chunk); + auto listChunk = as<RIFF::ListChunk>(_cursor); if (!listChunk) { SLANG_UNEXPECTED("invalid format in RIFF"); } - auto found = listChunk->findContainedList(propertyCode); + auto found = listChunk->findListChunk(propertyCode); if (!found) { SLANG_UNEXPECTED("invalid format in RIFF"); } - _chunk = found->getFirstContainedChunk(); + _cursor = found->getFirstChild(); } - bool hasElements() { return _chunk != nullptr; } + bool hasElements() { return _cursor != nullptr; } bool isNull() { - if (_chunk == nullptr) + if (_cursor == nullptr) return true; if (getTag() == SerialBinary::kNullFourCC) return true; @@ -472,34 +457,36 @@ public: if (!isNull()) return false; - if (_chunk != nullptr) + if (_cursor != nullptr) { - _chunk = _chunk->m_next; + _advanceCursor(); } return true; } + using Cursor = RIFF::BoundsCheckedChunkPtr; + struct WithArray { public: WithArray(Decoder& decoder) : _decoder(decoder) { - _saved = decoder._chunk; + _saved = decoder._cursor; decoder.beginArray(); } WithArray(Decoder& decoder, FourCC typeCode) : _decoder(decoder) { - _saved = decoder._chunk; + _saved = decoder._cursor; decoder.beginArray(typeCode); } - ~WithArray() { _decoder._chunk = _saved->m_next; } + ~WithArray() { _decoder._cursor = _saved.getNextSibling(); } private: - RiffContainer::Chunk* _saved; + Cursor _saved; Decoder& _decoder; }; @@ -509,21 +496,21 @@ public: WithObject(Decoder& decoder) : _decoder(decoder) { - _saved = decoder._chunk; + _saved = decoder._cursor; decoder.beginObject(); } WithObject(Decoder& decoder, FourCC typeCode) : _decoder(decoder) { - _saved = decoder._chunk; + _saved = decoder._cursor; decoder.beginObject(typeCode); } - ~WithObject() { _decoder._chunk = _saved->m_next; } + ~WithObject() { _decoder._cursor = _saved.getNextSibling(); } private: - RiffContainer::Chunk* _saved; + Cursor _saved; Decoder& _decoder; }; @@ -533,21 +520,21 @@ public: WithKeyValuePair(Decoder& decoder) : _decoder(decoder) { - _saved = decoder._chunk; + _saved = decoder._cursor; decoder.beginKeyValuePair(); } WithKeyValuePair(Decoder& decoder, FourCC typeCode) : _decoder(decoder) { - _saved = decoder._chunk; + _saved = decoder._cursor; _decoder.beginKeyValuePair(typeCode); } - ~WithKeyValuePair() { _decoder._chunk = _saved->m_next; } + ~WithKeyValuePair() { _decoder._cursor = _saved.getNextSibling(); } private: - RiffContainer::Chunk* _saved; + Cursor _saved; Decoder& _decoder; }; @@ -557,23 +544,26 @@ public: WithProperty(Decoder& decoder, FourCC typeCode) : _decoder(decoder) { - _saved = decoder._chunk; + _saved = decoder._cursor; _decoder.beginProperty(typeCode); } - ~WithProperty() { _decoder._chunk = _saved->m_next; } + ~WithProperty() { _decoder._cursor = _saved.getNextSibling(); } private: - RiffContainer::Chunk* _saved; + Cursor _saved; Decoder& _decoder; }; + Cursor getCursor() const { return _cursor; } + void setCursor(Cursor const& cursor) { _cursor = cursor; } - RiffContainer::Chunk* getCursor() { return _chunk; } - void setCursor(RiffContainer::Chunk* chunk) { _chunk = chunk; } + RIFF::Chunk const* getCurrentChunk() const { return getCursor(); } private: - RiffContainer::Chunk* _chunk = nullptr; + void _advanceCursor() { _cursor = _cursor.getNextSibling(); } + + Cursor _cursor; }; diff --git a/source/slang/slang-type-layout.cpp b/source/slang/slang-type-layout.cpp index 68df665d4..9dea0f167 100644 --- a/source/slang/slang-type-layout.cpp +++ b/source/slang/slang-type-layout.cpp @@ -4582,7 +4582,7 @@ static TypeLayoutResult _updateLayout( if (layoutResultPtr) { // Check the layout is the same! - SLANG_ASSERT(layoutResultPtr->layout.get() == result.layout); + SLANG_ASSERT(layoutResultPtr->layout == result.layout); // Update the info layoutResultPtr->info = result.info; } diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp index 613fb7090..602446cda 100644 --- a/source/slang/slang.cpp +++ b/source/slang/slang.cpp @@ -687,34 +687,31 @@ SlangResult Session::_readBuiltinModule( StringBuilder moduleFilename; moduleFilename << moduleName << ".slang-module"; - RiffContainer riffContainer; - { - // Load it - ComPtr<ISlangBlob> blob; - SLANG_RETURN_ON_FAIL(fileSystem->loadFile(moduleFilename.getBuffer(), blob.writeRef())); - - // Set up a stream - MemoryStreamBase stream(FileAccess::Read, blob->getBufferPointer(), blob->getBufferSize()); + // Load it + ComPtr<ISlangBlob> fileContents; + SLANG_RETURN_ON_FAIL(fileSystem->loadFile(moduleFilename.getBuffer(), fileContents.writeRef())); - // Load the riff container - SLANG_RETURN_ON_FAIL(RiffUtil::read(&stream, riffContainer)); + RIFF::RootChunk const* rootChunk = RIFF::RootChunk::getFromBlob(fileContents); + if (!rootChunk) + { + return SLANG_FAIL; } Linkage* linkage = getBuiltinLinkage(); SourceManager* sourceManager = getBuiltinSourceManager(); NamePool* sessionNamePool = &namePool; - auto moduleChunk = ModuleChunkRef::find(&riffContainer); + auto moduleChunk = ModuleChunk::find(rootChunk); if (!moduleChunk) return SLANG_FAIL; - SHA1::Digest moduleDigest = moduleChunk.getDigest(); + SHA1::Digest moduleDigest = moduleChunk->getDigest(); - auto irChunk = moduleChunk.findIR(); + auto irChunk = moduleChunk->findIR(); if (!irChunk) return SLANG_FAIL; - auto astChunk = moduleChunk.findAST(); + auto astChunk = moduleChunk->findAST(); if (!astChunk) return SLANG_FAIL; @@ -724,7 +721,7 @@ SlangResult Session::_readBuiltinModule( // in the IR and AST deserialization (if we find anything). // RefPtr<SerialSourceLocReader> sourceLocReader; - if (auto debugChunk = findDebugChunk(moduleChunk.ptr())) + if (auto debugChunk = DebugChunk::find(moduleChunk)) { SLANG_RETURN_ON_FAIL( readSourceLocationsFromDebugChunk(debugChunk, sourceManager, sourceLocReader)); @@ -4134,7 +4131,8 @@ void Linkage::loadParsedModule( } RefPtr<Module> Linkage::findOrLoadSerializedModuleForModuleLibrary( - ModuleChunkRef moduleChunk, + ModuleChunk const* moduleChunk, + RIFF::ListChunk const* libraryChunk, DiagnosticSink* sink) { RefPtr<Module> resultModule; @@ -4147,7 +4145,7 @@ RefPtr<Module> Linkage::findOrLoadSerializedModuleForModuleLibrary( // The first step is to simply decode the module name, and // see if we have a already loaded a matching module. - auto moduleName = getNamePool()->getName(moduleChunk.getName()); + auto moduleName = getNamePool()->getName(moduleChunk->getName()); if (mapNameToLoadedModules.tryGetValue(moduleName, resultModule)) return resultModule; @@ -4163,12 +4161,12 @@ RefPtr<Module> Linkage::findOrLoadSerializedModuleForModuleLibrary( // already. It isn't something that can be fixed in just one // place at this point. - auto fileDependenciesChunk = moduleChunk.getFileDependencies(); - auto firstFileDependencyChunk = fileDependenciesChunk.getFirst(); + auto fileDependenciesList = moduleChunk->getFileDependencies(); + auto firstFileDependencyChunk = fileDependenciesList.getFirst(); if (!firstFileDependencyChunk) return nullptr; - auto modulePathInfo = PathInfo::makePath(firstFileDependencyChunk.getValue()); + auto modulePathInfo = PathInfo::makePath(firstFileDependencyChunk->getValue()); if (mapPathToLoadedModule.tryGetValue(modulePathInfo.getMostUniqueIdentity(), resultModule)) return resultModule; @@ -4176,13 +4174,20 @@ RefPtr<Module> Linkage::findOrLoadSerializedModuleForModuleLibrary( // will go ahead and load the module from the serialized form. // PathInfo filePathInfo; - return loadSerializedModule(moduleName, modulePathInfo, moduleChunk, SourceLoc(), sink); + return loadSerializedModule( + moduleName, + modulePathInfo, + moduleChunk, + libraryChunk, + SourceLoc(), + sink); } RefPtr<Module> Linkage::loadSerializedModule( Name* moduleName, const PathInfo& moduleFilePathInfo, - ModuleChunkRef moduleChunk, + ModuleChunk const* moduleChunk, + RIFF::ListChunk const* containerChunk, SourceLoc const& requestingLoc, DiagnosticSink* sink) { @@ -4211,8 +4216,12 @@ RefPtr<Module> Linkage::loadSerializedModule( mapNameToLoadedModules.add(moduleName, module); try { - if (SLANG_FAILED( - loadSerializedModuleContents(module, moduleFilePathInfo, moduleChunk, sink))) + if (SLANG_FAILED(loadSerializedModuleContents( + module, + moduleFilePathInfo, + moduleChunk, + containerChunk, + sink))) { mapPathToLoadedModule.remove(mostUniqueIdentity); mapNameToLoadedModules.remove(moduleName); @@ -4240,24 +4249,17 @@ RefPtr<Module> Linkage::loadBinaryModuleImpl( auto astBuilder = getASTBuilder(); SLANG_AST_BUILDER_RAII(astBuilder); - // We start by reading the content of the file into + // We start by reading the content of the file as // an in-memory RIFF container. // - // TODO(tfoley): this is an unnecessary copy step, since - // we can simply use the contents of the blob directly - // and navigate it in-memory. - // - RiffContainer riffContainer; + auto rootChunk = RIFF::RootChunk::getFromBlob(moduleFileContents); + if (!rootChunk) { - MemoryStreamBase readStream( - FileAccess::Read, - moduleFileContents->getBufferPointer(), - moduleFileContents->getBufferSize()); - SLANG_RETURN_NULL_ON_FAIL(RiffUtil::read(&readStream, riffContainer)); + return nullptr; } - auto moduleChunkRef = ModuleChunkRef::find(&riffContainer); - if (!moduleChunkRef) + auto moduleChunk = ModuleChunk::find(rootChunk); + if (!moduleChunk) { return nullptr; } @@ -4271,7 +4273,7 @@ RefPtr<Module> Linkage::loadBinaryModuleImpl( SLANG_ASSERT(mostUniqueIdentity.getLength() > 0); if (m_optionSet.getBoolOption(CompilerOptionName::UseUpToDateBinaryModule)) { - if (!isBinaryModuleUpToDate(moduleFilePathInfo.foundPath, moduleChunkRef)) + if (!isBinaryModuleUpToDate(moduleFilePathInfo.foundPath, moduleChunk)) { return nullptr; } @@ -4280,8 +4282,13 @@ RefPtr<Module> Linkage::loadBinaryModuleImpl( // If everything seems reasonable, then we will go ahead and load // the module more completely from that serialized representation. // - RefPtr<Module> module = - loadSerializedModule(moduleName, moduleFilePathInfo, moduleChunkRef, requestingLoc, sink); + RefPtr<Module> module = loadSerializedModule( + moduleName, + moduleFilePathInfo, + moduleChunk, + rootChunk, + requestingLoc, + sink); return module; } @@ -4744,18 +4751,13 @@ SourceFile* Linkage::loadSourceFile(String pathFrom, String path) } // Check if a serialized module is up-to-date with current compiler options and source files. -bool Linkage::isBinaryModuleUpToDate(String fromPath, RiffContainer* riffContainer) +bool Linkage::isBinaryModuleUpToDate(String fromPath, RIFF::ListChunk const* baseChunk) { - auto moduleChunk = ModuleChunkRef::find(riffContainer); + auto moduleChunk = ModuleChunk::find(baseChunk); if (!moduleChunk) return false; - return isBinaryModuleUpToDate(fromPath, moduleChunk); -} - -bool Linkage::isBinaryModuleUpToDate(String fromPath, ModuleChunkRef moduleChunk) -{ - SHA1::Digest existingDigest = moduleChunk.getDigest(); + SHA1::Digest existingDigest = moduleChunk->getDigest(); DigestBuilder<SHA1> digestBuilder; auto version = String(getBuildTagString()); @@ -4765,10 +4767,10 @@ bool Linkage::isBinaryModuleUpToDate(String fromPath, ModuleChunkRef moduleChunk // Find the canonical path of the directory containing the module source file. String moduleSrcPath = ""; - auto dependencyChunks = moduleChunk.getFileDependencies(); + auto dependencyChunks = moduleChunk->getFileDependencies(); if (auto firstDependencyChunk = dependencyChunks.getFirst()) { - moduleSrcPath = firstDependencyChunk.getValue(); + moduleSrcPath = firstDependencyChunk->getValue(); IncludeSystem includeSystem( &getSearchDirectories(), @@ -4784,7 +4786,7 @@ bool Linkage::isBinaryModuleUpToDate(String fromPath, ModuleChunkRef moduleChunk for (auto dependencyChunk : dependencyChunks) { - auto file = dependencyChunk.getValue(); + auto file = dependencyChunk->getValue(); auto sourceFile = loadSourceFile(fromPath, file); if (!sourceFile) { @@ -4803,14 +4805,10 @@ bool Linkage::isBinaryModuleUpToDate(String fromPath, ModuleChunkRef moduleChunk SLANG_NO_THROW bool SLANG_MCALL Linkage::isBinaryModuleUpToDate(const char* modulePath, slang::IBlob* binaryModuleBlob) { - RiffContainer container; - MemoryStreamBase readStream( - FileAccess::Read, - binaryModuleBlob->getBufferPointer(), - binaryModuleBlob->getBufferSize()); - if (SLANG_FAILED(RiffUtil::read(&readStream, container))) + auto rootChunk = RIFF::RootChunk::getFromBlob(binaryModuleBlob); + if (!rootChunk) return false; - return isBinaryModuleUpToDate(modulePath, &container); + return isBinaryModuleUpToDate(modulePath, rootChunk); } SourceFile* Linkage::findFile(Name* name, SourceLoc loc, IncludeSystem& outIncludeSystem) @@ -6534,7 +6532,8 @@ void Linkage::setFileSystem(ISlangFileSystem* inFileSystem) SlangResult Linkage::loadSerializedModuleContents( Module* module, const PathInfo& moduleFilePathInfo, - ModuleChunkRef moduleChunk, + ModuleChunk const* moduleChunk, + RIFF::ListChunk const* containerChunk, DiagnosticSink* sink) { // At this point we've dealt with basically all of @@ -6542,19 +6541,49 @@ SlangResult Linkage::loadSerializedModuleContents( // to the real work of decoding the information // in the `moduleChunk`. + // + // TODO(tfoley): The fact that a separate `containerChunk` is getting + // passed in here is entirely byproduct of the support for "module libraries" + // that can (in principle) contain multiple serialized modules. When + // things are serialized in the "container" representation used for + // a module library, there is a single `DebugChunk` as a child of + // the container, with all of the `ModuleChunk`s sharing that debug info. + // + // In contrast, the more typical kind of serialized module that the compiler + // produces serializes a single `ModuleChunk`, and the `DebugChunk` is + // one of its direct children. Thus there are currently two different + // locations where debug information might be found. + // + // Prior to the change where we navigate the serialized RIFF hierarchy + // in memory without copying it, this issue was addressed by having + // the subroutine that looked for a `DebugChunk` start at the `ModuleChunk` + // and work its way up through the hierarchy using parent pointers that + // were created as part of RIFF loading. When navigating the RIFF in-place + // we don't have such parent pointers. + // + // As a short-term solution, we should deprecate and remove the support + // for "module libraries" so that the code doesn't have to handle two + // different layouts. + // + // In the longer term, we should be making some conscious design decisions + // around how we want to organize the top-level structure of our serialized + // intermediate/output formats, since there's quite a mix of different + // approaches currently in use. + // + auto sourceManager = getSourceManager(); RefPtr<SerialSourceLocReader> sourceLocReader; - if (auto debugChunk = findDebugChunk(moduleChunk.ptr())) + if (auto debugChunk = DebugChunk::find(moduleChunk, containerChunk)) { SLANG_RETURN_ON_FAIL( readSourceLocationsFromDebugChunk(debugChunk, sourceManager, sourceLocReader)); } - auto astChunk = moduleChunk.findAST(); + auto astChunk = moduleChunk->findAST(); if (!astChunk) return SLANG_FAIL; - auto irChunk = moduleChunk.findIR(); + auto irChunk = moduleChunk->findIR(); if (!irChunk) return SLANG_FAIL; @@ -6620,9 +6649,9 @@ SlangResult Linkage::loadSerializedModuleContents( module->clearFileDependency(); String moduleSourcePath = moduleFilePathInfo.foundPath; bool isFirst = true; - for (auto depenencyFileChunk : moduleChunk.getFileDependencies()) + for (auto depenencyFileChunk : moduleChunk->getFileDependencies()) { - auto encodedDependencyFilePath = depenencyFileChunk.getValue(); + auto encodedDependencyFilePath = depenencyFileChunk->getValue(); auto sourceFile = loadSourceFile(moduleFilePathInfo.foundPath, encodedDependencyFilePath); if (isFirst) @@ -6646,7 +6675,7 @@ SlangResult Linkage::loadSerializedModuleContents( } } module->setPathInfo(moduleFilePathInfo); - module->setDigest(moduleChunk.getDigest()); + module->setDigest(moduleChunk->getDigest()); module->_collectShaderParams(); module->_discoverEntryPoints(sink, targets); @@ -7159,8 +7188,8 @@ SlangResult _addLibraryReference( SourceMap::getTypeGuid(), ArtifactKeep::Yes, castable.writeRef())); - auto sourceMap = asBoxValue<SourceMap>(castable); - SLANG_ASSERT(sourceMap); + auto sourceMapBox = asBoxValue<SourceMap>(castable); + SLANG_ASSERT(sourceMapBox); // TODO(JS): // There is perhaps (?) a risk here that we might copy the obfuscated map @@ -7175,7 +7204,7 @@ SlangResult _addLibraryReference( // unit(s). for (auto module : library->m_modules) { - module->getIRModule()->setObfuscatedSourceMap(sourceMap); + module->getIRModule()->setObfuscatedSourceMap(sourceMapBox); } // Look up the source file @@ -7185,8 +7214,26 @@ SlangResult _addLibraryReference( if (name.getLength()) { + // Note(tfoley): There is a subtle requirement here, that any + // source file `name` that might be searched for here *must* + // have been added to the `sourceManager` already, as a + // byproduct of debug source location information getting + // deserialized as part of the call to `loadModuleLibrary()` above. + // + // The implicit dependency is frustrating, and could potentially + // break if somehow the debug info chunk was stripped from a binary, + // while the source map was left in (which should be valid, even if + // it is unlikely to be what a user wants). + // + // Ideally the source map would either be made an integral part of + // the debug source location chunk, so they are loaded together, + // or the `SourceManager` would be adapted so that it can store + // registered source maps independent of whether or not the + // corresponding source file(s) have been loaded. + auto sourceFile = sourceManager->findSourceFileByPathRecursively(name); - sourceFile->setSourceMap(sourceMap, SourceMapKind::Obfuscated); + SLANG_ASSERT(sourceFile); + sourceFile->setSourceMap(sourceMapBox, SourceMapKind::Obfuscated); } } } diff --git a/tools/gfx-unit-test/copy-texture-tests.cpp b/tools/gfx-unit-test/copy-texture-tests.cpp index f6f85f9f4..02df67378 100644 --- a/tools/gfx-unit-test/copy-texture-tests.cpp +++ b/tools/gfx-unit-test/copy-texture-tests.cpp @@ -37,7 +37,7 @@ struct BaseCopyTextureTest IDevice* device; UnitTestContext* context; - Size alignedRowStride; + gfx::Size alignedRowStride; RefPtr<TextureInfo> srcTextureInfo; RefPtr<TextureInfo> dstTextureInfo; diff --git a/tools/gfx-unit-test/gfx-test-texture-util.cpp b/tools/gfx-unit-test/gfx-test-texture-util.cpp index 7e6d738ee..a0212b7b0 100644 --- a/tools/gfx-unit-test/gfx-test-texture-util.cpp +++ b/tools/gfx-unit-test/gfx-test-texture-util.cpp @@ -42,7 +42,7 @@ TextureAspect getTextureAspect(Format format) } } -Size getTexelSize(Format format) +gfx::Size getTexelSize(Format format) { FormatInfo info; GFX_CHECK_CALL_ABORT(gfxGetFormatInfo(format, &info)); @@ -285,8 +285,8 @@ List<uint8_t> removePadding( ISlangBlob* pixels, GfxCount width, GfxCount height, - Size rowPitch, - Size pixelSize) + gfx::Size rowPitch, + gfx::Size pixelSize) { List<uint8_t> buffer; buffer.setCount(height * rowPitch); diff --git a/tools/gfx-unit-test/gfx-test-texture-util.h b/tools/gfx-unit-test/gfx-test-texture-util.h index 6a98eec43..d084628e2 100644 --- a/tools/gfx-unit-test/gfx-test-texture-util.h +++ b/tools/gfx-unit-test/gfx-test-texture-util.h @@ -12,9 +12,9 @@ namespace gfx_test { struct Strides { - Size x; - Size y; - Size z; + gfx::Size x; + gfx::Size y; + gfx::Size z; }; struct ValidationTextureFormatBase : RefObject @@ -194,7 +194,7 @@ struct TextureInfo : RefObject }; TextureAspect getTextureAspect(Format format); -Size getTexelSize(Format format); +gfx::Size getTexelSize(Format format); GfxIndex getSubresourceIndex(GfxIndex mipLevel, GfxCount mipLevelCount, GfxIndex baseArrayLayer); RefPtr<ValidationTextureFormatBase> getValidationTextureFormat(Format format); void generateTextureData( @@ -205,8 +205,8 @@ List<uint8_t> removePadding( ISlangBlob* pixels, GfxCount width, GfxCount height, - Size rowPitch, - Size pixelSize); + gfx::Size rowPitch, + gfx::Size pixelSize); Slang::Result writeImage(const char* filename, ISlangBlob* pixels, uint32_t width, uint32_t height); Slang::Result writeImage( const char* filename, diff --git a/tools/render-test/render-test-main.cpp b/tools/render-test/render-test-main.cpp index 45b39db67..81b937a2a 100644 --- a/tools/render-test/render-test-main.cpp +++ b/tools/render-test/render-test-main.cpp @@ -838,7 +838,7 @@ void RenderTestApp::_initializeAccelerationStructure() { AccelerationStructureInstanceDescType nativeInstanceDescType = getAccelerationStructureInstanceDescType(m_device); - Size nativeInstanceDescSize = + rhi::Size nativeInstanceDescSize = getAccelerationStructureInstanceDescSize(nativeInstanceDescType); List<AccelerationStructureInstanceDescGeneric> genericInstanceDescs; diff --git a/tools/slang-unit-test/unit-test-riff.cpp b/tools/slang-unit-test/unit-test-riff.cpp index 5f8b448f9..71856c789 100644 --- a/tools/slang-unit-test/unit-test-riff.cpp +++ b/tools/slang-unit-test/unit-test-riff.cpp @@ -9,7 +9,7 @@ using namespace Slang; static void _writeRandom( RandomGenerator* rand, size_t maxSize, - RiffContainer& ioContainer, + RIFF::BuildCursor& cursor, List<uint8_t>& ioData) { while (true) @@ -27,84 +27,257 @@ static void _writeRandom( rand->nextData(ioData.getBuffer() + oldCount, allocSize); // Write - ioContainer.write(ioData.getBuffer() + oldCount, allocSize); + cursor.addData(ioData.getBuffer() + oldCount, allocSize); } // Should be a single block with same data as the List - RiffContainer::DataChunk* dataChunk = - as<RiffContainer::DataChunk>(ioContainer.getCurrentChunk()); + auto dataChunk = as<RIFF::DataChunkBuilder>(cursor.getCurrentChunk()); SLANG_ASSERT(dataChunk); } -SLANG_UNIT_TEST(riff) +namespace +{ +struct DumpContext +{ +private: + WriterHelper _writer; + Count _indent = 0; + Count _hexByteCount = 0; + bool _isRoot = true; + +public: + DumpContext(ISlangWriter* writer) + : _writer(writer) + { + } + + void beginListChunk(RIFF::Chunk::Type type) + { + _dumpIndent(); + // If it's the root it's 'riff' + _dumpRiffType(_isRoot ? RIFF::RootChunk::kTag : RIFF::ListChunk::kTag); + _writer.put(" "); + _dumpRiffType(type); + _writer.put("\n"); + _indent++; + } + + void endListChunk() { _indent--; } + + void beginDataChunk(RIFF::Chunk::Type type) + { + _dumpIndent(); + // Write out the name + _dumpRiffType(type); + _writer.put("\n"); + _indent++; + + _hexByteCount = 0; + } + + void endDataChunk() { _indent--; } + + void handleData(void const* data, Size size) + { + auto cursor = static_cast<Byte const*>(data); + auto remainingSize = size; + while (remainingSize--) + { + auto byte = *cursor++; + + static const Count kBytesPerLine = 32; + static const Count kBytesPerCluster = 4; + if (_hexByteCount % kBytesPerLine == 0) + { + _writer.put("\n"); + _dumpIndent(); + } + else if (_hexByteCount % kBytesPerCluster == 0) + { + _writer.put(" "); + } + _hexByteCount++; + + char text[4] = {0, 0, ' ', 0}; + + char const* hexDigits = "0123456789abcdef"; + text[0] = hexDigits[(byte >> 4) & 0xF]; + text[1] = hexDigits[(byte >> 0) & 0xF]; + + _writer.put(text); + } + } + + void _dumpIndent() + { + for (int i = 0; i < _indent; ++i) + { + _writer.put(" "); + } + } + void _dumpRiffType(FourCC fourCC) + { + auto rawValue = FourCC::RawValue(fourCC); + + char text[5]; + for (int i = 0; i < 4; ++i) + { + text[i] = char(rawValue & 0xFF); + rawValue >>= 8; + } + text[4] = 0; + _writer.put(text); + } +}; + +} // namespace + +static void _dump(RIFF::Chunk const* chunk, DumpContext context) +{ + if (auto listChunk = as<RIFF::ListChunk>(chunk)) + { + context.beginListChunk(listChunk->getType()); + for (auto child : listChunk->getChildren()) + _dump(child, context); + context.endListChunk(); + } + else if (auto dataChunk = as<RIFF::DataChunk>(chunk)) + { + context.beginDataChunk(dataChunk->getType()); + context.handleData(dataChunk->getPayload(), dataChunk->getPayloadSize()); + context.endDataChunk(); + } +} + +static void _dump(RIFF::ChunkBuilder* chunk, DumpContext context) +{ + if (auto listChunk = as<RIFF::ListChunkBuilder>(chunk)) + { + context.beginListChunk(listChunk->getType()); + for (auto child : listChunk->getChildren()) + _dump(child, context); + context.endListChunk(); + } + else if (auto dataChunk = as<RIFF::DataChunkBuilder>(chunk)) + { + context.beginDataChunk(dataChunk->getType()); + for (auto shard : dataChunk->getShards()) + context.handleData(shard->getPayload(), shard->getPayloadSize()); + context.endDataChunk(); + } +} + +static bool _isSingleShard(RIFF::DataChunkBuilder* chunk) +{ + Count count = 0; + for (auto shard : chunk->getShards()) + { + count++; + if (count > 1) + break; + } + return count == 1; +} + +static bool _isEqual(RIFF::DataChunkBuilder* chunk, void const* data, Size size) { - typedef RiffContainer::ScopeChunk ScopeChunk; - typedef RiffContainer::Chunk::Kind Kind; + auto remainingData = static_cast<Byte const*>(data); + auto remainingSize = size; + + for (auto shard : chunk->getShards()) + { + // If there is more content in the chunk than remains + // to compare against, then there is no chance of a match. + // + auto shardSize = shard->getPayloadSize(); + if (shard->getPayloadSize() > remainingSize) + { + return false; + } + // Contents must match, byte-for-byte. + // + if (::memcmp(remainingData, shard->getPayload(), shardSize) != 0) + { + return false; + } + + remainingData += shardSize; + remainingSize -= shardSize; + } + + // If we reach the end of the chunk, then we have + // a match if there is no data remaining to + // compare against. + // + return remainingSize == 0; +} + +SLANG_UNIT_TEST(riff) +{ const FourCC markThings = SLANG_FOUR_CC('T', 'H', 'I', 'N'); const FourCC markData = SLANG_FOUR_CC('D', 'A', 'T', 'A'); { - RiffContainer container; + RIFF::Builder riffBuilder; + RIFF::BuildCursor cursor(riffBuilder); { - ScopeChunk scopeContainer(&container, Kind::List, markThings); + SLANG_SCOPED_RIFF_BUILDER_LIST_CHUNK(cursor, markThings); { - ScopeChunk scopeChunk(&container, Kind::Data, markData); + SLANG_SCOPED_RIFF_BUILDER_DATA_CHUNK(cursor, markData); const char hello[] = "Hello "; const char world[] = "World!"; - container.write(hello, sizeof(hello)); - container.write(world, sizeof(world)); + cursor.addData(hello, sizeof(hello)); + cursor.addData(world, sizeof(world)); } { - ScopeChunk scopeChunk(&container, Kind::Data, markData); + SLANG_SCOPED_RIFF_BUILDER_DATA_CHUNK(cursor, markData); const char test0[] = "Testing... "; const char test1[] = "Testing!"; - container.write(test0, sizeof(test0)); - container.write(test1, sizeof(test1)); + cursor.addData(test0, sizeof(test0)); + cursor.addData(test1, sizeof(test1)); } { - ScopeChunk innerScopeContainer(&container, Kind::List, markThings); + SLANG_SCOPED_RIFF_BUILDER_LIST_CHUNK(cursor, markThings); { - ScopeChunk scopeChunk(&container, Kind::Data, markData); + SLANG_SCOPED_RIFF_BUILDER_DATA_CHUNK(cursor, markData); const char another[] = "Another?"; - container.write(another, sizeof(another)); + cursor.addData(another, sizeof(another)); } } } - SLANG_CHECK(container.isFullyConstructed()); - SLANG_CHECK(RiffContainer::isChunkOk(container.getRoot())); + SLANG_CHECK(cursor.getCurrentChunk() == nullptr); + SLANG_CHECK(riffBuilder.getRootChunk() != nullptr); { StringBuilder builder; { - StringWriter writer(&builder, 0); - RiffUtil::dump(container.getRoot(), &writer); + StringWriter writer(&builder); + _dump(riffBuilder.getRootChunk(), &writer); } { - OwnedMemoryStream stream(FileAccess::ReadWrite); - SLANG_CHECK(SLANG_SUCCEEDED(RiffUtil::write(container.getRoot(), true, &stream))); - - stream.seek(SeekOrigin::Start, 0); + ComPtr<ISlangBlob> blob; + SLANG_CHECK(SLANG_SUCCEEDED(riffBuilder.writeToBlob(blob.writeRef()))); - RiffContainer readContainer; - SLANG_CHECK(SLANG_SUCCEEDED(RiffUtil::read(&stream, readContainer))); + auto rootChunk = RIFF::RootChunk::getFromBlob(blob); + SLANG_CHECK(rootChunk != nullptr); // Dump the read contents StringBuilder readBuilder; { StringWriter writer(&readBuilder, 0); - RiffUtil::dump(readContainer.getRoot(), &writer); + _dump(rootChunk, &writer); } // They should be the same @@ -116,29 +289,27 @@ SLANG_UNIT_TEST(riff) // Test writing as a stream only allocates a single data block (as long as there is enough // space). { - RiffContainer container; + RIFF::Builder builder; + RIFF::BuildCursor cursor(builder); - ScopeChunk scopeChunk(&container, Kind::List, markData); + SLANG_SCOPED_RIFF_BUILDER_LIST_CHUNK(cursor, markThings); { - ScopeChunk scopeChunk(&container, Kind::Data, markData); + SLANG_SCOPED_RIFF_BUILDER_DATA_CHUNK(cursor, markData); + RefPtr<RandomGenerator> rand = RandomGenerator::create(0x345234); List<uint8_t> data; - _writeRandom( - rand, - container.getMemoryArena().getBlockPayloadSize() / 2, - container, - data); + _writeRandom(rand, builder._getMemoryArena().getBlockPayloadSize() / 2, cursor, data); // Should be a single block with same data as the List - RiffContainer::DataChunk* dataChunk = - as<RiffContainer::DataChunk>(container.getCurrentChunk()); + RIFF::DataChunkBuilder* dataChunk = + as<RIFF::DataChunkBuilder>(cursor.getCurrentChunk()); SLANG_ASSERT(dataChunk); - // It should be a single block - SLANG_CHECK(dataChunk->getSingleData() != nullptr); + // It should be a single shard + SLANG_CHECK(_isSingleShard(dataChunk)); - SLANG_CHECK(dataChunk->isEqual(data.getBuffer(), data.getCount())); + SLANG_CHECK(_isEqual(dataChunk, data.getBuffer(), data.getCount())); } } @@ -148,23 +319,24 @@ SLANG_UNIT_TEST(riff) for (Int i = 0; i < 100; ++i) { - RiffContainer container; + RIFF::Builder builder; + RIFF::BuildCursor cursor(builder); const size_t maxSize = rand->nextInt32InRange( 1, - int32_t(container.getMemoryArena().getBlockPayloadSize() * 3)); + int32_t(builder._getMemoryArena().getBlockPayloadSize() * 3)); - ScopeChunk scopeChunk(&container, Kind::List, markData); + SLANG_SCOPED_RIFF_BUILDER_LIST_CHUNK(cursor, markThings); { - ScopeChunk scopeChunk(&container, Kind::Data, markData); + SLANG_SCOPED_RIFF_BUILDER_DATA_CHUNK(cursor, markData); List<uint8_t> data; - _writeRandom(rand, maxSize, container, data); + _writeRandom(rand, maxSize, cursor, data); // Should be a single block with same data as the List - RiffContainer::DataChunk* dataChunk = - as<RiffContainer::DataChunk>(container.getCurrentChunk()); - SLANG_CHECK(dataChunk && dataChunk->isEqual(data.getBuffer(), data.getCount())); + RIFF::DataChunkBuilder* dataChunk = + as<RIFF::DataChunkBuilder>(cursor.getCurrentChunk()); + SLANG_CHECK(dataChunk && _isEqual(dataChunk, data.getBuffer(), data.getCount())); } } } |