From c27b7d91aaf6bc764807a8998a9c885e57c22a1b Mon Sep 17 00:00:00 2001 From: jsmall-nvidia Date: Tue, 29 Oct 2019 08:51:24 -0400 Subject: Feature/container format (#1098) * WIP RiffContainer. * WIP riff container. * Testing out RiffContainer. * * Naming improvements * Visitor functions * Ability to dump riffs. * Renamed RiffChunk to RiffHeader * Remove m_ prefix on RiffHeader members. * Riff stream reading writing. Simple test of riff reading/writing. * Fix Riff alignment issue. Make IR serialization use the RiffContainer API. * Improve documentation. * Remove SubChunk fuctionality as not needed with RiffContainer. --- source/core/slang-hex-dump-util.cpp | 15 +- source/core/slang-hex-dump-util.h | 3 + source/core/slang-riff.cpp | 772 +++++++++++++++++++++++++++++++-- source/core/slang-riff.h | 286 +++++++++++- source/core/slang-stream.cpp | 33 +- source/core/slang-stream.h | 14 +- source/slang/slang-compiler.cpp | 56 +++ source/slang/slang-compiler.h | 2 +- source/slang/slang-diagnostic-defs.h | 1 + source/slang/slang-ir-serialize.cpp | 499 +++++++-------------- source/slang/slang-ir-serialize.h | 17 +- source/slang/slang-state-serialize.cpp | 4 +- source/slang/slang-state-serialize.h | 2 +- 13 files changed, 1274 insertions(+), 430 deletions(-) (limited to 'source') diff --git a/source/core/slang-hex-dump-util.cpp b/source/core/slang-hex-dump-util.cpp index 908f6c1b6..4a3d4ce06 100644 --- a/source/core/slang-hex-dump-util.cpp +++ b/source/core/slang-hex-dump-util.cpp @@ -12,6 +12,7 @@ namespace Slang static const UnownedStringSlice s_start = UnownedStringSlice::fromLiteral("--START--"); static const UnownedStringSlice s_end = UnownedStringSlice::fromLiteral("--END--"); +static const char s_hex[] = "0123456789abcdef"; /* static */SlangResult HexDumpUtil::dumpWithMarkers(const List& data, int maxBytesPerLine, ISlangWriter* writer) { @@ -26,6 +27,17 @@ static const UnownedStringSlice s_end = UnownedStringSlice::fromLiteral("--END-- return SLANG_OK; } +/* static */void HexDumpUtil::dump(uint32_t value, ISlangWriter* writer) +{ + char c[9]; + for (int i = 0; i < 8; ++i) + { + c[i] = s_hex[value >> 28]; + value <<= 4; + } + writer->write(c, 8); +} + /* static */SlangResult HexDumpUtil::dump(const List& data, int maxBytesPerLine, ISlangWriter* writer) { int maxCharsPerLine = 2 * maxBytesPerLine + 1 + maxBytesPerLine + 1; @@ -33,8 +45,7 @@ static const UnownedStringSlice s_end = UnownedStringSlice::fromLiteral("--END-- const uint8_t* cur = data.begin(); const uint8_t* end = data.end(); - static char s_hex[] = "0123456789abcdef"; - + while (cur < end) { size_t count = size_t(end - cur); diff --git a/source/core/slang-hex-dump-util.h b/source/core/slang-hex-dump-util.h index 580f5a120..fefe84a17 100644 --- a/source/core/slang-hex-dump-util.h +++ b/source/core/slang-hex-dump-util.h @@ -15,6 +15,9 @@ struct HexDumpUtil /// Dump data to writer, with lines starting with hex data static SlangResult dump(const List& data, int numBytesPerLine, ISlangWriter* writer); + /// Dump a single value + static void dump(uint32_t value, ISlangWriter* writer); + static SlangResult dumpWithMarkers(const List& data, int numBytesPerLine, ISlangWriter* writer); /// Parses lines formatted by dump, back into bytes diff --git a/source/core/slang-riff.cpp b/source/core/slang-riff.cpp index 0b0519b48..9849cfc26 100644 --- a/source/core/slang-riff.cpp +++ b/source/core/slang-riff.cpp @@ -2,16 +2,18 @@ #include "../../slang-com-helper.h" +#include "slang-hex-dump-util.h" + namespace Slang { -/* static */int64_t RiffUtil::calcChunkTotalSize(const RiffChunk& chunk) +/* static */int64_t RiffUtil::calcChunkTotalSize(const RiffHeader& chunk) { - int64_t size = chunk.m_size + sizeof(RiffChunk); - return (size + 3) & ~int64_t(3); + size_t size = chunk.size + sizeof(RiffHeader); + return getPadSize(size); } -/* static */SlangResult RiffUtil::skip(const RiffChunk& chunk, Stream* stream, int64_t* remainingBytesInOut) +/* static */SlangResult RiffUtil::skip(const RiffHeader& chunk, Stream* stream, int64_t* remainingBytesInOut) { int64_t chunkSize = calcChunkTotalSize(chunk); if (remainingBytesInOut) @@ -20,16 +22,15 @@ namespace Slang } // Skip the payload (we don't need to skip the Chunk because that was already read - stream->seek(SeekOrigin::Current, chunkSize - sizeof(RiffChunk)); + stream->seek(SeekOrigin::Current, chunkSize - sizeof(RiffHeader)); return SLANG_OK; } - -/* static */SlangResult RiffUtil::readChunk(Stream* stream, RiffChunk& outChunk) +/* static */SlangResult RiffUtil::readChunk(Stream* stream, RiffHeader& outChunk) { try { - stream->read(&outChunk, sizeof(RiffChunk)); + stream->read(&outChunk, sizeof(RiffHeader)); } catch (IOException&) { @@ -42,32 +43,38 @@ namespace Slang } -/* static */SlangResult RiffUtil::writeData(const RiffChunk* header, size_t headerSize, const void* payload, size_t payloadSize, Stream* out) +/* static */SlangResult RiffUtil::writeData(const RiffHeader* header, size_t headerSize, const void* payload, size_t payloadSize, Stream* out) { SLANG_ASSERT(uint64_t(payloadSize) <= uint64_t(0xfffffffff)); - SLANG_ASSERT(headerSize >= sizeof(RiffChunk)); - SLANG_ASSERT((headerSize & 3) == 0); + SLANG_ASSERT(headerSize >= sizeof(RiffHeader)); // TODO(JS): Could handle endianness here - RiffChunk chunk; - chunk.m_type = header->m_type; - chunk.m_size = uint32_t(headerSize - sizeof(RiffChunk) + payloadSize); + RiffHeader chunk; + chunk.type = header->type; + chunk.size = uint32_t(headerSize - sizeof(RiffHeader) + payloadSize); try { // The chunk - out->write(&chunk, sizeof(RiffChunk)); - // The rest of the header - out->write(header + 1, headerSize - sizeof(RiffChunk)); + out->write(&chunk, sizeof(RiffHeader)); + + // Remainder of header + if (headerSize > sizeof(RiffHeader)) + { + // The rest of the header + out->write(header + 1, headerSize - sizeof(RiffHeader)); + } + // Write the payload out->write(payload, payloadSize); - size_t remaining = payloadSize & 3; - if (remaining) + // The riff spec requires all chunks are 4 byte aligned (even if size is not) + size_t padSize = getPadSize(payloadSize); + if (padSize - payloadSize) { - uint8_t end[4] = { 0, 0, 0, 0}; - out->write(end, 4 - remaining); + uint8_t end[kRiffPadSize] = { 0 }; + out->write(end, padSize - payloadSize); } } catch (IOException&) @@ -78,11 +85,33 @@ namespace Slang return SLANG_OK; } -/* static */SlangResult RiffUtil::readData(Stream* stream, RiffChunk* outHeader, size_t headerSize, List& data) +/* static */SlangResult RiffUtil::readPayload(Stream* stream, size_t size, void* outData, size_t& outReadSize) { - RiffChunk chunk; + outReadSize = 0; + try + { + stream->read(outData, size); + const size_t alignedSize = getPadSize(size); + // Skip to the alignment + if (alignedSize > size) + { + stream->seek(SeekOrigin::Current, alignedSize - size); + } + outReadSize = alignedSize; + } + catch (IOException&) + { + return SLANG_FAIL; + } + + return SLANG_OK; +} + +/* static */SlangResult RiffUtil::readData(Stream* stream, RiffHeader* outHeader, size_t headerSize, List& data) +{ + RiffHeader chunk; SLANG_RETURN_ON_FAIL(readChunk(stream, chunk)); - if (chunk.m_size < headerSize) + if (chunk.size < headerSize) { return SLANG_FAIL; } @@ -92,27 +121,702 @@ namespace Slang try { // Read the header - stream->read(outHeader + 1, headerSize - sizeof(RiffChunk)); + if (headerSize > sizeof(RiffHeader)) + { + stream->read(outHeader + 1, headerSize - sizeof(RiffHeader)); + } + } + catch (IOException&) + { + return SLANG_FAIL; + } - const size_t payloadSize = chunk.m_size - (headerSize - sizeof(RiffChunk)); + const size_t payloadSize = chunk.size - (headerSize - sizeof(RiffHeader)); + size_t readSize; + data.setCount(payloadSize); + return readPayload(stream, payloadSize, data.getBuffer(), readSize); +} - data.setCount(payloadSize); +/* static */SlangResult RiffUtil::readHeader(Stream* stream, RiffListHeader& outHeader) +{ + // Need to read the chunk header + SLANG_RETURN_ON_FAIL(readChunk(stream, outHeader.chunk)); + outHeader.subType = 0; - stream->read(data.getBuffer(), payloadSize); + if (isListType(outHeader.chunk.type)) + { + // Read the sub type + try + { + stream->read(&outHeader.subType, sizeof(RiffListHeader) - sizeof(RiffHeader)); + } + catch (const IOException&) + { + return SLANG_FAIL; + } + } - // Skip to the alignment - uint32_t remaining = payloadSize & 3; - if (remaining) + return SLANG_OK; +} + +namespace { // anonymous + +struct DumpVisitor : public RiffContainer::Visitor +{ + typedef RiffContainer::Chunk Chunk; + typedef RiffContainer::ListChunk ListChunk; + typedef RiffContainer::DataChunk DataChunk; + + + // Visitor + virtual SlangResult enterList(ListChunk* list) SLANG_OVERRIDE + { + _dumpIndent(); + // If it's the root it's 'riff' + _dumpRiffType(list == m_rootChunk ? RiffFourCC::kRiff : RiffFourCC::kList); + m_writer.put(" "); + _dumpRiffType(list->m_subType); + m_writer.put("\n"); + m_indent++; + return SLANG_OK; + } + virtual SlangResult handleData(DataChunk* data) SLANG_OVERRIDE + { + _dumpIndent(); + // Write out the name + _dumpRiffType(data->m_type); + m_writer.put(" "); + + + int hash = data->calcHash(); + + // We don't know in general what the contents is or means... but we can display a hash + HexDumpUtil::dump(uint32_t(hash), m_writer.getWriter()); + m_writer.put(" "); + + m_writer.put("\n"); + return SLANG_OK; + } + virtual SlangResult leaveList(ListChunk* list) SLANG_OVERRIDE + { + SLANG_UNUSED(list); + m_indent--; + return SLANG_OK; + } + + DumpVisitor(WriterHelper writer, Chunk* rootChunk) : + m_writer(writer), + m_indent(0), + m_rootChunk(rootChunk) + { + } + + void _dumpIndent() + { + for (int i = 0; i < m_indent; ++i) { - stream->seek(SeekOrigin::Current, 4 - remaining); + m_writer.put(" "); } } - catch (IOException&) + void _dumpRiffType(FourCC fourCC) { - return SLANG_FAIL; + char c[5]; + for (int i = 0; i < 4; ++i) + { + c[i] = char(fourCC); + fourCC >>= 8; + } + c[4] = 0; + m_writer.put(c); + } + + Chunk* m_rootChunk; + + int m_indent; + WriterHelper m_writer; +}; + +} + +/* static */void RiffUtil::dump(RiffContainer::Chunk* chunk, WriterHelper writer) +{ + DumpVisitor visitor(writer, chunk); + chunk->visit(&visitor); +} + +/* static */SlangResult RiffUtil::write(RiffContainer::ListChunk* list, bool isRoot, Stream* stream) +{ + RiffListHeader listHeader; + + listHeader.chunk.type = isRoot ? RiffFourCC::kRiff : RiffFourCC::kList; + listHeader.chunk.size = uint32_t(list->m_payloadSize); + listHeader.subType = list->m_subType; + + try + { + // Write the header + stream->write(&listHeader, sizeof(listHeader)); + + // Write the contained chunks + Chunk* chunk = list->m_containedChunks; + while (chunk) + { + switch (chunk->m_kind) + { + case Chunk::Kind::List: + { + auto listChunk = static_cast(chunk); + // It's a container + SLANG_RETURN_ON_FAIL(write(listChunk, false, stream)); + break; + } + case Chunk::Kind::Data: + { + auto dataChunk = static_cast(chunk); + + // Must be a regular chunk with data + RiffHeader chunkHeader; + chunkHeader.type = dataChunk->m_type; + chunkHeader.size = uint32_t(dataChunk->m_payloadSize); + + stream->write(&chunkHeader, sizeof(chunkHeader)); + + RiffContainer::Data* data = dataChunk->m_dataList; + while (data) + { + stream->write(data->getPayload(), data->getSize()); + + // Next but of data + data = data->m_next; + } + + // Need to write for alignment + const size_t remainingSize = getPadSize(dataChunk->m_payloadSize) - dataChunk->m_payloadSize; + + if (remainingSize) + { + static const uint8_t trailing[kRiffPadSize] = { 0 }; + stream->write(trailing, remainingSize); + } + } + default: break; + } + + // Next + chunk = chunk->m_next; + } + } + catch (const IOException&) + { + return SLANG_FAIL; } return SLANG_OK; } +/* static */SlangResult RiffUtil::read(Stream* stream, RiffContainer& outContainer) +{ + typedef RiffContainer::ScopeChunk ScopeChunk; + typedef RiffContainer::ScopeChunk ScopeContainer; + outContainer.reset(); + + size_t remaining; + { + RiffListHeader header; + + SLANG_RETURN_ON_FAIL(readHeader(stream, header)); + if (!isListType(header.chunk.type)) + { + return SLANG_FAIL; + } + + remaining = getPadSize(header.chunk.size) - (sizeof(RiffListHeader) - sizeof(RiffHeader)); + outContainer.startChunk(Chunk::Kind::List, header.subType); + } + + List remainingStack; + while (true) + { + // It must be the end + if (remaining == 0) + { + // If it's a container then we pop container + outContainer.endChunk(); + if (remainingStack.getCount() <= 0) + { + break; + } + + remaining = remainingStack.getLast(); + remainingStack.removeLast(); + } + else + { + RiffListHeader header; + SLANG_RETURN_ON_FAIL(readHeader(stream, header)); + + // The amount of data can't be larger than what remains + if (header.chunk.size > remaining) + { + return SLANG_FAIL; + } + + if (header.chunk.type == RiffFourCC::kList) + { + if (header.chunk.size & kRiffPadMask) + { + SLANG_ASSERT(!"A list chunk can only have divisible by 2 size"); + return SLANG_FAIL; + } + + // Work out the pad size + const size_t padSize = getPadSize(header.chunk.size); + + // Subtract the size of this chunk from remaining of the current chunk + remaining -= sizeof(RiffHeader) + padSize; + // Push it, for when we hit the end + remainingStack.add(remaining); + + // Work out how much remains in this container + remaining = padSize - (sizeof(RiffListHeader) - sizeof(RiffHeader)); + + // Start a container + outContainer.startChunk(Chunk::Kind::List, header.subType); + } + else + { + ScopeChunk scopeChunk(&outContainer, Chunk::Kind::Data, header.chunk.type); + RiffContainer::Data* data = outContainer.addData(header.chunk.size); + + size_t readSize; + SLANG_RETURN_ON_FAIL(readPayload(stream, header.chunk.size, data->getPayload(), readSize)); + + // All read sizes must end up aligned + SLANG_ASSERT((readSize & kRiffPadMask) == 0); + + // Correct remaining + remaining -= sizeof(RiffHeader) + readSize; + } + } + } + + return outContainer.isFullyConstructed() ? SLANG_OK : SLANG_FAIL; +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! RiffContainer::Chunk !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +SlangResult RiffContainer::Chunk::visit(Visitor* visitor) +{ + switch (m_kind) + { + case Kind::Data: + { + return visitor->handleData(static_cast(this)); + } + case Kind::List: + { + auto list = static_cast(this); + SLANG_RETURN_ON_FAIL(visitor->enterList(list)); + + Chunk* chunk = list->m_containedChunks; + while (chunk) + { + SLANG_RETURN_ON_FAIL(chunk->visit(visitor)); + + chunk = chunk->m_next; + } + + SLANG_RETURN_ON_FAIL(visitor->leaveList(list)); + return SLANG_OK; + } + default: return SLANG_FAIL; + } +} + +SlangResult RiffContainer::Chunk::visitPreOrder(VisitorCallback callback, void* data) +{ + switch (m_kind) + { + case Kind::Data: + { + return callback(this, data); + } + case Kind::List: + { + auto list = static_cast(this); + // Do this containing node first + SLANG_RETURN_ON_FAIL(callback(this, data)); + + // Do the contents next + Chunk* chunk = list->m_containedChunks; + while (chunk) + { + SLANG_RETURN_ON_FAIL(chunk->visitPreOrder(callback, data)); + chunk = chunk->m_next; + } + return SLANG_OK; + } + default: return SLANG_FAIL; + } +} + +SlangResult RiffContainer::Chunk::visitPostOrder(VisitorCallback callback, void* data) +{ + switch (m_kind) + { + case Kind::Data: + { + return callback(this, data); + } + case Kind::List: + { + auto list = static_cast(this); + + // Do the contents first + Chunk* chunk = list->m_containedChunks; + while (chunk) + { + SLANG_RETURN_ON_FAIL(chunk->visitPostOrder(callback, data)); + chunk = chunk->m_next; + } + // Then the list node (so a post order) + SLANG_RETURN_ON_FAIL(callback(this, data)); + return SLANG_OK; + } + default: return SLANG_FAIL; + } +} + +size_t RiffContainer::Chunk::calcPayloadSize() +{ + switch (m_kind) + { + case Kind::Data: return static_cast(this)->calcPayloadSize(); + case Kind::List: return static_cast(this)->calcPayloadSize(); + default: return 0; + } +} + +RiffContainer::Data* RiffContainer::Chunk::getSingleData() const +{ + return (m_kind == Kind::Data) ? static_cast(this)->getSingleData(): nullptr; +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!! RiffContainer::ListChunk !!!!!!!!!!!!!!!!!!!!!! + +size_t RiffContainer::ListChunk::calcPayloadSize() +{ + // Have to include the part of the header not taken up by the RiffHeader + size_t size = sizeof(RiffListHeader) - sizeof(RiffHeader); + Chunk* chunk = m_containedChunks; + while (chunk) + { + size_t chunkSize = chunk->m_payloadSize + sizeof(RiffHeader); + // Align the contained chunk size + size += RiffUtil::getPadSize(chunkSize); + + chunk = chunk->m_next; + } + return size; +} + +RiffContainer::Chunk* RiffContainer::ListChunk::findContained(FourCC type) const +{ + Chunk* chunk = m_containedChunks; + while (chunk) + { + const FourCC checkType = (chunk->m_kind == Chunk::Kind::Data) ? + static_cast(chunk)->m_type : + static_cast(chunk)->m_subType; + if (checkType == type) + { + return chunk; + } + chunk = chunk->m_next; + } + return nullptr; +} + +RiffContainer::Data* RiffContainer::ListChunk::findContainedData(FourCC type) const +{ + Chunk* found = findContained(type); + if (found && found->m_kind == Kind::Data) + { + DataChunk* dataChunk = static_cast(found); + // Assumes that there is a single data chunk + + Data* data = dataChunk->m_dataList; + if (data && data->m_next == nullptr) + { + return data; + } + } + return nullptr; +} + +void* RiffContainer::ListChunk::findContainedData(FourCC type, size_t minSize) const +{ + Data* data = findContainedData(type); + return (data && data->m_size >= minSize) ? data->getPayload() : nullptr; +} + +static RiffContainer::ListChunk* _findListRec(RiffContainer::ListChunk* list, FourCC subType) +{ + RiffContainer::Chunk* chunk = list->m_containedChunks; + while (chunk) + { + if (auto childList = as(chunk)) + { + // Test if the child is the subtype, if so we are done + if (childList->m_subType == subType) + { + return childList; + } + auto found = _findListRec(childList, subType); + if (found) + { + return found; + } + } + chunk = chunk->m_next; + } + return nullptr; +} + +/* static */RiffContainer::ListChunk* RiffContainer::ListChunk::findListRec(FourCC subType) +{ + return (m_subType == subType) ? this : _findListRec(this, subType); +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!! RiffContainer::DataChunk !!!!!!!!!!!!!!!!!!!!!! + +RiffContainer::Data* RiffContainer::DataChunk::getSingleData() const +{ + if (m_kind == Kind::Data) + { + auto dataChunk = static_cast(this); + Data* data = dataChunk->m_dataList; + if (data && data->m_next == nullptr) + { + return data; + } + } + return nullptr; +} + +int RiffContainer::DataChunk::calcHash() const +{ + int hash = 0; + + Data* data = m_dataList; + while (data) + { + // This is a little contrived (in that we don't use the function GetHashCode), but the + // reason to be careful is we want the same result however many Data blocks there are. + const char* buffer = (const char*)data->getPayload(); + const size_t size = data->getSize(); + + for (size_t i = 0; i < size; ++i) + { + hash = int(buffer[i]) + (hash << 6) + (hash << 16) - hash; + } + + data = data->m_next; + } + + return hash; +} + +size_t RiffContainer::DataChunk::calcPayloadSize() const +{ + size_t size = 0; + Data* data = m_dataList; + while (data) + { + size += data->getSize(); + data = data->m_next; + } + return size; +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! RiffContainer !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +RiffContainer::RiffContainer() : + m_arena(4096) +{ + m_rootList = nullptr; + m_listChunk = nullptr; + m_dataChunk = nullptr; +} + +void RiffContainer::reset() +{ + m_arena.reset(); + + m_rootList = nullptr; + m_listChunk = nullptr; + m_dataChunk = nullptr; +} + +RiffContainer::ListChunk* RiffContainer::_newListChunk(FourCC subType) +{ + SLANG_ASSERT(!RiffUtil::isListType(subType)); + + ListChunk* chunk = (ListChunk*)m_arena.allocate(sizeof(ListChunk)); + chunk->init(subType); + return chunk; +} + +RiffContainer::DataChunk* RiffContainer::_newDataChunk(FourCC type) +{ + SLANG_ASSERT(!RiffUtil::isListType(type)); + + DataChunk* chunk = (DataChunk*)m_arena.allocate(sizeof(DataChunk)); + chunk->init(type); + return chunk; +} + +void RiffContainer::_addChunk(Chunk* chunk) +{ + if (m_listChunk) + { + chunk->m_parent = m_listChunk; + Chunk*& next = m_listChunk->m_endChunk ? m_listChunk->m_endChunk->m_next : m_listChunk->m_containedChunks; + SLANG_ASSERT(next == nullptr); + next = chunk; + m_listChunk->m_endChunk = chunk; + } +} + +void RiffContainer::startChunk(Chunk::Kind kind, FourCC fourCC) +{ + SLANG_ASSERT(m_listChunk || m_rootList == nullptr); + + switch (kind) + { + case Chunk::Kind::Data: + { + // We can only start a data chunk if we are in a container, and we can't already be in data chunk + SLANG_ASSERT(m_listChunk && m_dataChunk == nullptr); + + DataChunk* chunk = _newDataChunk(fourCC); + _addChunk(chunk); + m_dataChunk = chunk; + break; + } + case Chunk::Kind::List: + { + // We can't be in a data chunk + SLANG_ASSERT(m_dataChunk == nullptr); + + ListChunk* list = _newListChunk(fourCC); + + // If this is the first, make it the root + if (!m_rootList) + { + m_rootList = list; + } + + _addChunk(list); + + m_listChunk = list; + break; + } + } +} + +void RiffContainer::endChunk() +{ + size_t chunkPayloadSize; + + // The chunk we are popping + Chunk* chunk = nullptr; + + ListChunk* parent; + if (m_dataChunk) + { + chunk = m_dataChunk; + + parent = m_dataChunk->m_parent; + chunkPayloadSize = m_dataChunk->m_payloadSize; + + m_dataChunk = nullptr; + } + else + { + chunk = m_listChunk; + + SLANG_ASSERT(m_listChunk && m_dataChunk == nullptr); + parent = m_listChunk->m_parent; + chunkPayloadSize = m_listChunk->m_payloadSize; + } + + m_listChunk = parent; + + if (parent) + { + // Fix the size taking into account padding bytes requirement + chunkPayloadSize = RiffUtil::getPadSize(chunkPayloadSize); + // Update the parents size + parent->m_payloadSize += sizeof(RiffHeader) + chunkPayloadSize; + } + + // Check it's size seems ok + SLANG_ASSERT(isChunkOk(chunk)); +} + +RiffContainer::Data* RiffContainer::addData(size_t size) +{ + // We must be in a chunk + SLANG_ASSERT(m_dataChunk); + + // Add current chunks data + m_dataChunk->m_payloadSize += size; + + Data* data = (Data*)m_arena.allocate(sizeof(Data) + size); + + data->m_next = nullptr; + data->m_size = size; + + Data*& next = m_dataChunk->m_endData ? m_dataChunk->m_endData->m_next : m_dataChunk->m_dataList; + SLANG_ASSERT(next == nullptr); + + // Add to linked list + next = data; + // Make this the new end + m_dataChunk->m_endData = data; + return data; +} + +void RiffContainer::write(const void* inData, size_t size) +{ + auto data = addData(size); + ::memcpy(data->getPayload(), inData, size); +} + +static SlangResult _isChunkOk(RiffContainer::Chunk* chunk, void* data) +{ + SLANG_UNUSED(data); + return chunk->calcPayloadSize() == chunk->m_payloadSize ? SLANG_OK : SLANG_FAIL; +} + +/* static */bool RiffContainer::isChunkOk(Chunk* chunk) +{ + return SLANG_SUCCEEDED(chunk->visitPostOrder(&_isChunkOk, nullptr)); +} + +static SlangResult _calcAndSetSize(RiffContainer::Chunk* chunk, void* data) +{ + SLANG_UNUSED(data); + chunk->m_payloadSize = chunk->calcPayloadSize(); + return SLANG_OK; +} + +/* static */void RiffContainer::calcAndSetSize(Chunk* chunk) +{ + chunk->visitPostOrder(&_calcAndSetSize, nullptr); +} + + + } diff --git a/source/core/slang-riff.h b/source/core/slang-riff.h index f681d6d68..9dfe655b6 100644 --- a/source/core/slang-riff.h +++ b/source/core/slang-riff.h @@ -1,8 +1,10 @@ #ifndef SLANG_RIFF_H #define SLANG_RIFF_H -#include "../core/slang-basic.h" -#include "../core/slang-stream.h" +#include "slang-basic.h" +#include "slang-stream.h" +#include "slang-memory-arena.h" +#include "slang-writer.h" namespace Slang { @@ -10,11 +12,37 @@ namespace Slang // http://fileformats.archiveteam.org/wiki/RIFF // http://www.fileformat.info/format/riff/egff.htm -#define SLANG_FOUR_CC(c0, c1, c2, c3) ((uint32_t(c0) << 0) | (uint32_t(c1) << 8) | (uint32_t(c2) << 16) | (uint32_t(c3) << 24)) +typedef uint32_t FourCC; + +#define SLANG_FOUR_CC(c0, c1, c2, c3) FourCC((uint32_t(c0) << 0) | (uint32_t(c1) << 8) | (uint32_t(c2) << 16) | (uint32_t(c3) << 24)) + +enum +{ + kRiffPadSize = 2, ///< We only align to 2 bytes + kRiffPadMask = kRiffPadSize - 1, +}; + +struct RiffHeader +{ + FourCC type; ///< The FourCC code that identifies this chunk + uint32_t size; ///< Size does *NOT* include the riff chunk size. The size can be byte sized, but on storage it will always be treated as aligned up by 4. +}; + +struct RiffListHeader +{ + RiffHeader chunk; + FourCC subType; + // This is then followed by the contained subchunk/s +}; struct RiffFourCC { - static const uint32_t kRiff = SLANG_FOUR_CC('R', 'I', 'F', 'F'); + /// A 'riff' is the high level file container. It is followed by a subtype and then the contained chunks. + static const FourCC kRiff = SLANG_FOUR_CC('R', 'I', 'F', 'F'); + /// A list is the same as a 'riff' except can be placed anywhere in hierarchy. + static const FourCC kList = SLANG_FOUR_CC('L', 'I', 'S', 'T'); +private: + RiffFourCC() = delete; }; // Follows semantic version rules @@ -70,22 +98,256 @@ struct RiffSemanticVersion RawType m_raw; }; -struct RiffChunk +/* A container for data in RIFF format. Holds the contents in memory. + +With the data held in memory allows for adding or removing chunks at will. + +A future implementation does not necessarily have to be backed by memory when construction, +as data could be written to stream, and the chunk sizes written by seeking back over the file and setting the value. + +In normal usage the chunk sizes are calculated during construction. If the structure is changed, the sizes may +need to be recalculated, before serialization. +*/ +class RiffContainer { - uint32_t m_type; ///< The FourCC code that identifies this chunk - uint32_t m_size; ///< Size does *NOT* include the riff chunk size. The size can be byte sized, but on storage it will always be treated as aligned up by 4. +public: + + struct ListChunk; + struct DataChunk; + + struct Data + { + /// Get the payload + void* getPayload() { return (void*)(this + 1); } + size_t getSize() { return m_size; } + + void init() + { + m_size = 0; + m_next = nullptr; + } + + size_t m_size; + Data* m_next; + // Followed by the payload + }; + + struct Chunk; + typedef SlangResult(*VisitorCallback)(Chunk* chunk, void* data); + + class Visitor; + struct Chunk + { + enum class Kind + { + List, ///< Strictly speaking this can be a 'LIST' or a 'RIFF' as they have the same structure + Data, + }; + + void init(Kind kind) + { + m_kind = kind; + m_payloadSize = 0; + m_next = nullptr; + m_parent = nullptr; + } + + SlangResult visit(Visitor* visitor); + SlangResult visitPostOrder(VisitorCallback callback, void* data); + SlangResult visitPreOrder(VisitorCallback callback, void* data); + + /// Returns a single data chunk + Data* getSingleData() const; + + /// Get the unique type + FourCC getUniqueType() const; + + size_t calcPayloadSize(); + + // TODO(JS): + // We might want to consider moving subType/type to the Chunk, because in practice they typically + // mean the same thing, and as it has been arranged, for list chunk the overall chunk type is implied + // by the kind. + Kind m_kind; ///< Kind of chunk + size_t m_payloadSize; ///< The payload size (ie does NOT include RiffChunk header). + Chunk* m_next; ///< Next chunk in this list + ListChunk* m_parent; ///< The chunk this belongs to + }; + + struct ListChunk : public Chunk + { + typedef Chunk Super; + SLANG_FORCE_INLINE static bool isType(const Chunk* chunk) { return chunk->m_kind == Kind::List; } + + void init(FourCC subType) + { + Super::init(Kind::List); + m_subType = subType; + m_containedChunks = nullptr; + m_endChunk = nullptr; + + m_payloadSize = uint32_t(sizeof(RiffListHeader) - sizeof(RiffHeader)); + } + + /// Finds chunk (list or data) that matches type. For List/Riff, type is the subtype + Chunk* findContained(FourCC type) const; + + void* findContainedData(FourCC type, size_t minSize) const; + + /// Finds the contained data. NOTE! Assumes that there is only as single data block, and will return nullptr if there is not + Data* findContainedData(FourCC type) const; + + template + T* findContainedData(FourCC type) const { return (T*)findContainedData(type, sizeof(T)); } + + /// Find the list (including self) that matches subtype recursively + ListChunk* findListRec(FourCC subType); + + /// NOTE! Assumes all contained chunks have correct payload sizes + size_t calcPayloadSize(); + + FourCC m_subType; ///< The subtype of this contained + Chunk* m_containedChunks; ///< The contained chunks + Chunk* m_endChunk; ///< The last chunk (only set when pushed, and used when popped) + }; + + struct DataChunk : public Chunk + { + typedef Chunk Super; + SLANG_FORCE_INLINE static bool isType(const Chunk* chunk) { return chunk->m_kind == Kind::Data; } + + /// Calculate a hash (not necessarily very fast) + int calcHash() const; + /// Calculate the payload size + size_t calcPayloadSize() const; + + /// Get single data payload. + Data* getSingleData() const; + + void init(FourCC type) + { + Super::init(Kind::Data); + m_type = type; + m_dataList = nullptr; + m_endData = nullptr; + } + + FourCC m_type; ///< Type of chunk + Data* m_dataList; ///< List of 0 or more data items + Data* m_endData; ///< The last data point + }; + + class ScopeChunk + { + public: + ScopeChunk(RiffContainer* container, Chunk::Kind kind, FourCC fourCC) : + m_container(container) + { + container->startChunk(kind, fourCC); + } + ~ScopeChunk() + { + m_container->endChunk(); + } + private: + RiffContainer* m_container; + }; + + class Visitor + { + public: + virtual SlangResult enterList(ListChunk* list) = 0; + virtual SlangResult handleData(DataChunk* data) = 0; + virtual SlangResult leaveList(ListChunk* list) = 0; + }; + + /// Start a chunk + void startChunk(Chunk::Kind kind, FourCC type); + + /// Write data into a chunk (can only be inside a Kind::Data) + void write(const void* data, size_t size); + Data* addData(size_t size); + + /// End a chunk + void endChunk(); + + /// Get the root + ListChunk* getRoot() const { return m_rootList; } + + /// Reset the container + void reset(); + + /// true if has a root container, and nothing remains open + bool isFullyConstructed() { return m_rootList && m_listChunk == nullptr && m_dataChunk == nullptr; } + + /// The if the list and sublists appear correct + static bool isChunkOk(Chunk* chunk); + + /// Traverses over chunk hierarchy and sets the sizes + static void calcAndSetSize(Chunk* chunk); + + /// Ctor + RiffContainer(); + +protected: + void _addChunk(Chunk* chunk); + ListChunk* _newListChunk(FourCC subType); + DataChunk* _newDataChunk(FourCC type); + + ListChunk* m_rootList; ///< Root list + + ListChunk* m_listChunk; + DataChunk* m_dataChunk; + + MemoryArena m_arena; }; +// ----------------------------------------------------------------------------- +SLANG_FORCE_INLINE FourCC RiffContainer::Chunk::getUniqueType() const +{ + return (m_kind == Kind::Data) ? static_cast(this)->m_type : static_cast(this)->m_subType; +} +// ----------------------------------------------------------------------------- +template +T* as(RiffContainer::Chunk* chunk) +{ + return T::isType(chunk) ? static_cast(chunk) : nullptr; +} + + struct RiffUtil { - static int64_t calcChunkTotalSize(const RiffChunk& chunk); + typedef RiffContainer::Chunk Chunk; + typedef RiffContainer::ListChunk ListChunk; + typedef RiffContainer::DataChunk DataChunk; + + static int64_t calcChunkTotalSize(const RiffHeader& chunk); + + static SlangResult skip(const RiffHeader& chunk, Stream* stream, int64_t* remainingBytesInOut); + + static SlangResult readChunk(Stream* stream, RiffHeader& outChunk); + + static SlangResult writeData(const RiffHeader* header, size_t headerSize, const void* payload, size_t payloadSize, Stream* out); + static SlangResult readData(Stream* stream, RiffHeader* outHeader, size_t headerSize, List& data); + + static SlangResult readPayload(Stream* stream, size_t size, void* outData, size_t& outReadSize); + + /// Read a header. Handles special case of list/riff types + static SlangResult readHeader(Stream* stream, RiffListHeader& outHeader); + + /// True if the type is a container type + static bool isListType(FourCC type) { return type == RiffFourCC::kRiff || type == RiffFourCC::kList; } - static SlangResult skip(const RiffChunk& chunk, Stream* stream, int64_t* remainingBytesInOut); + /// Dump the chunk structure + static void dump(Chunk* chunk, WriterHelper writer); - static SlangResult readChunk(Stream* stream, RiffChunk& outChunk); + /// Get the size taking into account padding + static size_t getPadSize(size_t in) { return (in + kRiffPadMask) & ~size_t(kRiffPadMask); } - static SlangResult writeData(const RiffChunk* header, size_t headerSize, const void* payload, size_t payloadSize, Stream* out); - static SlangResult readData(Stream* stream, RiffChunk* outHeader, size_t headerSize, List& data); + /// Write a container and contents to a stream + static SlangResult write(ListChunk* container, bool isRoot, Stream* stream); + /// Read the stream into the container + static SlangResult read(Stream* stream, RiffContainer& outContainer); }; } diff --git a/source/core/slang-stream.cpp b/source/core/slang-stream.cpp index b5eef1d9b..bfbe43b0c 100644 --- a/source/core/slang-stream.cpp +++ b/source/core/slang-stream.cpp @@ -181,9 +181,10 @@ void FileStream::seek(SeekOrigin origin, Int64 offset) throw IOException("FileStream seek failed."); } } -Int64 FileStream::read(void* buffer, Int64 length) + +size_t FileStream::read(void* buffer, size_t length) { - auto bytes = fread_s(buffer, (size_t)length, 1, (size_t)length, m_handle); + auto bytes = fread_s(buffer, length, 1, length, m_handle); if (bytes == 0 && length > 0) { if (!feof(m_handle)) @@ -192,25 +193,29 @@ Int64 FileStream::read(void* buffer, Int64 length) throw EndOfStreamException("End of file is reached."); m_endReached = true; } - return (int)bytes; + return bytes; } -Int64 FileStream::write(const void* buffer, Int64 length) + +size_t FileStream::write(const void* buffer, size_t length) { - auto bytes = (Int64)fwrite(buffer, 1, (size_t)length, m_handle); + auto bytes = fwrite(buffer, 1, length, m_handle); if (bytes < length) { throw IOException("FileStream write failed."); } return bytes; } + bool FileStream::canRead() { return ((int)m_fileAccess & (int)FileAccess::Read) != 0; } + bool FileStream::canWrite() { return ((int)m_fileAccess & (int)FileAccess::Write) != 0; } + void FileStream::close() { if (m_handle) @@ -219,6 +224,7 @@ void FileStream::close() m_handle = 0; } } + bool FileStream::isEnd() { return m_endReached; @@ -251,18 +257,17 @@ void MemoryStreamBase::seek(SeekOrigin origin, Int64 offset) pos = (pos < 0) ? 0 : pos; pos = (pos > Int64(m_contentsSize)) ? Int64(m_contentsSize) : pos; - m_position = UInt(pos); + m_position = ptrdiff_t(pos); } -Int64 MemoryStreamBase::read(void* buffer, Int64 length) +size_t MemoryStreamBase::read(void* buffer, size_t length) { if (!canRead()) { throw IOException("Cannot read this stream."); } - const Int64 maxRead = Int64(m_contentsSize - m_position); - + const size_t maxRead = size_t(m_contentsSize - m_position); if (maxRead == 0 && length > 0) { m_atEnd = true; @@ -271,14 +276,14 @@ Int64 MemoryStreamBase::read(void* buffer, Int64 length) length = length > maxRead ? maxRead : length; - ::memcpy(buffer, m_contents + m_position, size_t(length)); - m_position += UInt(length); + ::memcpy(buffer, m_contents + m_position, length); + m_position += ptrdiff_t(length); return maxRead; } // !!!!!!!!!!!!!!!!!!!!!!!!!!!!! OwnedMemoryStream !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Int64 OwnedMemoryStream::write(const void * buffer, Int64 length) +size_t OwnedMemoryStream::write(const void * buffer, size_t length) { if (!canWrite()) { @@ -287,11 +292,11 @@ Int64 OwnedMemoryStream::write(const void * buffer, Int64 length) if (m_position == m_ownedContents.getCount()) { - m_ownedContents.addRange((const uint8_t*)buffer, UInt(length)); + m_ownedContents.addRange((const uint8_t*)buffer, Index(length)); } else { - m_ownedContents.insertRange(m_position, (const uint8_t*)buffer, UInt(length)); + m_ownedContents.insertRange(m_position, (const uint8_t*)buffer, Index(length)); } m_contents = m_ownedContents.getBuffer(); diff --git a/source/core/slang-stream.h b/source/core/slang-stream.h index 8b8a6decf..212437ab5 100644 --- a/source/core/slang-stream.h +++ b/source/core/slang-stream.h @@ -39,8 +39,8 @@ public: virtual ~Stream() {} virtual Int64 getPosition()=0; virtual void seek(SeekOrigin origin, Int64 offset)=0; - virtual Int64 read(void * buffer, Int64 length) = 0; - virtual Int64 write(const void * buffer, Int64 length) = 0; + virtual size_t read(void * buffer, size_t length) = 0; + virtual size_t write(const void * buffer, size_t length) = 0; virtual bool isEnd() = 0; virtual bool canRead() = 0; virtual bool canWrite() = 0; @@ -70,8 +70,8 @@ public: virtual Int64 getPosition() SLANG_OVERRIDE { return m_position; } virtual void seek(SeekOrigin origin, Int64 offset) SLANG_OVERRIDE; - virtual Int64 read(void * buffer, Int64 length) SLANG_OVERRIDE; - virtual Int64 write(const void * buffer, Int64 length) SLANG_OVERRIDE { SLANG_UNUSED(buffer); SLANG_UNUSED(length); return 0; } + virtual size_t read(void * buffer, size_t length) SLANG_OVERRIDE; + virtual size_t write(const void * buffer, size_t length) SLANG_OVERRIDE { SLANG_UNUSED(buffer); SLANG_UNUSED(length); return 0; } virtual bool isEnd() SLANG_OVERRIDE { return m_atEnd; } virtual bool canRead() SLANG_OVERRIDE { return (int(m_access) & int(FileAccess::Read)) != 0; } virtual bool canWrite() SLANG_OVERRIDE { return (int(m_access) & int(FileAccess::Write)) != 0; } @@ -117,7 +117,7 @@ class OwnedMemoryStream : public MemoryStreamBase public: typedef MemoryStreamBase Super; - virtual Int64 write(const void* buffer, Int64 length) SLANG_OVERRIDE; + virtual size_t write(const void* buffer, size_t length) SLANG_OVERRIDE; /// Set the contents void setContent(const void* contents, size_t contentsSize) @@ -150,8 +150,8 @@ public: // Stream interface virtual Int64 getPosition(); virtual void seek(SeekOrigin origin, Int64 offset); - virtual Int64 read(void* buffer, Int64 length); - virtual Int64 write(const void* buffer, Int64 length); + virtual size_t read(void* buffer, size_t length); + virtual size_t write(const void* buffer, size_t length); virtual bool canRead(); virtual bool canWrite(); virtual void close(); diff --git a/source/slang/slang-compiler.cpp b/source/slang/slang-compiler.cpp index 3fee24003..87a5e01a8 100644 --- a/source/slang/slang-compiler.cpp +++ b/source/slang/slang-compiler.cpp @@ -2183,7 +2183,58 @@ SlangResult dissassembleDXILUsingDXC( } } + static SlangResult _writeContainerFile( + EndToEndCompileRequest* endToEndReq, + Stream* stream) + { + SLANG_UNUSED(stream); + + auto linkage = endToEndReq->getLinkage(); + auto sink = endToEndReq->getSink(); + auto frontEndReq = endToEndReq->getFrontEndReq(); + + for (auto translationUnit : frontEndReq->translationUnits) + { + auto module = translationUnit->module; + auto irModule = module->getIRModule(); + + SLANG_UNUSED(irModule); + // Okay, we need to serialize this module to our container file, + // including both its name and generated IR code. + } + + auto program = endToEndReq->getSpecializedGlobalAndEntryPointsComponentType(); + + + + // TODO: in the case where we have specialization, we might need + // to serialize IR related to `program`... + + for (auto target : linkage->targets) + { + auto targetProgram = program->getTargetProgram(target); + auto irModule = targetProgram->getOrCreateIRModuleForLayout(sink); + + // Okay, we need to serialize this target program and its IR too... + } + + return SLANG_OK; + } + + /// Write out a "container" file with the stuff that has + /// been compiled as part of this request. + /// + static void _writeContainerFile( + EndToEndCompileRequest* endToEndReq, + const String& fileName) + { + FileStream stream(fileName, FileMode::Create, FileAccess::Write, FileShare::ReadWrite); + if (SLANG_FAILED(_writeContainerFile(endToEndReq, &stream))) + { + endToEndReq->getSink()->diagnose(SourceLoc(), Diagnostics::unableToWriteModuleContainer, fileName); + } + } static void _generateOutput( BackEndCompileRequest* compileRequest, @@ -2229,6 +2280,11 @@ SlangResult dissassembleDXILUsingDXC( ee); } } + + if (compileRequest->containerOutputPath.getLength() != 0) + { + _writeContainerFile(compileRequest, compileRequest->containerOutputPath); + } } } diff --git a/source/slang/slang-compiler.h b/source/slang/slang-compiler.h index 872b075d3..a527d3a31 100644 --- a/source/slang/slang-compiler.h +++ b/source/slang/slang-compiler.h @@ -70,7 +70,7 @@ namespace Slang CodeGenTarget calcCodeGenTargetFromName(const UnownedStringSlice& name); UnownedStringSlice getCodeGenTargetName(CodeGenTarget target); - enum class ContainerFormat + enum class ContainerFormat : SlangContainerFormat { None = SLANG_CONTAINER_FORMAT_NONE, SlangModule = SLANG_CONTAINER_FORMAT_SLANG_MODULE, diff --git a/source/slang/slang-diagnostic-defs.h b/source/slang/slang-diagnostic-defs.h index 3a708712b..6623b7195 100644 --- a/source/slang/slang-diagnostic-defs.h +++ b/source/slang/slang-diagnostic-defs.h @@ -110,6 +110,7 @@ DIAGNOSTIC( 80, Error, duplicateOutputPathsForEntryPointAndTarget, "multiple DIAGNOSTIC( 81, Error, parametersAfterLoadReproIgnored, "parameters after -load-repro [file] are ignored") DIAGNOSTIC( 82, Error, unableToWriteReproFile, "unable to write repro file '%0'"); +DIAGNOSTIC( 83, Error, unableToWriteModuleContainer, "unable to write module container '%0'"); // // 1xxxx - Lexical anaylsis diff --git a/source/slang/slang-ir-serialize.cpp b/source/slang/slang-ir-serialize.cpp index c82592527..ebbb792f4 100644 --- a/source/slang/slang-ir-serialize.cpp +++ b/source/slang/slang-ir-serialize.cpp @@ -792,64 +792,27 @@ Result IRSerialWriter::write(IRModule* module, SourceManager* sourceManager, Opt return SLANG_OK; } -template -static size_t _calcChunkSize(IRSerialBinary::CompressionType compressionType, const List& array) +static Result _writeArrayChunk(IRSerialBinary::CompressionType compressionType, uint32_t chunkId, const void* data, size_t numEntries, size_t typeSize, RiffContainer* container) { - typedef IRSerialBinary Bin; + typedef RiffContainer::Chunk Chunk; + typedef RiffContainer::ScopeChunk ScopeChunk; - if (array.getCount()) - { - switch (compressionType) - { - case Bin::CompressionType::None: - { - const size_t size = sizeof(Bin::ArrayHeader) + sizeof(T) * array.getCount(); - return (size + 3) & ~size_t(3); - } - case Bin::CompressionType::VariableByteLite: - { - const size_t payloadSize = ByteEncodeUtil::calcEncodeLiteSizeUInt32((const uint32_t*)array.begin(), (array.getCount() * sizeof(T)) / sizeof(uint32_t)); - const size_t size = sizeof(Bin::CompressedArrayHeader) + payloadSize; - return (size + 3) & ~size_t(3); - } - default: - { - SLANG_ASSERT(!"Unhandled compression type"); - return 0; - } - } - } - else - { - return 0; - } -} - -static Result _writeArrayChunk(IRSerialBinary::CompressionType compressionType, uint32_t chunkId, const void* data, size_t numEntries, size_t typeSize, Stream* stream) -{ typedef IRSerialBinary Bin; - if (numEntries == 0) { return SLANG_OK; } - size_t payloadSize; - switch (compressionType) { case Bin::CompressionType::None: { - payloadSize = sizeof(Bin::ArrayHeader) - sizeof(RiffChunk) + typeSize * numEntries; - + ScopeChunk scope(container, Chunk::Kind::Data, chunkId); Bin::ArrayHeader header; - header.m_chunk.m_type = chunkId; - header.m_chunk.m_size = uint32_t(payloadSize); header.m_numEntries = uint32_t(numEntries); - stream->write(&header, sizeof(header)); - - stream->write(data, typeSize * numEntries); + container->write(&header, sizeof(header)); + container->write(data, typeSize * numEntries); break; } case Bin::CompressionType::VariableByteLite: @@ -857,20 +820,14 @@ static Result _writeArrayChunk(IRSerialBinary::CompressionType compressionType, List compressedPayload; size_t numCompressedEntries = (numEntries * typeSize) / sizeof(uint32_t); - ByteEncodeUtil::encodeLiteUInt32((const uint32_t*)data, numCompressedEntries, compressedPayload); - payloadSize = sizeof(Bin::CompressedArrayHeader) - sizeof(RiffChunk) + compressedPayload.getCount(); - Bin::CompressedArrayHeader header; - header.m_chunk.m_type = SLANG_MAKE_COMPRESSED_FOUR_CC(chunkId); - header.m_chunk.m_size = uint32_t(payloadSize); header.m_numEntries = uint32_t(numEntries); header.m_numCompressedEntries = uint32_t(numCompressedEntries); - stream->write(&header, sizeof(header)); - - stream->write(compressedPayload.begin(), compressedPayload.getCount()); + container->write(&header, sizeof(header)); + container->write(compressedPayload.begin(), compressedPayload.getCount()); break; } default: @@ -878,22 +835,13 @@ static Result _writeArrayChunk(IRSerialBinary::CompressionType compressionType, return SLANG_FAIL; } } - // All chunks have sizes rounded to dword size - if (payloadSize & 3) - { - const uint8_t pad[4] = { 0, 0, 0, 0 }; - // Pad outs - int padSize = 4 - (payloadSize & 3); - stream->write(pad, padSize); - } - return SLANG_OK; } template -Result _writeArrayChunk(IRSerialBinary::CompressionType compressionType, uint32_t chunkId, const List& array, Stream* stream) +Result _writeArrayChunk(IRSerialBinary::CompressionType compressionType, uint32_t chunkId, const List& array, RiffContainer* container) { - return _writeArrayChunk(compressionType, chunkId, array.begin(), size_t(array.getCount()), sizeof(T), stream); + return _writeArrayChunk(compressionType, chunkId, array.begin(), size_t(array.getCount()), sizeof(T), container); } Result _encodeInsts(IRSerialBinary::CompressionType compressionType, const List& instsIn, List& encodeArrayOut) @@ -986,8 +934,11 @@ Result _encodeInsts(IRSerialBinary::CompressionType compressionType, const List< return SLANG_OK; } -Result _writeInstArrayChunk(IRSerialBinary::CompressionType compressionType, uint32_t chunkId, const List& array, Stream* stream) +Result _writeInstArrayChunk(IRSerialBinary::CompressionType compressionType, uint32_t chunkId, const List& array, RiffContainer* container) { + typedef RiffContainer::Chunk Chunk; + typedef RiffContainer::ScopeChunk ScopeChunk; + typedef IRSerialBinary Bin; if (array.getCount() == 0) { @@ -998,32 +949,21 @@ Result _writeInstArrayChunk(IRSerialBinary::CompressionType compressionType, uin { case Bin::CompressionType::None: { - return _writeArrayChunk(compressionType, chunkId, array, stream); + return _writeArrayChunk(compressionType, chunkId, array, container); } case Bin::CompressionType::VariableByteLite: { List compressedPayload; SLANG_RETURN_ON_FAIL(_encodeInsts(compressionType, array, compressedPayload)); - - size_t payloadSize = sizeof(Bin::CompressedArrayHeader) - sizeof(RiffChunk) + compressedPayload.getCount(); + + ScopeChunk scope(container, Chunk::Kind::Data, SLANG_MAKE_COMPRESSED_FOUR_CC(chunkId)); Bin::CompressedArrayHeader header; - header.m_chunk.m_type = SLANG_MAKE_COMPRESSED_FOUR_CC(chunkId); - header.m_chunk.m_size = uint32_t(payloadSize); header.m_numEntries = uint32_t(array.getCount()); header.m_numCompressedEntries = 0; - stream->write(&header, sizeof(header)); - stream->write(compressedPayload.begin(), compressedPayload.getCount()); - - // All chunks have sizes rounded to dword size - if (payloadSize & 3) - { - const uint8_t pad[4] = { 0, 0, 0, 0 }; - // Pad outs - int padSize = 4 - (payloadSize & 3); - stream->write(pad, padSize); - } + container->write(&header, sizeof(header)); + container->write(compressedPayload.begin(), compressedPayload.getCount()); return SLANG_OK; } default: break; @@ -1031,133 +971,47 @@ Result _writeInstArrayChunk(IRSerialBinary::CompressionType compressionType, uin return SLANG_FAIL; } -static size_t _calcInstChunkSize(IRSerialBinary::CompressionType compressionType, const List& instsIn) -{ - typedef IRSerialBinary Bin; - typedef IRSerialData::Inst::PayloadType PayloadType; - - switch (compressionType) - { - case Bin::CompressionType::None: - { - return _calcChunkSize(compressionType, instsIn); - } - case Bin::CompressionType::VariableByteLite: - { - size_t size = sizeof(Bin::CompressedArrayHeader); - - size_t numInsts = size_t(instsIn.getCount()); - size += numInsts * 2; // op and payload - - IRSerialData::Inst* insts = instsIn.begin(); - - for (size_t i = 0; i < numInsts; ++i) - { - const auto& inst = insts[i]; - - size += ByteEncodeUtil::calcEncodeLiteSizeUInt32((uint32_t)inst.m_resultTypeIndex); - - switch (inst.m_payloadType) - { - case PayloadType::Empty: - { - break; - } - case PayloadType::Operand_1: - case PayloadType::String_1: - case PayloadType::UInt32: - { - // 1 UInt32 - size += ByteEncodeUtil::calcEncodeLiteSizeUInt32((uint32_t)inst.m_payload.m_operands[0]); - break; - } - case PayloadType::Operand_2: - case PayloadType::OperandAndUInt32: - case PayloadType::OperandExternal: - case PayloadType::String_2: - { - // 2 UInt32 - size += ByteEncodeUtil::calcEncodeLiteSizeUInt32((uint32_t)inst.m_payload.m_operands[0]); - size += ByteEncodeUtil::calcEncodeLiteSizeUInt32((uint32_t)inst.m_payload.m_operands[1]); - break; - } - case PayloadType::Float64: - { - size += sizeof(inst.m_payload.m_float64); - break; - } - case PayloadType::Int64: - { - size += sizeof(inst.m_payload.m_int64); - break; - } - } - } - - return (size + 3) & ~size_t(3); - } - default: break; - } - - SLANG_ASSERT(!"Unhandled compression type"); - return 0; -} - -/* static */Result IRSerialWriter::writeStream(const IRSerialData& data, Bin::CompressionType compressionType, Stream* stream) +/* static */Result IRSerialWriter::writeContainer(const IRSerialData& data, Bin::CompressionType compressionType, RiffContainer* container) { - size_t totalSize = 0; - - totalSize += sizeof(Bin::SlangHeader) + - _calcInstChunkSize(compressionType, data.m_insts) + - _calcChunkSize(compressionType, data.m_childRuns) + - _calcChunkSize(compressionType, data.m_externalOperands) + - _calcChunkSize(Bin::CompressionType::None, data.m_stringTable) + - _calcChunkSize(Bin::CompressionType::None, data.m_rawSourceLocs); + typedef RiffContainer::Chunk Chunk; + typedef RiffContainer::ScopeChunk ScopeChunk; - if (data.m_debugSourceInfos.getCount()) - { - totalSize += _calcChunkSize(Bin::CompressionType::None, data.m_debugStringTable) + - _calcChunkSize(Bin::CompressionType::None, data.m_debugLineInfos) + - _calcChunkSize(Bin::CompressionType::None, data.m_debugAdjustedLineInfos) + - _calcChunkSize(Bin::CompressionType::None, data.m_debugSourceInfos) + - _calcChunkSize(compressionType, data.m_debugSourceLocRuns); - } + ScopeChunk scopeModule(container, Chunk::Kind::List, Bin::kSlangModuleFourCc); + // Write the header { - RiffChunk riffHeader; - riffHeader.m_type = Bin::kRiffFourCc; - riffHeader.m_size = uint32_t(totalSize); - - stream->write(&riffHeader, sizeof(riffHeader)); - } - { - Bin::SlangHeader slangHeader; - slangHeader.m_chunk.m_type = Bin::kSlangFourCc; - slangHeader.m_chunk.m_size = uint32_t(sizeof(slangHeader) - sizeof(RiffChunk)); - slangHeader.m_compressionType = uint32_t(Bin::CompressionType::VariableByteLite); - - stream->write(&slangHeader, sizeof(slangHeader)); + Bin::ModuleHeader moduleHeader; + moduleHeader.m_compressionType = uint32_t(Bin::CompressionType::VariableByteLite); + ScopeChunk scopeHeader(container, Chunk::Kind::Data, Bin::kSlangModuleHeaderFourCc); + container->write(&moduleHeader, sizeof(moduleHeader)); } - SLANG_RETURN_ON_FAIL(_writeInstArrayChunk(compressionType, Bin::kInstFourCc, data.m_insts, stream)); - SLANG_RETURN_ON_FAIL(_writeArrayChunk(compressionType, Bin::kChildRunFourCc, data.m_childRuns, stream)); - SLANG_RETURN_ON_FAIL(_writeArrayChunk(compressionType, Bin::kExternalOperandsFourCc, data.m_externalOperands, stream)); - SLANG_RETURN_ON_FAIL(_writeArrayChunk(Bin::CompressionType::None, Bin::kStringFourCc, data.m_stringTable, stream)); + SLANG_RETURN_ON_FAIL(_writeInstArrayChunk(compressionType, Bin::kInstFourCc, data.m_insts, container)); + SLANG_RETURN_ON_FAIL(_writeArrayChunk(compressionType, Bin::kChildRunFourCc, data.m_childRuns, container)); + SLANG_RETURN_ON_FAIL(_writeArrayChunk(compressionType, Bin::kExternalOperandsFourCc, data.m_externalOperands, container)); + SLANG_RETURN_ON_FAIL(_writeArrayChunk(Bin::CompressionType::None, Bin::kStringFourCc, data.m_stringTable, container)); - SLANG_RETURN_ON_FAIL(_writeArrayChunk(Bin::CompressionType::None, Bin::kUInt32SourceLocFourCc, data.m_rawSourceLocs, stream)); + SLANG_RETURN_ON_FAIL(_writeArrayChunk(Bin::CompressionType::None, Bin::kUInt32SourceLocFourCc, data.m_rawSourceLocs, container)); if (data.m_debugSourceInfos.getCount()) { - _writeArrayChunk(Bin::CompressionType::None, Bin::kDebugStringFourCc, data.m_debugStringTable, stream); - _writeArrayChunk(Bin::CompressionType::None, Bin::kDebugLineInfoFourCc, data.m_debugLineInfos, stream); - _writeArrayChunk(Bin::CompressionType::None, Bin::kDebugAdjustedLineInfoFourCc, data.m_debugAdjustedLineInfos, stream); - _writeArrayChunk(Bin::CompressionType::None, Bin::kDebugSourceInfoFourCc, data.m_debugSourceInfos, stream); - _writeArrayChunk(compressionType, Bin::kDebugSourceLocRunFourCc, data.m_debugSourceLocRuns, stream); + _writeArrayChunk(Bin::CompressionType::None, Bin::kDebugStringFourCc, data.m_debugStringTable, container); + _writeArrayChunk(Bin::CompressionType::None, Bin::kDebugLineInfoFourCc, data.m_debugLineInfos, container); + _writeArrayChunk(Bin::CompressionType::None, Bin::kDebugAdjustedLineInfoFourCc, data.m_debugAdjustedLineInfos, container); + _writeArrayChunk(Bin::CompressionType::None, Bin::kDebugSourceInfoFourCc, data.m_debugSourceInfos, container); + _writeArrayChunk(compressionType, Bin::kDebugSourceLocRunFourCc, data.m_debugSourceLocRuns, container); } return SLANG_OK; } +/* static */Result IRSerialWriter::writeStream(const IRSerialData& data, Bin::CompressionType compressionType, Stream* stream) +{ + RiffContainer container; + SLANG_RETURN_ON_FAIL(writeContainer(data, compressionType, &container)); + return RiffUtil::write(container.getRoot(), true, stream); +} + // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! IRSerialReader !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! class ListResizer @@ -1192,100 +1046,95 @@ class ListResizerForType: public ListResizer List& m_list; }; -static Result _readArrayChunk(IRSerialBinary::CompressionType compressionType, const RiffChunk& chunk, Stream* stream, size_t* numReadInOut, ListResizer& listOut) +static Result _readArrayChunk(IRSerialBinary::CompressionType compressionType, RiffContainer::DataChunk* dataChunk, ListResizer& listOut) { typedef IRSerialBinary Bin; + RiffContainer::Data* data = dataChunk->getSingleData(); + const uint8_t* cur = (const uint8_t*)data->getPayload(); + const uint8_t* end = cur + data->m_size; + const size_t typeSize = listOut.getTypeSize(); switch (compressionType) { case Bin::CompressionType::VariableByteLite: { - // We have a compressed header - Bin::CompressedArrayHeader header; - header.m_chunk = chunk; - - stream->read(&header.m_chunk + 1, sizeof(header) - sizeof(RiffChunk)); - *numReadInOut += sizeof(header) - sizeof(RiffChunk); + if (cur + sizeof(Bin::CompressedArrayHeader) > end) + { + return SLANG_FAIL; + } - void* data = listOut.setSize(header.m_numEntries); + // We have a compressed header + Bin::CompressedArrayHeader header = *(const Bin::CompressedArrayHeader*)cur; + cur += sizeof(Bin::CompressedArrayHeader); + + void* dst = listOut.setSize(header.m_numEntries); // Need to read all the compressed data... - size_t payloadSize = header.m_chunk.m_size - (sizeof(header) - sizeof(RiffChunk)); + size_t payloadSize = size_t(end - cur); List compressedPayload; compressedPayload.setCount(payloadSize); - stream->read(compressedPayload.begin(), payloadSize); - *numReadInOut += payloadSize; - SLANG_ASSERT(header.m_numCompressedEntries == uint32_t((header.m_numEntries * typeSize) / sizeof(uint32_t))); // Decode.. - ByteEncodeUtil::decodeLiteUInt32(compressedPayload.begin(), header.m_numCompressedEntries, (uint32_t*)data); + ByteEncodeUtil::decodeLiteUInt32(cur, header.m_numCompressedEntries, (uint32_t*)dst); break; } case Bin::CompressionType::None: { // Read uncompressed - Bin::ArrayHeader header; - header.m_chunk = chunk; + if (cur + sizeof(Bin::ArrayHeader) > end) + { + return SLANG_FAIL; + } - stream->read(&header.m_chunk + 1, sizeof(header) - sizeof(RiffChunk)); - *numReadInOut += sizeof(header) - sizeof(RiffChunk); + Bin::ArrayHeader header = *(const Bin::ArrayHeader*)cur; + cur += sizeof(Bin::ArrayHeader); const size_t payloadSize = header.m_numEntries * typeSize; - void* data = listOut.setSize(header.m_numEntries); + SLANG_ASSERT(payloadSize == size_t(end - cur)); + void* dst = listOut.setSize(header.m_numEntries); - stream->read(data, payloadSize); - *numReadInOut += payloadSize; + ::memcpy(dst, cur, payloadSize); break; } } - - // All chunks have sizes rounded to dword size - if (*numReadInOut & 3) - { - const uint8_t pad[4] = { 0, 0, 0, 0 }; - // Pad outs - int padSize = 4 - int(*numReadInOut & 3); - stream->seek(SeekOrigin::Current, padSize); - - *numReadInOut += padSize; - } - return SLANG_OK; } template -static Result _readArrayChunk(const IRSerialBinary::SlangHeader& header, const RiffChunk& chunk, Stream* stream, size_t* numReadInOut, List& arrayOut) +static Result _readArrayChunk(const IRSerialBinary::ModuleHeader* header, RiffContainer::DataChunk* dataChunk, List& arrayOut) { typedef IRSerialBinary Bin; Bin::CompressionType compressionType = Bin::CompressionType::None; - if (chunk.m_type == SLANG_MAKE_COMPRESSED_FOUR_CC(chunk.m_type)) + if (dataChunk->m_type == SLANG_MAKE_COMPRESSED_FOUR_CC(dataChunk->m_type)) { // If it has compression, use the compression type set in the header - compressionType = Bin::CompressionType(header.m_compressionType); + compressionType = Bin::CompressionType(header->m_compressionType); } ListResizerForType resizer(arrayOut); - return _readArrayChunk(compressionType, chunk, stream, numReadInOut, resizer); + return _readArrayChunk(compressionType, dataChunk, resizer); } template -static Result _readArrayUncompressedChunk(const IRSerialBinary::SlangHeader& header, const RiffChunk& chunk, Stream* stream, size_t* numReadInOut, List& arrayOut) +static Result _readArrayUncompressedChunk(const IRSerialBinary::ModuleHeader* header, RiffContainer::DataChunk* chunk, List& arrayOut) { typedef IRSerialBinary Bin; SLANG_UNUSED(header); ListResizerForType resizer(arrayOut); - return _readArrayChunk(Bin::CompressionType::None, chunk, stream, numReadInOut, resizer); + return _readArrayChunk(Bin::CompressionType::None, chunk, resizer); } -static Result _decodeInsts(IRSerialBinary::CompressionType compressionType, const List& encodeIn, List& instsOut) +static Result _decodeInsts(IRSerialBinary::CompressionType compressionType, const uint8_t* encodeCur, size_t encodeInSize, List& instsOut) { + const uint8_t* encodeEnd = encodeCur + encodeInSize; + typedef IRSerialBinary Bin; typedef IRSerialData::Inst::PayloadType PayloadType; @@ -1297,10 +1146,14 @@ static Result _decodeInsts(IRSerialBinary::CompressionType compressionType, cons const size_t numInsts = size_t(instsOut.getCount()); IRSerialData::Inst* insts = instsOut.begin(); - const uint8_t* encodeCur = encodeIn.begin(); - for (size_t i = 0; i < numInsts; ++i) { + if (encodeCur >= encodeEnd) + { + SLANG_ASSERT(!"Invalid decode"); + return SLANG_FAIL; + } + auto& inst = insts[i]; inst.m_op = *encodeCur++; @@ -1351,14 +1204,14 @@ static Result _decodeInsts(IRSerialBinary::CompressionType compressionType, cons return SLANG_OK; } -static Result _readInstArrayChunk(const IRSerialBinary::SlangHeader& slangHeader, const RiffChunk& chunk, Stream* stream, size_t* numReadInOut, List& arrayOut) +static Result _readInstArrayChunk(const IRSerialBinary::ModuleHeader* moduleHeader, RiffContainer::DataChunk* chunk, List& arrayOut) { typedef IRSerialBinary Bin; Bin::CompressionType compressionType = Bin::CompressionType::None; - if (chunk.m_type == SLANG_MAKE_COMPRESSED_FOUR_CC(chunk.m_type)) + if (chunk->m_type == SLANG_MAKE_COMPRESSED_FOUR_CC(chunk->m_type)) { - compressionType = Bin::CompressionType(slangHeader.m_compressionType); + compressionType = Bin::CompressionType(moduleHeader->m_compressionType); } switch (compressionType) @@ -1366,29 +1219,26 @@ static Result _readInstArrayChunk(const IRSerialBinary::SlangHeader& slangHeader case Bin::CompressionType::None: { ListResizerForType resizer(arrayOut); - return _readArrayChunk(compressionType, chunk, stream, numReadInOut, resizer); + return _readArrayChunk(compressionType, chunk, resizer); } case Bin::CompressionType::VariableByteLite: { - // We have a compressed header - Bin::CompressedArrayHeader header; - header.m_chunk = chunk; - - stream->read(&header.m_chunk + 1, sizeof(header) - sizeof(RiffChunk)); - *numReadInOut += sizeof(header) - sizeof(RiffChunk); + RiffContainer::Data* data = chunk->getSingleData(); + const uint8_t* cur = (const uint8_t*)data->getPayload(); + const uint8_t* end = cur + data->m_size; + if (cur + sizeof(Bin::CompressedArrayHeader) <= end) + { + return SLANG_FAIL; + } + Bin::CompressedArrayHeader header = *(const Bin::CompressedArrayHeader*)cur; + cur += sizeof(Bin::CompressedArrayHeader); - // Need to read all the compressed data... - size_t payloadSize = header.m_chunk.m_size - (sizeof(header) - sizeof(RiffChunk)); - - List compressedPayload; - compressedPayload.setCount(payloadSize); - - stream->read(compressedPayload.begin(), payloadSize); - *numReadInOut += payloadSize; - arrayOut.setCount(header.m_numEntries); - SLANG_RETURN_ON_FAIL(_decodeInsts(compressionType, compressedPayload, arrayOut)); + // Need to read all the compressed data... + size_t payloadSize = size_t(end - cur); + + SLANG_RETURN_ON_FAIL(_decodeInsts(compressionType, cur, payloadSize, arrayOut)); break; } default: @@ -1397,131 +1247,87 @@ static Result _readInstArrayChunk(const IRSerialBinary::SlangHeader& slangHeader } } - // All chunks have sizes rounded to dword size - if (*numReadInOut & 3) - { - // Pad outs - int padSize = 4 - int(*numReadInOut & 3); - stream->seek(SeekOrigin::Current, padSize); - *numReadInOut += padSize; - } - return SLANG_OK; } -/* static */Result IRSerialReader::readStream(Stream* stream, IRSerialData* dataOut) +/* static */Result IRSerialReader::readContainer(RiffContainer::ListChunk* module, IRSerialData* outData) { typedef IRSerialBinary Bin; - dataOut->clear(); + outData->clear(); - int64_t remainingBytes = 0; + Bin::ModuleHeader* header = module->findContainedData(Bin::kSlangModuleHeaderFourCc); + if (!header) { - RiffChunk header; - SLANG_RETURN_ON_FAIL(RiffUtil::readChunk(stream, header)); - if (header.m_type != Bin::kRiffFourCc) - { - return SLANG_FAIL; - } - - remainingBytes = header.m_size; + return SLANG_FAIL; } - - // Header - // Chunk will not be kSlangFourCC if not read yet - Bin::SlangHeader slangHeader; - memset(&slangHeader, 0, sizeof(slangHeader)); - while (remainingBytes > 0) + for (RiffContainer::Chunk* chunk = module->m_containedChunks; chunk; chunk = chunk->m_next) { - RiffChunk chunk; - SLANG_RETURN_ON_FAIL(RiffUtil::readChunk(stream, chunk)); - - size_t bytesRead = sizeof(chunk); - - switch (chunk.m_type) + RiffContainer::DataChunk* dataChunk = as(chunk); + if (!dataChunk) + { + continue; + } + + switch (dataChunk->m_type) { - case Bin::kSlangFourCc: - { - // Slang header - slangHeader.m_chunk = chunk; - - // NOTE! Really we should only read what we know the size to be... - // and skip if it's larger - - stream->read(&slangHeader.m_chunk + 1, sizeof(slangHeader) - sizeof(chunk)); - - remainingBytes -= RiffUtil::calcChunkTotalSize(chunk); - break; - } case SLANG_MAKE_COMPRESSED_FOUR_CC(Bin::kInstFourCc): case Bin::kInstFourCc: { - SLANG_RETURN_ON_FAIL(_readInstArrayChunk(slangHeader, chunk, stream, &bytesRead, dataOut->m_insts)); - remainingBytes -= RiffUtil::calcChunkTotalSize(chunk); - break; + SLANG_RETURN_ON_FAIL(_readInstArrayChunk(header, dataChunk, outData->m_insts)); + break; } case SLANG_MAKE_COMPRESSED_FOUR_CC(Bin::kChildRunFourCc): case Bin::kChildRunFourCc: { - SLANG_RETURN_ON_FAIL(_readArrayChunk(slangHeader, chunk, stream, &bytesRead, dataOut->m_childRuns)); - remainingBytes -= RiffUtil::calcChunkTotalSize(chunk); + SLANG_RETURN_ON_FAIL(_readArrayChunk(header, dataChunk, outData->m_childRuns)); break; } case SLANG_MAKE_COMPRESSED_FOUR_CC(Bin::kExternalOperandsFourCc): case Bin::kExternalOperandsFourCc: { - SLANG_RETURN_ON_FAIL(_readArrayChunk(slangHeader, chunk, stream, &bytesRead, dataOut->m_externalOperands)); - remainingBytes -= RiffUtil::calcChunkTotalSize(chunk); + SLANG_RETURN_ON_FAIL(_readArrayChunk(header, dataChunk, outData->m_externalOperands)); break; } case Bin::kStringFourCc: { - SLANG_RETURN_ON_FAIL(_readArrayUncompressedChunk(slangHeader, chunk, stream, &bytesRead, dataOut->m_stringTable)); - remainingBytes -= RiffUtil::calcChunkTotalSize(chunk); + SLANG_RETURN_ON_FAIL(_readArrayUncompressedChunk(header, dataChunk, outData->m_stringTable)); break; } case Bin::kUInt32SourceLocFourCc: { - SLANG_RETURN_ON_FAIL(_readArrayUncompressedChunk(slangHeader, chunk, stream, &bytesRead, dataOut->m_rawSourceLocs)); - remainingBytes -= RiffUtil::calcChunkTotalSize(chunk); + SLANG_RETURN_ON_FAIL(_readArrayUncompressedChunk(header, dataChunk, outData->m_rawSourceLocs)); break; } case Bin::kDebugStringFourCc: { - SLANG_RETURN_ON_FAIL(_readArrayUncompressedChunk(slangHeader, chunk, stream, &bytesRead, dataOut->m_debugStringTable)); - remainingBytes -= RiffUtil::calcChunkTotalSize(chunk); + SLANG_RETURN_ON_FAIL(_readArrayUncompressedChunk(header, dataChunk, outData->m_debugStringTable)); break; } case Bin::kDebugLineInfoFourCc: { - SLANG_RETURN_ON_FAIL(_readArrayUncompressedChunk(slangHeader, chunk, stream, &bytesRead, dataOut->m_debugLineInfos)); - remainingBytes -= RiffUtil::calcChunkTotalSize(chunk); + SLANG_RETURN_ON_FAIL(_readArrayUncompressedChunk(header, dataChunk, outData->m_debugLineInfos)); break; } case Bin::kDebugAdjustedLineInfoFourCc: { - SLANG_RETURN_ON_FAIL(_readArrayUncompressedChunk(slangHeader, chunk, stream, &bytesRead, dataOut->m_debugAdjustedLineInfos)); - remainingBytes -= RiffUtil::calcChunkTotalSize(chunk); + SLANG_RETURN_ON_FAIL(_readArrayUncompressedChunk(header, dataChunk, outData->m_debugAdjustedLineInfos)); break; } case Bin::kDebugSourceInfoFourCc: { - SLANG_RETURN_ON_FAIL(_readArrayChunk(slangHeader, chunk, stream, &bytesRead, dataOut->m_debugSourceInfos)); - remainingBytes -= RiffUtil::calcChunkTotalSize(chunk); + SLANG_RETURN_ON_FAIL(_readArrayChunk(header, dataChunk, outData->m_debugSourceInfos)); break; } case SLANG_MAKE_COMPRESSED_FOUR_CC(Bin::kDebugSourceLocRunFourCc): case Bin::kDebugSourceLocRunFourCc: { - SLANG_RETURN_ON_FAIL(_readArrayChunk(slangHeader, chunk, stream, &bytesRead, dataOut->m_debugSourceLocRuns)); - remainingBytes -= RiffUtil::calcChunkTotalSize(chunk); + SLANG_RETURN_ON_FAIL(_readArrayChunk(header, dataChunk, outData->m_debugSourceLocRuns)); break; - } - + } default: { - SLANG_RETURN_ON_FAIL(RiffUtil::skip(chunk, stream, &remainingBytes)); break; } } @@ -1530,6 +1336,27 @@ static Result _readInstArrayChunk(const IRSerialBinary::SlangHeader& slangHeader return SLANG_OK; } +/* static */Result IRSerialReader::readStream(Stream* stream, IRSerialData* outData) +{ + typedef IRSerialBinary Bin; + + outData->clear(); + + // Read the riff file + RiffContainer container; + SLANG_RETURN_ON_FAIL(RiffUtil::read(stream, container)); + + // Find the riff module + RiffContainer::ListChunk* module = container.getRoot()->findListRec(Bin::kSlangModuleFourCc); + if (!module) + { + return SLANG_FAIL; + } + + SLANG_RETURN_ON_FAIL(readContainer(module, outData)); + return SLANG_OK; +} + static SourceRange _toSourceRange(const IRSerialData::DebugSourceInfo& info) { SourceRange range; @@ -2071,34 +1898,4 @@ static int _calcFixSourceLoc(const IRSerialData::DebugSourceInfo& info, SourceVi return SLANG_OK; } -// !!!!!!!!!!!!!!!!!!!!!!!!!!!! Free functions !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -#if 0 - -Result serializeModule(IRModule* module, SourceManager* sourceManager, Stream* stream) -{ - IRSerialWriter serializer; - IRSerialData serialData; - - SLANG_RETURN_ON_FAIL(serializer.write(module, sourceManager, IRSerialWriter::OptionFlag::RawSourceLocation, &serialData)); - - if (stream) - { - SLANG_RETURN_ON_FAIL(IRSerialWriter::writeStream(serialData, IRSerialBinary::CompressionType::VariableByteLite, stream)); - } - - return SLANG_OK; -} - -Result readModule(Session* session, Stream* stream, RefPtr& moduleOut) -{ - IRSerialData serialData; - IRSerialReader::readStream(stream, &serialData); - - IRSerialReader reader; - return reader.read(serialData, session, moduleOut); -} - -#endif - } // namespace Slang diff --git a/source/slang/slang-ir-serialize.h b/source/slang/slang-ir-serialize.h index 3cd4098e0..ed9441874 100644 --- a/source/slang/slang-ir-serialize.h +++ b/source/slang/slang-ir-serialize.h @@ -351,7 +351,6 @@ SLANG_FORCE_INLINE int IRSerialData::getOperands(const Inst& inst, const InstInd struct IRSerialBinary { - enum class CompressionType { None, @@ -359,7 +358,10 @@ struct IRSerialBinary }; static const uint32_t kRiffFourCc = RiffFourCC::kRiff; - static const uint32_t kSlangFourCc = SLANG_FOUR_CC('S', 'L', 'N', 'G'); ///< Holds all the slang specific chunks + + static const uint32_t kSlangModuleFourCc = SLANG_FOUR_CC('S', 'L', 'm', 'd'); ///< Holds all the slang specific chunks + + static const uint32_t kSlangModuleHeaderFourCc = SLANG_FOUR_CC('S', 'L', 'h', 'd'); static const uint32_t kInstFourCc = SLANG_FOUR_CC('S', 'L', 'i', 'n'); static const uint32_t kChildRunFourCc = SLANG_FOUR_CC('S', 'L', 'c', 'r'); @@ -379,19 +381,16 @@ struct IRSerialBinary static const uint32_t kDebugSourceInfoFourCc = SLANG_FOUR_CC('S', 'd', 's', 'o'); static const uint32_t kDebugSourceLocRunFourCc = SLANG_FOUR_CC('S', 'd', 's', 'r'); - struct SlangHeader + struct ModuleHeader { - RiffChunk m_chunk; uint32_t m_compressionType; ///< Holds the compression type used (if used at all) }; struct ArrayHeader { - RiffChunk m_chunk; uint32_t m_numEntries; }; struct CompressedArrayHeader { - RiffChunk m_chunk; uint32_t m_numEntries; ///< The number of entries uint32_t m_numCompressedEntries; ///< The amount of compressed entries }; @@ -418,6 +417,9 @@ struct IRSerialWriter static Result writeStream(const IRSerialData& data, Bin::CompressionType compressionType, Stream* stream); + /// Write to a container + static Result writeContainer(const IRSerialData& data, Bin::CompressionType compressionType, RiffContainer* container); + /// Get an instruction index from an instruction Ser::InstIndex getInstIndex(IRInst* inst) const { return inst ? Ser::InstIndex(m_instMap[inst]) : Ser::InstIndex(0); } @@ -499,6 +501,9 @@ struct IRSerialReader /// Read a stream to fill in dataOut IRSerialData static Result readStream(Stream* stream, IRSerialData* dataOut); + /// Read a stream to fill in dataOut IRSerialData + static Result readContainer(RiffContainer::ListChunk* module, IRSerialData* outData); + /// Read a module from serial data Result read(const IRSerialData& data, Session* session, SourceManager* sourceManager, RefPtr& moduleOut); diff --git a/source/slang/slang-state-serialize.cpp b/source/slang/slang-state-serialize.cpp index 4c2651475..e93495fdb 100644 --- a/source/slang/slang-state-serialize.cpp +++ b/source/slang/slang-state-serialize.cpp @@ -969,7 +969,7 @@ struct LoadContext SLANG_RETURN_ON_FAIL(store(request, container, requestState)); Header header; - header.m_chunk.m_type = kSlangStateFourCC; + header.m_chunk.type = kSlangStateFourCC; header.m_semanticVersion = g_semanticVersion; header.m_typeHash = _getTypeHash(); @@ -1002,7 +1002,7 @@ struct LoadContext Header header; SLANG_RETURN_ON_FAIL(RiffUtil::readData(stream, &header.m_chunk, sizeof(header), buffer)); - if (header.m_chunk.m_type != kSlangStateFourCC) + if (header.m_chunk.type != kSlangStateFourCC) { return SLANG_FAIL; } diff --git a/source/slang/slang-state-serialize.h b/source/slang/slang-state-serialize.h index d02bfae8c..1a4c9b227 100644 --- a/source/slang/slang-state-serialize.h +++ b/source/slang/slang-state-serialize.h @@ -28,7 +28,7 @@ struct StateSerializeUtil struct Header { - RiffChunk m_chunk; ///< The chunk + RiffHeader m_chunk; ///< The chunk RiffSemanticVersion m_semanticVersion; ///< The semantic version uint32_t m_typeHash; ///< A hash based on the binary representation. If doesn't match then not binary compatible (extra check over semantic versioning) }; -- cgit v1.2.3