From 2b20e9e150225f8cb75f7be2061f5ec99cbb2f66 Mon Sep 17 00:00:00 2001 From: Theresa Foley <10618364+tangent-vector@users.noreply.github.com> Date: Wed, 16 Apr 2025 12:28:39 -0700 Subject: Remove support for ad hoc Slang IR compression (#6834) * Remove support for ad hoc Slang IR compression This change is part of a larger effort to clean up the approach to serialization in the Slang compiler. The overall goal is to simplify and streamline all of the serialization-related logic, so that we are left with code that is less "clever," and easier to understand for contributors to the codebase. Removing support for compression of serialized Slang IR has benefits that include: * Reduction in code complexity: consider things like the subtle way that the `FOURCC`s for compressed chunks were being computed from the uncompressed versions, and the mental overhead that goes into understanding that, for anybody who would dare to touch this code. * Reduction in testing burden: there have been, de facto, two very different code paths for serialization of the Slang IR, and it is not clear that the existing test corpus for Slang has sufficient coverage for both options. By having only a single code path, every test that performs any amount of IR serialization helps with test coverage of that one path. * Opportunity to explore alternatives. This is perhaps a reiteration of the first point, but once the code is stripped down to the simplest thing that could possibly work (I am not claiming it has reached that point yet), it becomes easier for contributors to understand, and it becomes more tractable for somebody to come along with an improved approach that performs better (in either compression ratio or performance) while still being maintainable. In my own local setup, I found that removing support for Slang IR compression led to the `slang-core-module-generated.h` file increasing in size from 46.1MB to 47.4MB. This increase in the `.h` file size for the core library binary only resulted in a release build of `slang.dll` increasing from 20.0MB to 20.2MB. Removing the ad hoc compression support has almost no impact on the size of actual binary Slang modules *so long* as the additional LZ4 compression step is being applied to them. * format code --------- Co-authored-by: slangbot <186143334+slangbot@users.noreply.github.com> --- source/slang/slang-compiler-options.h | 2 +- source/slang/slang-compiler.cpp | 3 - source/slang/slang-options.cpp | 6 +- source/slang/slang-serialize-container.cpp | 48 +---- source/slang/slang-serialize-container.h | 3 - source/slang/slang-serialize-ir-types.cpp | 11 -- source/slang/slang-serialize-ir-types.h | 5 - source/slang/slang-serialize-ir.cpp | 297 ++-------------------------- source/slang/slang-serialize-ir.h | 10 +- source/slang/slang-serialize-source-loc.cpp | 29 +-- source/slang/slang-serialize-source-loc.h | 6 +- source/slang/slang-serialize-types.cpp | 136 ++----------- source/slang/slang-serialize-types.h | 84 +------- source/slang/slang.cpp | 1 - 14 files changed, 54 insertions(+), 587 deletions(-) (limited to 'source/slang') diff --git a/source/slang/slang-compiler-options.h b/source/slang/slang-compiler-options.h index c28bbe83d..9de2a2e80 100644 --- a/source/slang/slang-compiler-options.h +++ b/source/slang/slang-compiler-options.h @@ -274,7 +274,7 @@ struct CompilerOptionSet result->getCount() != 0 && (*result)[0].kind == CompilerOptionValueKind::Int); return (*result)[0].intValue; } - return getDefault(name).intValue != 0; + return getDefault(name).intValue; } String getStringOption(CompilerOptionName name) { diff --git a/source/slang/slang-compiler.cpp b/source/slang/slang-compiler.cpp index 55f3846af..2c9475004 100644 --- a/source/slang/slang-compiler.cpp +++ b/source/slang/slang-compiler.cpp @@ -2165,9 +2165,6 @@ SlangResult EndToEndCompileRequest::writeContainerToStream(Stream* stream) // Set up options SerialContainerUtil::WriteOptions options; - options.compressionType = linkage->m_optionSet.getEnumOption( - CompilerOptionName::IrCompression); - // If debug information is enabled, enable writing out source locs if (_shouldWriteSourceLocs(linkage)) { diff --git a/source/slang/slang-options.cpp b/source/slang/slang-options.cpp index ed7775e84..b09e36a1b 100644 --- a/source/slang/slang-options.cpp +++ b/source/slang/slang-options.cpp @@ -2424,11 +2424,7 @@ SlangResult OptionsParser::_parse(int argc, char const* const* argv) { CommandLineArg name; SLANG_RETURN_ON_FAIL(m_reader.expectArg(name)); - SerialCompressionType compressionType; - SLANG_RETURN_ON_FAIL(SerialParseUtil::parseCompressionType( - name.value.getUnownedSlice(), - compressionType)); - linkage->m_optionSet.set(optionKind, compressionType); + // TODO: doagnose deprecated option break; } case OptionKind::EmbedDownstreamIR: diff --git a/source/slang/slang-serialize-container.cpp b/source/slang/slang-serialize-container.cpp index b5121d373..f82357459 100644 --- a/source/slang/slang-serialize-container.cpp +++ b/source/slang/slang-serialize-container.cpp @@ -290,19 +290,6 @@ namespace Slang RiffContainer::Chunk::Kind::List, SerialBinary::kContainerFourCc); - // Write the header - { - SerialBinary::ContainerHeader containerHeader; - // Save the compression type if used - as can only serialize with a single compression type - containerHeader.compressionType = uint32_t(options.compressionType); - - RiffContainer::ScopeChunk scopeHeader( - container, - RiffContainer::Chunk::Kind::Data, - SerialBinary::kContainerHeaderFourCc); - container->write(&containerHeader, sizeof(containerHeader)); - } - if (data.modules.getCount() && (options.optionFlags & (SerialOptionFlag::IRModule | SerialOptionFlag::ASTModule))) { @@ -356,8 +343,7 @@ namespace Slang sourceLocWriter, options.optionFlags, &serialData)); - SLANG_RETURN_ON_FAIL( - IRSerialWriter::writeContainer(serialData, options.compressionType, container)); + SLANG_RETURN_ON_FAIL(IRSerialWriter::writeContainer(serialData, container)); } // Write the AST information @@ -451,7 +437,7 @@ namespace Slang SerialSourceLocData debugData; sourceLocWriter->write(&debugData); - debugData.writeContainer(options.compressionType, container); + debugData.writeContainer(container); } // Write the container string table @@ -501,18 +487,6 @@ static List& _getCandidateExtensionList( return SLANG_FAIL; } - SerialBinary::ContainerHeader* containerHeader = - containerChunk->findContainedData( - SerialBinary::kContainerHeaderFourCc); - if (!containerHeader) - { - // Didn't find the header - return SLANG_FAIL; - } - - const SerialCompressionType containerCompressionType = - SerialCompressionType(containerHeader->compressionType); - StringSlicePool containerStringPool(StringSlicePool::Style::Default); if (RiffContainer::Data* stringTableData = @@ -532,7 +506,7 @@ static List& _getCandidateExtensionList( { // Read into data SerialSourceLocData sourceLocData; - SLANG_RETURN_ON_FAIL(sourceLocData.readContainer(containerCompressionType, debugChunk)); + SLANG_RETURN_ON_FAIL(sourceLocData.readContainer(debugChunk)); // Turn into DebugReader sourceLocReader = new SerialSourceLocReader; @@ -610,10 +584,7 @@ static List& _getCandidateExtensionList( if (!options.readHeaderOnly) { IRSerialData serialData; - SLANG_RETURN_ON_FAIL(IRSerialReader::readContainer( - irChunk, - containerCompressionType, - &serialData)); + SLANG_RETURN_ON_FAIL(IRSerialReader::readContainer(irChunk, &serialData)); // Read IR back from serialData IRSerialReader reader; @@ -961,8 +932,7 @@ static List& _getCandidateExtensionList( SLANG_RETURN_ON_FAIL( writer.write(module, sourceLocWriter, options.optionFlags, &irData)); } - SLANG_RETURN_ON_FAIL( - IRSerialWriter::writeContainer(irData, options.compressionType, &riffContainer)); + SLANG_RETURN_ON_FAIL(IRSerialWriter::writeContainer(irData, &riffContainer)); // Write the debug info Riff container if (sourceLocWriter) @@ -970,8 +940,7 @@ static List& _getCandidateExtensionList( SerialSourceLocData serialData; sourceLocWriter->write(&serialData); - SLANG_RETURN_ON_FAIL( - serialData.writeContainer(options.compressionType, &riffContainer)); + SLANG_RETURN_ON_FAIL(serialData.writeContainer(&riffContainer)); } SLANG_RETURN_ON_FAIL(RiffUtil::write(&riffContainer, &memoryStream)); @@ -1003,7 +972,7 @@ static List& _getCandidateExtensionList( return SLANG_FAIL; } SerialSourceLocData sourceLocData; - SLANG_RETURN_ON_FAIL(sourceLocData.readContainer(options.compressionType, debugList)); + SLANG_RETURN_ON_FAIL(sourceLocData.readContainer(debugList)); sourceLocReader = new SerialSourceLocReader; SLANG_RETURN_ON_FAIL(sourceLocReader->read(&sourceLocData, &workSourceManager)); @@ -1020,8 +989,7 @@ static List& _getCandidateExtensionList( { IRSerialData irReadData; IRSerialReader reader; - SLANG_RETURN_ON_FAIL( - reader.readContainer(irList, options.compressionType, &irReadData)); + SLANG_RETURN_ON_FAIL(reader.readContainer(irList, &irReadData)); // Check the stream read data is the same if (irData != irReadData) diff --git a/source/slang/slang-serialize-container.h b/source/slang/slang-serialize-container.h index c8e9f2a7b..8ddc5072a 100644 --- a/source/slang/slang-serialize-container.h +++ b/source/slang/slang-serialize-container.h @@ -83,9 +83,6 @@ struct SerialContainerUtil { struct WriteOptions { - SerialCompressionType compressionType = - SerialCompressionType::VariableByteLite; ///< If compression is used what type to use - ///< (only some parts can be compressed) SerialOptionFlags optionFlags = SerialOptionFlag::ASTModule | SerialOptionFlag::IRModule; ///< Flags controlling what is written diff --git a/source/slang/slang-serialize-ir-types.cpp b/source/slang/slang-serialize-ir-types.cpp index e7d42b642..47e35f2f8 100644 --- a/source/slang/slang-serialize-ir-types.cpp +++ b/source/slang/slang-serialize-ir-types.cpp @@ -37,17 +37,6 @@ casts it. {0, 0} // Int64, }; -// Check all compressible chunk ids, start with upper case 'S' -SLANG_COMPILE_TIME_ASSERT(SLANG_FOUR_CC_GET_FIRST_CHAR(IRSerialBinary::kInstFourCc) == 'S'); -SLANG_COMPILE_TIME_ASSERT(SLANG_FOUR_CC_GET_FIRST_CHAR(IRSerialBinary::kChildRunFourCc) == 'S'); -SLANG_COMPILE_TIME_ASSERT( - SLANG_FOUR_CC_GET_FIRST_CHAR(IRSerialBinary::kExternalOperandsFourCc) == 'S'); - -// Compressed version starts with 's' -SLANG_COMPILE_TIME_ASSERT( - SLANG_FOUR_CC_GET_FIRST_CHAR(SLANG_MAKE_COMPRESSED_FOUR_CC(IRSerialBinary::kInstFourCc)) == - 's'); - struct PrefixString; // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! IRSerialData !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! diff --git a/source/slang/slang-serialize-ir-types.h b/source/slang/slang-serialize-ir-types.h index 55a249f5b..b3daf1772 100644 --- a/source/slang/slang-serialize-ir-types.h +++ b/source/slang/slang-serialize-ir-types.h @@ -29,11 +29,6 @@ struct IRSerialBinary static const FourCC kChildRunFourCc = SLANG_FOUR_CC('S', 'L', 'c', 'r'); static const FourCC kExternalOperandsFourCc = SLANG_FOUR_CC('S', 'L', 'e', 'o'); - static const FourCC kCompressedInstFourCc = SLANG_MAKE_COMPRESSED_FOUR_CC(kInstFourCc); - static const FourCC kCompressedChildRunFourCc = SLANG_MAKE_COMPRESSED_FOUR_CC(kChildRunFourCc); - static const FourCC kCompressedExternalOperandsFourCc = - SLANG_MAKE_COMPRESSED_FOUR_CC(kExternalOperandsFourCc); - static const FourCC kUInt32RawSourceLocFourCc = SLANG_FOUR_CC('S', 'r', 's', '4'); /// Debug information is held elsewhere, but if this optional section exists, it maps diff --git a/source/slang/slang-serialize-ir.cpp b/source/slang/slang-serialize-ir.cpp index b8a1183b3..bdea2faaa 100644 --- a/source/slang/slang-serialize-ir.cpp +++ b/source/slang/slang-serialize-ir.cpp @@ -331,111 +331,7 @@ Result IRSerialWriter::write( return SLANG_OK; } -Result _encodeInsts( - SerialCompressionType compressionType, - const List& instsIn, - List& encodeArrayOut) -{ - typedef IRSerialData::Inst::PayloadType PayloadType; - - if (compressionType != SerialCompressionType::VariableByteLite) - { - return SLANG_FAIL; - } - - encodeArrayOut.clear(); - - const size_t numInsts = size_t(instsIn.getCount()); - const IRSerialData::Inst* insts = instsIn.begin(); - - uint8_t* encodeOut = encodeArrayOut.begin(); - uint8_t* encodeEnd = encodeArrayOut.end(); - - // Calculate the maximum instruction size with worst case possible encoding - // 2 bytes hold the payload size, and the result type - // Note that if there were some free bits, we could encode some of this stuff into bits, but if - // we remove payloadType, then there are no free bits - const size_t maxInstSize = 1 + ByteEncodeUtil::kMaxLiteEncodeUInt16 + - Math::Max( - sizeof(insts->m_payload.m_float64), - size_t(2 * ByteEncodeUtil::kMaxLiteEncodeUInt32)); - - for (size_t i = 0; i < numInsts; ++i) - { - const auto& inst = insts[i]; - - // Make sure there is space for the largest possible instruction - if (encodeOut + maxInstSize >= encodeEnd) - { - const size_t offset = size_t(encodeOut - encodeArrayOut.begin()); - - const UInt oldCapacity = encodeArrayOut.getCapacity(); - - encodeArrayOut.reserve(oldCapacity + (oldCapacity >> 1) + maxInstSize); - const UInt capacity = encodeArrayOut.getCapacity(); - encodeArrayOut.setCount(capacity); - - encodeOut = encodeArrayOut.begin() + offset; - encodeEnd = encodeArrayOut.end(); - } - encodeOut += ByteEncodeUtil::encodeLiteUInt32(inst.m_op, encodeOut); - - *encodeOut++ = uint8_t(inst.m_payloadType); - - encodeOut += ByteEncodeUtil::encodeLiteUInt32((uint32_t)inst.m_resultTypeIndex, encodeOut); - - switch (inst.m_payloadType) - { - case PayloadType::Empty: - { - break; - } - case PayloadType::Operand_1: - case PayloadType::String_1: - case PayloadType::UInt32: - { - // 1 UInt32 - encodeOut += ByteEncodeUtil::encodeLiteUInt32( - (uint32_t)inst.m_payload.m_operands[0], - encodeOut); - break; - } - case PayloadType::Operand_2: - case PayloadType::OperandAndUInt32: - case PayloadType::OperandExternal: - case PayloadType::String_2: - { - // 2 UInt32 - encodeOut += ByteEncodeUtil::encodeLiteUInt32( - (uint32_t)inst.m_payload.m_operands[0], - encodeOut); - encodeOut += ByteEncodeUtil::encodeLiteUInt32( - (uint32_t)inst.m_payload.m_operands[1], - encodeOut); - break; - } - case PayloadType::Float64: - { - memcpy(encodeOut, &inst.m_payload.m_float64, sizeof(inst.m_payload.m_float64)); - encodeOut += sizeof(inst.m_payload.m_float64); - break; - } - case PayloadType::Int64: - { - memcpy(encodeOut, &inst.m_payload.m_int64, sizeof(inst.m_payload.m_int64)); - encodeOut += sizeof(inst.m_payload.m_int64); - break; - } - } - } - - // Fix the size - encodeArrayOut.setCount(UInt(encodeOut - encodeArrayOut.begin())); - return SLANG_OK; -} - Result _writeInstArrayChunk( - SerialCompressionType compressionType, FourCC chunkId, const List& array, RiffContainer* container) @@ -448,37 +344,11 @@ Result _writeInstArrayChunk( return SLANG_OK; } - switch (compressionType) - { - case SerialCompressionType::None: - { - return SerialRiffUtil::writeArrayChunk(compressionType, chunkId, array, container); - } - case SerialCompressionType::VariableByteLite: - { - List compressedPayload; - SLANG_RETURN_ON_FAIL(_encodeInsts(compressionType, array, compressedPayload)); - - ScopeChunk scope(container, Chunk::Kind::Data, SLANG_MAKE_COMPRESSED_FOUR_CC(chunkId)); - - SerialBinary::CompressedArrayHeader header; - header.numEntries = uint32_t(array.getCount()); - header.numCompressedEntries = 0; - - container->write(&header, sizeof(header)); - container->write(compressedPayload.getBuffer(), compressedPayload.getCount()); - - return SLANG_OK; - } - default: - break; - } - return SLANG_FAIL; + return SerialRiffUtil::writeArrayChunk(chunkId, array, container); } /* static */ Result IRSerialWriter::writeContainer( const IRSerialData& data, - SerialCompressionType compressionType, RiffContainer* container) { typedef RiffContainer::Chunk Chunk; @@ -486,26 +356,19 @@ Result _writeInstArrayChunk( ScopeChunk scopeModule(container, Chunk::Kind::List, Bin::kIRModuleFourCc); + SLANG_RETURN_ON_FAIL(_writeInstArrayChunk(Bin::kInstFourCc, data.m_insts, container)); SLANG_RETURN_ON_FAIL( - _writeInstArrayChunk(compressionType, Bin::kInstFourCc, data.m_insts, container)); - SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayChunk( - compressionType, - Bin::kChildRunFourCc, - data.m_childRuns, - container)); + SerialRiffUtil::writeArrayChunk(Bin::kChildRunFourCc, data.m_childRuns, container)); SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayChunk( - compressionType, Bin::kExternalOperandsFourCc, data.m_externalOperands, container)); SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayChunk( - SerialCompressionType::None, SerialBinary::kStringTableFourCc, data.m_stringTable, container)); SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayChunk( - SerialCompressionType::None, Bin::kUInt32RawSourceLocFourCc, data.m_rawSourceLocs, container)); @@ -513,7 +376,6 @@ Result _writeInstArrayChunk( if (data.m_debugSourceLocRuns.getCount()) { SerialRiffUtil::writeArrayChunk( - compressionType, Bin::kDebugSourceLocRunFourCc, data.m_debugSourceLocRuns, container); @@ -555,133 +417,16 @@ Result _writeInstArrayChunk( // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! IRSerialReader !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -static Result _decodeInsts( - SerialCompressionType compressionType, - const uint8_t* encodeCur, - size_t encodeInSize, - List& instsOut) -{ - const uint8_t* encodeEnd = encodeCur + encodeInSize; - - typedef IRSerialData::Inst::PayloadType PayloadType; - - if (compressionType != SerialCompressionType::VariableByteLite) - { - return SLANG_FAIL; - } - - const size_t numInsts = size_t(instsOut.getCount()); - IRSerialData::Inst* insts = instsOut.begin(); - - for (size_t i = 0; i < numInsts; ++i) - { - if (encodeCur >= encodeEnd) - { - SLANG_ASSERT(!"Invalid decode"); - return SLANG_FAIL; - } - - auto& inst = insts[i]; - uint32_t instOp = 0; - encodeCur += ByteEncodeUtil::decodeLiteUInt32(encodeCur, &instOp); - inst.m_op = (uint16_t)instOp; - - const PayloadType payloadType = PayloadType(*encodeCur++); - inst.m_payloadType = payloadType; - - // Read the result value - encodeCur += - ByteEncodeUtil::decodeLiteUInt32(encodeCur, (uint32_t*)&inst.m_resultTypeIndex); - - switch (inst.m_payloadType) - { - case PayloadType::Empty: - { - break; - } - case PayloadType::Operand_1: - case PayloadType::String_1: - case PayloadType::UInt32: - { - // 1 UInt32 - encodeCur += ByteEncodeUtil::decodeLiteUInt32( - encodeCur, - (uint32_t*)&inst.m_payload.m_operands[0]); - break; - } - case PayloadType::Operand_2: - case PayloadType::OperandAndUInt32: - case PayloadType::OperandExternal: - case PayloadType::String_2: - { - // 2 UInt32 - encodeCur += ByteEncodeUtil::decodeLiteUInt32( - encodeCur, - 2, - (uint32_t*)&inst.m_payload.m_operands[0]); - break; - } - case PayloadType::Float64: - { - memcpy(&inst.m_payload.m_float64, encodeCur, sizeof(inst.m_payload.m_float64)); - encodeCur += sizeof(inst.m_payload.m_float64); - break; - } - case PayloadType::Int64: - { - memcpy(&inst.m_payload.m_int64, encodeCur, sizeof(inst.m_payload.m_int64)); - encodeCur += sizeof(inst.m_payload.m_int64); - break; - } - } - } - - return SLANG_OK; -} - static Result _readInstArrayChunk( - SerialCompressionType containerCompressionType, RiffContainer::DataChunk* chunk, List& arrayOut) { - SerialCompressionType compressionType = SerialCompressionType::None; - if (chunk->m_fourCC == SLANG_MAKE_COMPRESSED_FOUR_CC(chunk->m_fourCC)) - { - compressionType = SerialCompressionType(containerCompressionType); - } - - switch (compressionType) - { - case SerialCompressionType::None: - { - SerialRiffUtil::ListResizerForType resizer(arrayOut); - return SerialRiffUtil::readArrayChunk(compressionType, chunk, resizer); - } - case SerialCompressionType::VariableByteLite: - { - RiffReadHelper read = chunk->asReadHelper(); - - SerialBinary::CompressedArrayHeader header; - SLANG_RETURN_ON_FAIL(read.read(header)); - - arrayOut.setCount(header.numEntries); - - SLANG_RETURN_ON_FAIL( - _decodeInsts(compressionType, read.getData(), read.getRemainingSize(), arrayOut)); - break; - } - default: - { - return SLANG_FAIL; - } - } - - return SLANG_OK; + SerialRiffUtil::ListResizerForType resizer(arrayOut); + return SerialRiffUtil::readArrayChunk(chunk, resizer); } /* static */ Result IRSerialReader::readContainer( RiffContainer::ListChunk* module, - SerialCompressionType containerCompressionType, IRSerialData* outData) { typedef IRSerialBinary Bin; @@ -698,51 +443,39 @@ static Result _readInstArrayChunk( switch (dataChunk->m_fourCC) { - case SLANG_MAKE_COMPRESSED_FOUR_CC(Bin::kInstFourCc): case Bin::kInstFourCc: { - SLANG_RETURN_ON_FAIL( - _readInstArrayChunk(containerCompressionType, dataChunk, outData->m_insts)); + SLANG_RETURN_ON_FAIL(_readInstArrayChunk(dataChunk, outData->m_insts)); break; } - case SLANG_MAKE_COMPRESSED_FOUR_CC(Bin::kChildRunFourCc): case Bin::kChildRunFourCc: { - SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayChunk( - containerCompressionType, - dataChunk, - outData->m_childRuns)); + SLANG_RETURN_ON_FAIL( + SerialRiffUtil::readArrayChunk(dataChunk, outData->m_childRuns)); break; } - case SLANG_MAKE_COMPRESSED_FOUR_CC(Bin::kExternalOperandsFourCc): case Bin::kExternalOperandsFourCc: { - SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayChunk( - containerCompressionType, - dataChunk, - outData->m_externalOperands)); + SLANG_RETURN_ON_FAIL( + SerialRiffUtil::readArrayChunk(dataChunk, outData->m_externalOperands)); break; } case SerialBinary::kStringTableFourCc: { SLANG_RETURN_ON_FAIL( - SerialRiffUtil::readArrayUncompressedChunk(dataChunk, outData->m_stringTable)); + SerialRiffUtil::readArrayChunk(dataChunk, outData->m_stringTable)); break; } case Bin::kUInt32RawSourceLocFourCc: { - SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayUncompressedChunk( - dataChunk, - outData->m_rawSourceLocs)); + SLANG_RETURN_ON_FAIL( + SerialRiffUtil::readArrayChunk(dataChunk, outData->m_rawSourceLocs)); break; } - case SLANG_MAKE_COMPRESSED_FOUR_CC(Bin::kDebugSourceLocRunFourCc): case Bin::kDebugSourceLocRunFourCc: { - SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayChunk( - containerCompressionType, - dataChunk, - outData->m_debugSourceLocRuns)); + SLANG_RETURN_ON_FAIL( + SerialRiffUtil::readArrayChunk(dataChunk, outData->m_debugSourceLocRuns)); break; } default: diff --git a/source/slang/slang-serialize-ir.h b/source/slang/slang-serialize-ir.h index 10f8bf08d..96121516b 100644 --- a/source/slang/slang-serialize-ir.h +++ b/source/slang/slang-serialize-ir.h @@ -26,10 +26,7 @@ struct IRSerialWriter IRSerialData* serialData); /// Write to a container - static Result writeContainer( - const IRSerialData& data, - SerialCompressionType compressionType, - RiffContainer* container); + static Result writeContainer(const IRSerialData& data, RiffContainer* container); /// Get an instruction index from an instruction Ser::InstIndex getInstIndex(IRInst* inst) const @@ -96,10 +93,7 @@ struct IRSerialReader typedef IRSerialData Ser; /// Read a stream to fill in dataOut IRSerialData - static Result readContainer( - RiffContainer::ListChunk* module, - SerialCompressionType containerCompressionType, - IRSerialData* outData); + static Result readContainer(RiffContainer::ListChunk* module, IRSerialData* outData); /// Read a module from serial data Result read( diff --git a/source/slang/slang-serialize-source-loc.cpp b/source/slang/slang-serialize-source-loc.cpp index 0c6f8996c..b24324048 100644 --- a/source/slang/slang-serialize-source-loc.cpp +++ b/source/slang/slang-serialize-source-loc.cpp @@ -385,29 +385,26 @@ SlangResult SerialSourceLocReader::read( /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DebugSerialData !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ -/* static */ Result SerialSourceLocData::writeContainer( - SerialCompressionType moduleCompressionType, - RiffContainer* container) +/* static */ Result SerialSourceLocData::writeContainer(RiffContainer* container) { RiffContainer::ScopeChunk debugChunkScope( container, RiffContainer::Chunk::Kind::List, SerialSourceLocData::kDebugFourCc); - SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayUncompressedChunk( + SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayChunk( SerialSourceLocData::kDebugStringFourCc, m_stringTable, container)); - SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayUncompressedChunk( + SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayChunk( SerialSourceLocData::kDebugLineInfoFourCc, m_lineInfos, container)); - SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayUncompressedChunk( + SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayChunk( SerialSourceLocData::kDebugAdjustedLineInfoFourCc, m_adjustedLineInfos, container)); SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayChunk( - moduleCompressionType, SerialSourceLocData::kDebugSourceInfoFourCc, m_sourceInfos, container)); @@ -415,9 +412,7 @@ SlangResult SerialSourceLocReader::read( return SLANG_OK; } -/* static */ Result SerialSourceLocData::readContainer( - SerialCompressionType moduleCompressionType, - RiffContainer::ListChunk* listChunk) +/* static */ Result SerialSourceLocData::readContainer(RiffContainer::ListChunk* listChunk) { SLANG_ASSERT(listChunk->getSubType() == SerialSourceLocData::kDebugFourCc); @@ -434,29 +429,23 @@ SlangResult SerialSourceLocReader::read( { case SerialSourceLocData::kDebugStringFourCc: { - SLANG_RETURN_ON_FAIL( - SerialRiffUtil::readArrayUncompressedChunk(dataChunk, m_stringTable)); + SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayChunk(dataChunk, m_stringTable)); break; } case SerialSourceLocData::kDebugLineInfoFourCc: { - SLANG_RETURN_ON_FAIL( - SerialRiffUtil::readArrayUncompressedChunk(dataChunk, m_lineInfos)); + SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayChunk(dataChunk, m_lineInfos)); break; } case SerialSourceLocData::kDebugAdjustedLineInfoFourCc: { SLANG_RETURN_ON_FAIL( - SerialRiffUtil::readArrayUncompressedChunk(dataChunk, m_adjustedLineInfos)); + SerialRiffUtil::readArrayChunk(dataChunk, m_adjustedLineInfos)); break; } - case SLANG_MAKE_COMPRESSED_FOUR_CC(SerialSourceLocData::kDebugSourceInfoFourCc): case SerialSourceLocData::kDebugSourceInfoFourCc: { - SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayChunk( - moduleCompressionType, - dataChunk, - m_sourceInfos)); + SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayChunk(dataChunk, m_sourceInfos)); break; } } diff --git a/source/slang/slang-serialize-source-loc.h b/source/slang/slang-serialize-source-loc.h index f63b602f6..10d084fb6 100644 --- a/source/slang/slang-serialize-source-loc.h +++ b/source/slang/slang-serialize-source-loc.h @@ -135,10 +135,8 @@ public: bool operator==(const ThisType& rhs) const; - Result writeContainer(SerialCompressionType moduleCompressionType, RiffContainer* container); - Result readContainer( - SerialCompressionType moduleCompressionType, - RiffContainer::ListChunk* listChunk); + Result writeContainer(RiffContainer* container); + Result readContainer(RiffContainer::ListChunk* listChunk); List m_stringTable; ///< String table for debug use only List m_lineInfos; ///< Line information diff --git a/source/slang/slang-serialize-types.cpp b/source/slang/slang-serialize-types.cpp index ea6cd9c34..2e6e56bb4 100644 --- a/source/slang/slang-serialize-types.cpp +++ b/source/slang/slang-serialize-types.cpp @@ -145,7 +145,6 @@ struct ByteReader // !!!!!!!!!!!!!!!!!!!!!!!!!!!! SerialRiffUtil !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! /* static */ Result SerialRiffUtil::writeArrayChunk( - SerialCompressionType compressionType, FourCC chunkId, const void* data, size_t numEntries, @@ -160,52 +159,17 @@ struct ByteReader return SLANG_OK; } - // Make compressed fourCC - chunkId = (compressionType != SerialCompressionType::None) - ? SLANG_MAKE_COMPRESSED_FOUR_CC(chunkId) - : chunkId; - ScopeChunk scope(container, Chunk::Kind::Data, chunkId); - switch (compressionType) - { - case SerialCompressionType::None: - { - SerialBinary::ArrayHeader header; - header.numEntries = uint32_t(numEntries); - - container->write(&header, sizeof(header)); - container->write(data, typeSize * numEntries); - break; - } - case SerialCompressionType::VariableByteLite: - { - List compressedPayload; - - size_t numCompressedEntries = (numEntries * typeSize) / sizeof(uint32_t); - ByteEncodeUtil::encodeLiteUInt32( - (const uint32_t*)data, - numCompressedEntries, - compressedPayload); - - SerialBinary::CompressedArrayHeader header; - header.numEntries = uint32_t(numEntries); - header.numCompressedEntries = uint32_t(numCompressedEntries); - - container->write(&header, sizeof(header)); - container->write(compressedPayload.getBuffer(), compressedPayload.getCount()); - break; - } - default: - { - return SLANG_FAIL; - } - } + SerialBinary::ArrayHeader header; + header.numEntries = uint32_t(numEntries); + + container->write(&header, sizeof(header)); + container->write(data, typeSize * numEntries); return SLANG_OK; } /* static */ Result SerialRiffUtil::readArrayChunk( - SerialCompressionType compressionType, RiffContainer::DataChunk* dataChunk, ListResizer& listOut) { @@ -214,90 +178,14 @@ struct ByteReader RiffReadHelper read = dataChunk->asReadHelper(); const size_t typeSize = listOut.getTypeSize(); - switch (compressionType) - { - case SerialCompressionType::VariableByteLite: - { - Bin::CompressedArrayHeader header; - SLANG_RETURN_ON_FAIL(read.read(header)); - - void* dst = listOut.setSize(header.numEntries); - SLANG_ASSERT( - header.numCompressedEntries == - uint32_t((header.numEntries * typeSize) / sizeof(uint32_t))); - - // Decode.. - ByteEncodeUtil::decodeLiteUInt32( - read.getData(), - header.numCompressedEntries, - (uint32_t*)dst); - break; - } - case SerialCompressionType::None: - { - // Read uncompressed - Bin::ArrayHeader header; - SLANG_RETURN_ON_FAIL(read.read(header)); - const size_t payloadSize = header.numEntries * typeSize; - SLANG_ASSERT(payloadSize == read.getRemainingSize()); - void* dst = listOut.setSize(header.numEntries); - ::memcpy(dst, read.getData(), payloadSize); - break; - } - } - return SLANG_OK; -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!! SerialParseUtil !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -// clang-format off -#define SLANG_SERIAL_BINARY_COMPRESSION_TYPE(x) \ - x(None, none) \ - x(VariableByteLite, lite) -// clang-format on - -/* static */ SlangResult SerialParseUtil::parseCompressionType( - const UnownedStringSlice& text, - SerialCompressionType& outType) -{ - struct Pair - { - UnownedStringSlice name; - SerialCompressionType type; - }; - - // clang-format off -#define SLANG_SERIAL_BINARY_PAIR(type, name) \ - {UnownedStringSlice::fromLiteral(#name), SerialCompressionType::type}, - // clang-format on - - static const Pair s_pairs[] = {SLANG_SERIAL_BINARY_COMPRESSION_TYPE(SLANG_SERIAL_BINARY_PAIR)}; - - for (const auto& pair : s_pairs) - { - if (pair.name == text) - { - outType = pair.type; - return SLANG_OK; - } - } - return SLANG_FAIL; -} + Bin::ArrayHeader header; + SLANG_RETURN_ON_FAIL(read.read(header)); + const size_t payloadSize = header.numEntries * typeSize; + SLANG_ASSERT(payloadSize == read.getRemainingSize()); + void* dst = listOut.setSize(header.numEntries); + ::memcpy(dst, read.getData(), payloadSize); -/* static */ UnownedStringSlice SerialParseUtil::getText(SerialCompressionType type) -{ -#define SLANG_SERIAL_BINARY_CASE(type, name) \ - case SerialCompressionType::type: \ - return UnownedStringSlice::fromLiteral(#name); - switch (type) - { - SLANG_SERIAL_BINARY_COMPRESSION_TYPE(SLANG_SERIAL_BINARY_CASE) - default: - break; - } - SLANG_ASSERT(!"Unknown compression type"); - return UnownedStringSlice::fromLiteral("unknown"); + return SLANG_OK; } - } // namespace Slang diff --git a/source/slang/slang-serialize-types.h b/source/slang/slang-serialize-types.h index 87180df79..217c14b44 100644 --- a/source/slang/slang-serialize-types.h +++ b/source/slang/slang-serialize-types.h @@ -40,16 +40,6 @@ struct SerialOptionFlag }; typedef SerialOptionFlag::Type SerialOptionFlags; - -// Compression styles - -enum class SerialCompressionType : uint8_t -{ - None, - VariableByteLite, -}; - - struct SerialStringData { enum class StringIndex : uint32_t; @@ -93,16 +83,6 @@ struct SerialStringTableUtil List& indexMap); }; -struct SerialParseUtil -{ - /// Given text, finds the compression type - static SlangResult parseCompressionType( - const UnownedStringSlice& text, - SerialCompressionType& outType); - /// Given a compression type, return text - static UnownedStringSlice getText(SerialCompressionType type); -}; - struct SerialListUtil { template @@ -160,31 +140,15 @@ struct SerialBinary /// An entry point static const FourCC kEntryPointFourCc = SLANG_FOUR_CC('E', 'P', 'n', 't'); - /// Container - static const FourCC kContainerHeaderFourCc = SLANG_FOUR_CC('S', 'c', 'h', 'd'); - // Module header static const FourCC kModuleHeaderFourCc = SLANG_FOUR_CC('S', 'm', 'h', 'd'); - struct ContainerHeader - { - uint32_t compressionType; ///< Holds the compression type used (if used at all) - }; - struct ArrayHeader { uint32_t numEntries; }; - struct CompressedArrayHeader - { - uint32_t numEntries; ///< The number of entries - uint32_t numCompressedEntries; ///< The amount of compressed entries - }; }; -// Replace first char with 's' -#define SLANG_MAKE_COMPRESSED_FOUR_CC(fourCc) SLANG_FOUR_CC_REPLACE_FIRST_CHAR(fourCc, 's') - struct SerialRiffUtil { class ListResizer @@ -223,7 +187,6 @@ struct SerialRiffUtil }; static Result writeArrayChunk( - SerialCompressionType compressionType, FourCC chunkId, const void* data, size_t numEntries, @@ -231,14 +194,9 @@ struct SerialRiffUtil RiffContainer* container); template - static Result writeArrayChunk( - SerialCompressionType compressionType, - FourCC chunkId, - const List& array, - RiffContainer* container) + static Result writeArrayChunk(FourCC chunkId, const List& array, RiffContainer* container) { return writeArrayChunk( - compressionType, chunkId, array.begin(), size_t(array.getCount()), @@ -246,47 +204,13 @@ struct SerialRiffUtil container); } - template - static Result writeArrayUncompressedChunk( - FourCC chunkId, - const List& array, - RiffContainer* container) - { - return writeArrayChunk( - SerialCompressionType::None, - chunkId, - array.begin(), - size_t(array.getCount()), - sizeof(T), - container); - } - - static Result readArrayChunk( - SerialCompressionType compressionType, - RiffContainer::DataChunk* dataChunk, - ListResizer& listOut); - - template - static Result readArrayChunk( - SerialCompressionType moduleCompressionType, - RiffContainer::DataChunk* dataChunk, - List& arrayOut) - { - SerialCompressionType compressionType = SerialCompressionType::None; - if (dataChunk->m_fourCC == SLANG_MAKE_COMPRESSED_FOUR_CC(dataChunk->m_fourCC)) - { - // If it has compression, use the compression type set in the header - compressionType = moduleCompressionType; - } - ListResizerForType resizer(arrayOut); - return readArrayChunk(compressionType, dataChunk, resizer); - } + static Result readArrayChunk(RiffContainer::DataChunk* dataChunk, ListResizer& listOut); template - static Result readArrayUncompressedChunk(RiffContainer::DataChunk* chunk, List& arrayOut) + static Result readArrayChunk(RiffContainer::DataChunk* dataChunk, List& arrayOut) { ListResizerForType resizer(arrayOut); - return readArrayChunk(SerialCompressionType::None, chunk, resizer); + return readArrayChunk(dataChunk, resizer); } }; diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp index 4ebe02736..351ab6f06 100644 --- a/source/slang/slang.cpp +++ b/source/slang/slang.cpp @@ -3486,7 +3486,6 @@ void FrontEndCompileRequest::generateIR() { SerialContainerUtil::WriteOptions options; - options.compressionType = SerialCompressionType::None; options.sourceManager = getSourceManager(); options.optionFlags |= SerialOptionFlag::SourceLocation; -- cgit v1.2.3