diff options
23 files changed, 3110 insertions, 2514 deletions
diff --git a/source/core/slang-riff.cpp b/source/core/slang-riff.cpp index 9fc23fbc8..2162c40be 100644 --- a/source/core/slang-riff.cpp +++ b/source/core/slang-riff.cpp @@ -313,6 +313,11 @@ struct DumpVisitor : public RiffContainer::Visitor return SLANG_OK; } +/* static */SlangResult RiffUtil::write(RiffContainer* container, Stream* stream) +{ + return write(container->getRoot(), true, stream); +} + /* static */SlangResult RiffUtil::read(Stream* stream, RiffContainer& outContainer) { typedef RiffContainer::ScopeChunk ScopeChunk; @@ -561,6 +566,20 @@ void RiffContainer::ListChunk::findContained(FourCC type, List<DataChunk*>& out) } } +RiffContainer::ListChunk* RiffContainer::ListChunk::findContainedList(FourCC type) +{ + Chunk* chunk = m_containedChunks; + while (chunk) + { + if (chunk->m_fourCC == type && chunk->m_kind == Chunk::Kind::List) + { + return static_cast<ListChunk*>(chunk); + } + chunk = chunk->m_next; + } + return nullptr; +} + RiffContainer::Data* RiffContainer::ListChunk::findContainedData(FourCC type) const { Chunk* found = findContained(type); @@ -841,11 +860,11 @@ void RiffContainer::setPayload(Data* data, const void* payload, size_t size) // Add current chunks data m_dataChunk->m_payloadSize += size; - + data->m_ownership = Ownership::Arena; data->m_size = size; - data->m_payload = m_arena.allocate(size); + data->m_payload = m_arena.allocateAligned(size, PAYLOAD_MIN_ALIGNMENT); if (payload) { @@ -922,7 +941,7 @@ RiffContainer::Data* RiffContainer::makeSingleData(DataChunk* dataChunk) // Okay lets combine all into one block const size_t payloadSize = dataChunk->calcPayloadSize(); - void* dst = m_arena.allocate(payloadSize); + void* dst = m_arena.allocateAligned(payloadSize, PAYLOAD_MIN_ALIGNMENT); dataChunk->getPayload(dst); // Remove other datas diff --git a/source/core/slang-riff.h b/source/core/slang-riff.h index 9bdbcfdeb..43c5e285d 100644 --- a/source/core/slang-riff.h +++ b/source/core/slang-riff.h @@ -177,6 +177,14 @@ need to be recalculated, before serialization. class RiffContainer { public: + enum + { + // This alignment is only made for arena based allocations. + // For external blocks it's client code to have appropriate alignment. + // This is needed because when reading a RiffContainer, all allocation is arena based, and + // if the payload contains 8 byte aligned data, the overall payload needs to be 8 byte aligned. + PAYLOAD_MIN_ALIGNMENT = 8, + }; enum class Ownership { @@ -186,10 +194,6 @@ public: Owned, ///< It's owned, but wasn't allocated on the arena }; - struct ListChunk; - struct DataChunk; - - struct Data { /// Get the payload @@ -214,8 +218,11 @@ public: }; struct Chunk; - typedef SlangResult(*VisitorCallback)(Chunk* chunk, void* data); + struct ListChunk; + struct DataChunk; + typedef SlangResult(*VisitorCallback)(Chunk* chunk, void* data); + class Visitor; struct Chunk { @@ -270,6 +277,8 @@ public: void* findContainedData(FourCC type, size_t minSize) const; + ListChunk* findContainedList(FourCC type); + /// Finds the contained data. NOTE! Assumes that there is only as single data block, and will return nullptr if there is not Data* findContainedData(FourCC type) const; @@ -291,6 +300,9 @@ public: /// Get the sub type FourCC getSubType() const { return m_fourCC; } + /// A singly linked list of contained chunks directly contained in this chunk + Chunk* getFirstContainedChunk() const { return m_containedChunks; } + Chunk* m_containedChunks; ///< The contained chunks Chunk* m_endChunk; ///< The last chunk (only set when pushed, and used when popped) }; @@ -416,9 +428,14 @@ protected: template <typename T> T* as(RiffContainer::Chunk* chunk) { - return T::isType(chunk) ? static_cast<T*>(chunk) : nullptr; + return chunk && T::isType(chunk) ? static_cast<T*>(chunk) : nullptr; +} +// ----------------------------------------------------------------------------- +template <typename T> +T* as(RiffContainer::Chunk* chunk, FourCC fourCC) +{ + return chunk && chunk->m_fourCC == fourCC && T::isType(chunk) ? static_cast<T*>(chunk) : nullptr; } - struct RiffUtil { @@ -449,8 +466,11 @@ struct RiffUtil /// Get the size taking into account padding static size_t getPadSize(size_t in) { return (in + kRiffPadMask) & ~size_t(kRiffPadMask); } - /// Write a container and contents to a stream - static SlangResult write(ListChunk* container, bool isRoot, Stream* stream); + /// Write a chunk list and contents to a stream + static SlangResult write(ListChunk* listChunk, bool isRoot, Stream* stream); + /// Write a container to the stream + static SlangResult write(RiffContainer* container, Stream* stream); + /// Read the stream into the container static SlangResult read(Stream* stream, RiffContainer& outContainer); }; diff --git a/source/slang/slang-compiler.cpp b/source/slang/slang-compiler.cpp index 37a79e6c4..d8da642f4 100755 --- a/source/slang/slang-compiler.cpp +++ b/source/slang/slang-compiler.cpp @@ -23,9 +23,7 @@ #include "slang-glsl-extension-tracker.h" #include "slang-emit-cuda.h" -#include "slang-ast-serialize.h" - -#include "slang-ir-serialize.h" +#include "slang-serialize-container.h" // Enable calling through to `fxc` or `dxc` to // generate code on Windows. @@ -143,6 +141,13 @@ namespace Slang { if (downstreamResult) { + // TODO(JS): + // This seems a little questionable. As it stands downstreamResult, if it doesn't have a blob + // can try and read a file. How this currently works is that every getBlob will potentially try to read that file. + // Setting result to None would stop this, but is that reasonable as the state. + // Perhaps downstreamResult should hold some state that the read failed. + // For now we don't worry though. + SLANG_RETURN_ON_FAIL(downstreamResult->getBinary(blob)); } break; @@ -2342,104 +2347,28 @@ SlangResult dissassembleDXILUsingDXC( SlangResult EndToEndCompileRequest::writeContainerToStream(Stream* stream) { - RiffContainer container; + auto linkage = getLinkage(); - const IRSerialBinary::CompressionType compressionType = getLinkage()->irCompressionType; + // Set up options + SerialContainerUtil::WriteOptions options; + options.compressionType = linkage->serialCompressionType; + if (linkage->debugInfoLevel != DebugInfoLevel::None) { - // Module list - RiffContainer::ScopeChunk listScope(&container, RiffContainer::Chunk::Kind::List, SerialBinary::kSlangModuleListFourCc); - - auto linkage = getLinkage(); - auto sink = getSink(); - auto frontEndReq = getFrontEndReq(); - - IRSerialWriter::OptionFlags optionFlags = 0; - - if (linkage->debugInfoLevel != DebugInfoLevel::None) - { - optionFlags |= IRSerialWriter::OptionFlag::DebugInfo; - } - - RefPtr<ASTSerialClasses> astClasses = new ASTSerialClasses; - - SourceManager* sourceManager = frontEndReq->getSourceManager(); - - for (TranslationUnitRequest* translationUnit : frontEndReq->translationUnits) - { - auto module = translationUnit->module; - auto irModule = module->getIRModule(); - - // Okay, we need to serialize this module to our container file. - // We currently don't serialize it's name..., but support for that could be added. - - // Write the IR information - { - IRSerialData serialData; - IRSerialWriter writer; - SLANG_RETURN_ON_FAIL(writer.write(irModule, sourceManager, optionFlags, &serialData)); - SLANG_RETURN_ON_FAIL(IRSerialWriter::writeContainer(serialData, compressionType, &container)); - } - - // Write the AST information - { - ModuleDecl* moduleDecl = translationUnit->getModuleDecl(); - - ModuleASTSerialFilter filter(moduleDecl); - ASTSerialWriter writer(astClasses, &filter); - - // Add the module and everything that isn't filtered out in the filter. - writer.addPointer(moduleDecl); - - // We can now serialize it into the riff container. - SLANG_RETURN_ON_FAIL(writer.writeIntoContainer(&container)); - } - } - - auto program = getSpecializedGlobalAndEntryPointsComponentType(); - - // TODO: in the case where we have specialization, we might need - // to serialize IR related to `program`... - - for (auto target : linkage->targets) - { - auto targetProgram = program->getTargetProgram(target); - auto irModule = targetProgram->getOrCreateIRModuleForLayout(sink); - - // Okay, we need to serialize this target program and its IR too... - IRSerialData serialData; - IRSerialWriter writer; - SLANG_RETURN_ON_FAIL(writer.write(irModule, sourceManager, optionFlags, &serialData)); - SLANG_RETURN_ON_FAIL(IRSerialWriter::writeContainer(serialData, compressionType, &container)); - } + options.optionFlags |= SerialOptionFlag::DebugInfo; + } - auto entryPointCount = program->getEntryPointCount(); - for( Index ii = 0; ii < entryPointCount; ++ii ) + { + RiffContainer container; { - auto entryPoint = program->getEntryPoint(ii); - auto entryPointMangledName = program->getEntryPointMangledName(ii); - - RiffContainer::ScopeChunk entryPointScope(&container, RiffContainer::Chunk::Kind::Data, SerialBinary::kEntryPointFourCc); - - auto writeString = [&](String const& str) - { - uint32_t length = (uint32_t) str.getLength(); - container.write(&length, sizeof(length)); - container.write(str.getBuffer(), length+1); - }; - - writeString(entryPoint->getName()->text); - - Profile profile = entryPoint->getProfile(); - container.write(&profile, sizeof(profile)); - - writeString(entryPointMangledName); + SerialContainerData data; + SLANG_RETURN_ON_FAIL(SerialContainerUtil::requestToData(this, options, data)); + SLANG_RETURN_ON_FAIL(SerialContainerUtil::write(data, options, &container)); } + // We now write the RiffContainer to the stream + SLANG_RETURN_ON_FAIL(RiffUtil::write(container.getRoot(), true, stream)); } - // We now write the RiffContainer to the stream - SLANG_RETURN_ON_FAIL(RiffUtil::write(container.getRoot(), true, stream)); - return SLANG_OK; } diff --git a/source/slang/slang-compiler.h b/source/slang/slang-compiler.h index cbd62d2bc..f7a5d98b1 100755 --- a/source/slang/slang-compiler.h +++ b/source/slang/slang-compiler.h @@ -18,7 +18,7 @@ #include "slang-include-system.h" -#include "slang-ir-serialize-types.h" +#include "slang-serialize-ir-types.h" #include "../../slang.h" @@ -50,7 +50,7 @@ namespace Slang ComputeShader, }; - enum class CodeGenTarget + enum class CodeGenTarget { Unknown = SLANG_TARGET_UNKNOWN, None = SLANG_TARGET_NONE, @@ -1375,7 +1375,7 @@ namespace Slang OptimizationLevel optimizationLevel = OptimizationLevel::Default; - IRSerialCompressionType irCompressionType = IRSerialCompressionType::VariableByteLite; + SerialCompressionType serialCompressionType = SerialCompressionType::VariableByteLite; bool m_requireCacheFileSystem = false; bool m_useFalcorCustomSharedKeywordSemantics = false; diff --git a/source/slang/slang-ir-serialize-types.cpp b/source/slang/slang-ir-serialize-types.cpp deleted file mode 100644 index b57c46b58..000000000 --- a/source/slang/slang-ir-serialize-types.cpp +++ /dev/null @@ -1,333 +0,0 @@ -// slang-ir-serialize-types.cpp -#include "slang-ir-serialize-types.h" - -#include "../core/slang-text-io.h" -#include "../core/slang-byte-encode-util.h" - -#include "slang-ir-insts.h" - -#include "../core/slang-math.h" - -namespace Slang { - -// Needed for linkage with some compilers -/* static */ const IRSerialData::StringIndex IRSerialData::kNullStringIndex; -/* static */ const IRSerialData::StringIndex IRSerialData::kEmptyStringIndex; - -/* Note that an IRInst can be derived from, but when it derived from it's new members are IRUse variables, and they in -effect alias over the operands - and reflected in the operand count. There _could_ be other members after these IRUse -variables, but only a few types include extra data, and these do not have any operands: - -* IRConstant - Needs special-case handling -* IRModuleInst - Presumably we can just set to the module pointer on reconstruction - -Note! That on an IRInst there is an IRType* variable (accessed as getFullType()). As it stands it may NOT actually point -to an IRType derived type. Its 'ok' as long as it's an instruction that can be used in the place of the type. So this code does not -bother to check if it's correct, and just casts it. -*/ - -/* static */const IRSerialData::PayloadInfo IRSerialData::s_payloadInfos[int(Inst::PayloadType::CountOf)] = -{ - { 0, 0 }, // Empty - { 1, 0 }, // Operand_1 - { 2, 0 }, // Operand_2 - { 1, 0 }, // OperandAndUInt32, - { 0, 0 }, // OperandExternal - This isn't correct, Operand has to be specially handled - { 0, 1 }, // String_1, - { 0, 2 }, // String_2, - { 0, 0 }, // UInt32, - { 0, 0 }, // Float64, - { 0, 0 } // Int64, -}; - -// Check all compressible chunk ids, start with upper case 'S' -SLANG_COMPILE_TIME_ASSERT(SLANG_FOUR_CC_GET_FIRST_CHAR(IRSerialBinary::kInstFourCc) == 'S'); -SLANG_COMPILE_TIME_ASSERT(SLANG_FOUR_CC_GET_FIRST_CHAR(IRSerialBinary::kChildRunFourCc) == 'S'); -SLANG_COMPILE_TIME_ASSERT(SLANG_FOUR_CC_GET_FIRST_CHAR(IRSerialBinary::kExternalOperandsFourCc) == 'S'); - -// Compressed version starts with 's' -SLANG_COMPILE_TIME_ASSERT(SLANG_FOUR_CC_GET_FIRST_CHAR(SLANG_MAKE_COMPRESSED_FOUR_CC(IRSerialBinary::kInstFourCc)) == 's'); - -struct PrefixString; - -namespace { // anonymous - -struct CharReader -{ - char operator()(int pos) const { SLANG_UNUSED(pos); return *m_pos++; } - CharReader(const char* pos) :m_pos(pos) {} - mutable const char* m_pos; -}; - -} // anonymous - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! StringRepresentationCache !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -StringRepresentationCache::StringRepresentationCache(): - m_stringTable(nullptr) -{ -} - -void StringRepresentationCache::init(const List<char>* stringTable) -{ - m_stringTable = stringTable; - - // Decode the table - m_entries.setCount(StringSlicePool::kDefaultHandlesCount); - SLANG_COMPILE_TIME_ASSERT(StringSlicePool::kDefaultHandlesCount == 2); - - { - Entry& entry = m_entries[0]; - entry.m_numChars = 0; - entry.m_startIndex = 0; - } - { - Entry& entry = m_entries[1]; - entry.m_numChars = 0; - entry.m_startIndex = 0; - } - - { - const char* start = stringTable->begin(); - const char* cur = start; - const char* end = stringTable->end(); - - while (cur < end) - { - CharReader reader(cur); - const int len = GetUnicodePointFromUTF8(reader); - - Entry entry; - entry.m_startIndex = uint32_t(reader.m_pos - start); - entry.m_numChars = len; - - m_entries.add(entry); - - cur = reader.m_pos + len; - } - } - - m_entries.compress(); -} - -UnownedStringSlice StringRepresentationCache::getStringSlice(Handle handle) const -{ - const Entry& entry = m_entries[int(handle)]; - const char* start = m_stringTable->begin(); - - return UnownedStringSlice(start + entry.m_startIndex, int(entry.m_numChars)); -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! SerialStringTableUtil !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -/* static */void SerialStringTableUtil::encodeStringTable(const StringSlicePool& pool, List<char>& stringTable) -{ - // Skip the default handles -> nothing is encoded via them - return encodeStringTable(pool.getAdded(), stringTable); -} - -/* static */void SerialStringTableUtil::encodeStringTable(const ConstArrayView<UnownedStringSlice>& slices, List<char>& stringTable) -{ - stringTable.clear(); - for (const auto& slice : slices) - { - const int len = int(slice.getLength()); - - // We need to write into the the string array - char prefixBytes[6]; - const int numPrefixBytes = EncodeUnicodePointToUTF8(prefixBytes, len); - const Index baseIndex = stringTable.getCount(); - - stringTable.setCount(baseIndex + numPrefixBytes + len); - - char* dst = stringTable.begin() + baseIndex; - - memcpy(dst, prefixBytes, numPrefixBytes); - memcpy(dst + numPrefixBytes, slice.begin(), len); - } -} - -/* static */void SerialStringTableUtil::appendDecodedStringTable(const List<char>& stringTable, List<UnownedStringSlice>& slicesOut) -{ - const char* start = stringTable.begin(); - const char* cur = start; - const char* end = stringTable.end(); - - while (cur < end) - { - CharReader reader(cur); - const int len = GetUnicodePointFromUTF8(reader); - slicesOut.add(UnownedStringSlice(reader.m_pos, len)); - cur = reader.m_pos + len; - } -} - -/* static */void SerialStringTableUtil::decodeStringTable(const List<char>& stringTable, List<UnownedStringSlice>& slicesOut) -{ - slicesOut.setCount(2); - slicesOut[0] = UnownedStringSlice(nullptr, size_t(0)); - slicesOut[1] = UnownedStringSlice("", size_t(0)); - - appendDecodedStringTable(stringTable, slicesOut); -} - -/* static */void SerialStringTableUtil::calcStringSlicePoolMap(const List<UnownedStringSlice>& slices, StringSlicePool& pool, List<StringSlicePool::Handle>& indexMapOut) -{ - SLANG_ASSERT(slices.getCount() >= StringSlicePool::kDefaultHandlesCount); - SLANG_ASSERT(slices[int(StringSlicePool::kNullHandle)] == "" && slices[int(StringSlicePool::kNullHandle)].begin() == nullptr); - SLANG_ASSERT(slices[int(StringSlicePool::kEmptyHandle)] == ""); - - indexMapOut.setCount(slices.getCount()); - // Set up all of the defaults - for (int i = 0; i < StringSlicePool::kDefaultHandlesCount; ++i) - { - indexMapOut[i] = StringSlicePool::Handle(i); - } - - const Index numSlices = slices.getCount(); - for (Index i = StringSlicePool::kDefaultHandlesCount; i < numSlices ; ++i) - { - indexMapOut[i] = pool.add(slices[i]); - } -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! IRSerialData !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -template<typename T> -static size_t _calcArraySize(const List<T>& list) -{ - return list.getCount() * sizeof(T); -} - -size_t IRSerialData::calcSizeInBytes() const -{ - return - _calcArraySize(m_insts) + - _calcArraySize(m_childRuns) + - _calcArraySize(m_externalOperands) + - _calcArraySize(m_stringTable) + - /* Raw source locs */ - _calcArraySize(m_rawSourceLocs) + - /* Debug */ - _calcArraySize(m_debugStringTable) + - _calcArraySize(m_debugLineInfos) + - _calcArraySize(m_debugSourceInfos) + - _calcArraySize(m_debugAdjustedLineInfos) + - _calcArraySize(m_debugSourceLocRuns); -} - -IRSerialData::IRSerialData() -{ - clear(); -} - -void IRSerialData::clear() -{ - // First Instruction is null - m_insts.setCount(1); - memset(&m_insts[0], 0, sizeof(Inst)); - - m_childRuns.clear(); - m_externalOperands.clear(); - m_rawSourceLocs.clear(); - - m_stringTable.clear(); - - // Debug data - m_debugLineInfos.clear(); - m_debugAdjustedLineInfos.clear(); - m_debugSourceInfos.clear(); - m_debugSourceLocRuns.clear(); - m_debugStringTable.clear(); -} - -template <typename T> -static bool _isEqual(const List<T>& aIn, const List<T>& bIn) -{ - if (aIn.getCount() != bIn.getCount()) - { - return false; - } - - size_t size = size_t(aIn.getCount()); - - const T* a = aIn.begin(); - const T* b = bIn.begin(); - - if (a == b) - { - return true; - } - - for (size_t i = 0; i < size; ++i) - { - if (a[i] != b[i]) - { - return false; - } - } - - return true; -} - -bool IRSerialData::operator==(const ThisType& rhs) const -{ - return (this == &rhs) || - (_isEqual(m_insts, rhs.m_insts) && - _isEqual(m_childRuns, rhs.m_childRuns) && - _isEqual(m_externalOperands, rhs.m_externalOperands) && - _isEqual(m_rawSourceLocs, rhs.m_rawSourceLocs) && - _isEqual(m_stringTable, rhs.m_stringTable) && - /* Debug */ - _isEqual(m_debugStringTable, rhs.m_debugStringTable) && - _isEqual(m_debugLineInfos, rhs.m_debugLineInfos) && - _isEqual(m_debugAdjustedLineInfos, rhs.m_debugAdjustedLineInfos) && - _isEqual(m_debugSourceInfos, rhs.m_debugSourceInfos) && - _isEqual(m_debugSourceLocRuns, rhs.m_debugSourceLocRuns)); -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!! IRSerialTypeUtil !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -#define SLANG_SERIAL_BINARY_COMPRESSION_TYPE(x) \ - x(None, none) \ - x(VariableByteLite, lite) - -/* static */SlangResult IRSerialTypeUtil::parseCompressionType(const UnownedStringSlice& text, IRSerialCompressionType& outType) -{ - struct Pair - { - UnownedStringSlice name; - IRSerialCompressionType type; - }; - -#define SLANG_SERIAL_BINARY_PAIR(type, name) { UnownedStringSlice::fromLiteral(#name), IRSerialCompressionType::type}, - - static const Pair s_pairs[] = { - SLANG_SERIAL_BINARY_COMPRESSION_TYPE(SLANG_SERIAL_BINARY_PAIR) - }; - - for (const auto& pair : s_pairs) - { - if (pair.name == text) - { - outType = pair.type; - return SLANG_OK; - } - } - return SLANG_FAIL; -} - -/* static */UnownedStringSlice IRSerialTypeUtil::getText(IRSerialCompressionType type) -{ -#define SLANG_SERIAL_BINARY_CASE(type, name) case IRSerialCompressionType::type: return UnownedStringSlice::fromLiteral(#name); - switch (type) - { - SLANG_SERIAL_BINARY_COMPRESSION_TYPE(SLANG_SERIAL_BINARY_CASE) - default: break; - } - - SLANG_ASSERT(!"Unknown compression type"); - return UnownedStringSlice::fromLiteral("unknown"); -} - -} // namespace Slang diff --git a/source/slang/slang-ir-serialize.cpp b/source/slang/slang-ir-serialize.cpp deleted file mode 100644 index 1aa6b1d85..000000000 --- a/source/slang/slang-ir-serialize.cpp +++ /dev/null @@ -1,1589 +0,0 @@ -// slang-ir-serialize.cpp -#include "slang-ir-serialize.h" - -#include "../core/slang-text-io.h" -#include "../core/slang-byte-encode-util.h" - -#include "slang-ir-insts.h" - -#include "../core/slang-math.h" - -namespace Slang { - -static bool _isTextureTypeBase(IROp opIn) -{ - const int op = (kIROpMeta_OpMask & opIn); - return op >= kIROp_FirstTextureTypeBase && op <= kIROp_LastTextureTypeBase; -} - -static bool _isConstant(IROp opIn) -{ - const int op = (kIROpMeta_OpMask & opIn); - return op >= kIROp_FirstConstant && op <= kIROp_LastConstant; -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! IRSerialWriter !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -void IRSerialWriter::_addInstruction(IRInst* inst) -{ - // It cannot already be in the map - SLANG_ASSERT(!m_instMap.ContainsKey(inst)); - - // Add to the map - m_instMap.Add(inst, Ser::InstIndex(m_insts.getCount())); - m_insts.add(inst); -} - -void IRSerialWriter::_addDebugSourceLocRun(SourceLoc sourceLoc, uint32_t startInstIndex, uint32_t numInsts) -{ - SourceView* sourceView = m_sourceManager->findSourceView(sourceLoc); - if (!sourceView) - { - return; - } - - SourceFile* sourceFile = sourceView->getSourceFile(); - DebugSourceFile* debugSourceFile; - { - RefPtr<DebugSourceFile>* ptrDebugSourceFile = m_debugSourceFileMap.TryGetValue(sourceFile); - if (ptrDebugSourceFile == nullptr) - { - const SourceLoc::RawValue baseSourceLoc = m_debugFreeSourceLoc; - m_debugFreeSourceLoc += SourceLoc::RawValue(sourceView->getRange().getSize() + 1); - - debugSourceFile = new DebugSourceFile(sourceFile, baseSourceLoc); - m_debugSourceFileMap.Add(sourceFile, debugSourceFile); - } - else - { - debugSourceFile = *ptrDebugSourceFile; - } - } - - // We need to work out the line index - - int offset = sourceView->getRange().getOffset(sourceLoc); - int lineIndex = sourceFile->calcLineIndexFromOffset(offset); - - IRSerialData::DebugLineInfo lineInfo; - lineInfo.m_lineStartOffset = sourceFile->getLineBreakOffsets()[lineIndex]; - lineInfo.m_lineIndex = lineIndex; - - if (!debugSourceFile->hasLineIndex(lineIndex)) - { - // Add the information about the line - int entryIndex = sourceView->findEntryIndex(sourceLoc); - if (entryIndex < 0) - { - debugSourceFile->m_lineInfos.add(lineInfo); - } - else - { - const auto& entry = sourceView->getEntries()[entryIndex]; - - IRSerialData::DebugAdjustedLineInfo adjustedLineInfo; - adjustedLineInfo.m_lineInfo = lineInfo; - adjustedLineInfo.m_pathStringIndex = Ser::kNullStringIndex; - - const auto& pool = sourceView->getSourceManager()->getStringSlicePool(); - SLANG_ASSERT(pool.getStyle() == StringSlicePool::Style::Default); - - if (!pool.isDefaultHandle(entry.m_pathHandle)) - { - UnownedStringSlice slice = pool.getSlice(entry.m_pathHandle); - SLANG_ASSERT(slice.getLength() > 0); - adjustedLineInfo.m_pathStringIndex = Ser::StringIndex(m_debugStringSlicePool.add(slice)); - } - - adjustedLineInfo.m_adjustedLineIndex = lineIndex + entry.m_lineAdjust; - - debugSourceFile->m_adjustedLineInfos.add(adjustedLineInfo); - } - - debugSourceFile->setHasLineIndex(lineIndex); - } - - // Add the run - IRSerialData::SourceLocRun sourceLocRun; - sourceLocRun.m_numInst = numInsts; - sourceLocRun.m_startInstIndex = IRSerialData::InstIndex(startInstIndex); - sourceLocRun.m_sourceLoc = uint32_t(debugSourceFile->m_baseSourceLoc + offset); - - m_serialData->m_debugSourceLocRuns.add(sourceLocRun); -} - -Result IRSerialWriter::_calcDebugInfo() -{ - // We need to find the unique source Locs - // We are not going to store SourceLocs directly, because there may be multiple views mapping down to - // the same underlying source file - - // First find all the unique locs - struct InstLoc - { - typedef InstLoc ThisType; - - SLANG_FORCE_INLINE bool operator<(const ThisType& rhs) const { return sourceLoc < rhs.sourceLoc || (sourceLoc == rhs.sourceLoc && instIndex < rhs.instIndex); } - - uint32_t instIndex; - uint32_t sourceLoc; - }; - - // Find all of the source locations and their associated instructions - List<InstLoc> instLocs; - const Index numInsts = m_insts.getCount(); - for (Index i = 1; i < numInsts; i++) - { - IRInst* srcInst = m_insts[i]; - if (!srcInst->sourceLoc.isValid()) - { - continue; - } - InstLoc instLoc; - instLoc.instIndex = uint32_t(i); - instLoc.sourceLoc = uint32_t(srcInst->sourceLoc.getRaw()); - instLocs.add(instLoc); - } - - // Sort them - instLocs.sort(); - m_debugFreeSourceLoc = 1; - - // Look for runs - const InstLoc* startInstLoc = instLocs.begin(); - const InstLoc* endInstLoc = instLocs.end(); - - while (startInstLoc < endInstLoc) - { - const uint32_t startSourceLoc = startInstLoc->sourceLoc; - - // Find the run with the same source loc - - const InstLoc* curInstLoc = startInstLoc + 1; - uint32_t curInstIndex = startInstLoc->instIndex + 1; - - // Find the run size with same source loc and run of instruction indices - for (; curInstLoc < endInstLoc && curInstLoc->sourceLoc == startSourceLoc && curInstLoc->instIndex == curInstIndex; ++curInstLoc, ++curInstIndex) - { - } - - // Try adding the run - _addDebugSourceLocRun(SourceLoc::fromRaw(startSourceLoc), startInstLoc->instIndex, curInstIndex - startInstLoc->instIndex); - - // Next - startInstLoc = curInstLoc; - } - - // Okay we can now calculate the final source information - - for (auto& pair : m_debugSourceFileMap) - { - DebugSourceFile* debugSourceFile = pair.Value; - SourceFile* sourceFile = debugSourceFile->m_sourceFile; - - IRSerialData::DebugSourceInfo sourceInfo; - - sourceInfo.m_numLines = uint32_t(debugSourceFile->m_sourceFile->getLineBreakOffsets().getCount()); - - sourceInfo.m_startSourceLoc = uint32_t(debugSourceFile->m_baseSourceLoc); - sourceInfo.m_endSourceLoc = uint32_t(debugSourceFile->m_baseSourceLoc + sourceFile->getContentSize()); - - sourceInfo.m_pathIndex = Ser::StringIndex(m_debugStringSlicePool.add(sourceFile->getPathInfo().foundPath)); - - sourceInfo.m_lineInfosStartIndex = uint32_t(m_serialData->m_debugLineInfos.getCount()); - sourceInfo.m_adjustedLineInfosStartIndex = uint32_t(m_serialData->m_debugAdjustedLineInfos.getCount()); - - sourceInfo.m_numLineInfos = uint32_t(debugSourceFile->m_lineInfos.getCount()); - sourceInfo.m_numAdjustedLineInfos = uint32_t(debugSourceFile->m_adjustedLineInfos.getCount()); - - // Add the line infos - m_serialData->m_debugLineInfos.addRange(debugSourceFile->m_lineInfos.begin(), debugSourceFile->m_lineInfos.getCount()); - m_serialData->m_debugAdjustedLineInfos.addRange(debugSourceFile->m_adjustedLineInfos.begin(), debugSourceFile->m_adjustedLineInfos.getCount()); - - // Add the source info - m_serialData->m_debugSourceInfos.add(sourceInfo); - } - - // Convert the string pool - SerialStringTableUtil::encodeStringTable(m_debugStringSlicePool, m_serialData->m_debugStringTable); - - return SLANG_OK; -} - -Result IRSerialWriter::write(IRModule* module, SourceManager* sourceManager, OptionFlags options, IRSerialData* serialData) -{ - typedef Ser::Inst::PayloadType PayloadType; - - m_sourceManager = sourceManager; - m_serialData = serialData; - - serialData->clear(); - - // We reserve 0 for null - m_insts.clear(); - m_insts.add(nullptr); - - // Reset - m_instMap.Clear(); - m_decorations.clear(); - - // Stack for parentInst - List<IRInst*> parentInstStack; - - IRModuleInst* moduleInst = module->getModuleInst(); - parentInstStack.add(moduleInst); - - // Add to the map - _addInstruction(moduleInst); - - // Traverse all of the instructions - while (parentInstStack.getCount()) - { - // If it's in the stack it is assumed it is already in the inst map - IRInst* parentInst = parentInstStack.getLast(); - parentInstStack.removeLast(); - SLANG_ASSERT(m_instMap.ContainsKey(parentInst)); - - // Okay we go through each of the children in order. If they are IRInstParent derived, we add to stack to process later - // cos we want breadth first so the order of children is the same as their index order, meaning we don't need to store explicit indices - const Ser::InstIndex startChildInstIndex = Ser::InstIndex(m_insts.getCount()); - - IRInstListBase childrenList = parentInst->getDecorationsAndChildren(); - for (IRInst* child : childrenList) - { - // This instruction can't be in the map... - SLANG_ASSERT(!m_instMap.ContainsKey(child)); - - _addInstruction(child); - - parentInstStack.add(child); - } - - // If it had any children, then store the information about it - if (Ser::InstIndex(m_insts.getCount()) != startChildInstIndex) - { - Ser::InstRun run; - run.m_parentIndex = m_instMap[parentInst]; - run.m_startInstIndex = startChildInstIndex; - run.m_numChildren = Ser::SizeType(m_insts.getCount() - int(startChildInstIndex)); - - m_serialData->m_childRuns.add(run); - } - } - -#if 0 - { - List<IRInst*> workInsts; - calcInstructionList(module, workInsts); - SLANG_ASSERT(workInsts.Count() == m_insts.Count()); - for (UInt i = 0; i < workInsts.Count(); ++i) - { - SLANG_ASSERT(workInsts[i] == m_insts[i]); - } - } -#endif - - // Set to the right size - m_serialData->m_insts.setCount(m_insts.getCount()); - // Clear all instructions - memset(m_serialData->m_insts.begin(), 0, sizeof(Ser::Inst) * m_serialData->m_insts.getCount()); - - // Need to set up the actual instructions - { - const Index numInsts = m_insts.getCount(); - - for (Index i = 1; i < numInsts; ++i) - { - IRInst* srcInst = m_insts[i]; - Ser::Inst& dstInst = m_serialData->m_insts[i]; - - dstInst.m_op = uint8_t(srcInst->op & kIROpMeta_OpMask); - dstInst.m_payloadType = PayloadType::Empty; - - dstInst.m_resultTypeIndex = getInstIndex(srcInst->getFullType()); - - IRConstant* irConst = as<IRConstant>(srcInst); - if (irConst) - { - switch (srcInst->op) - { - // Special handling for the ir const derived types - case kIROp_StringLit: - { - auto stringLit = static_cast<IRStringLit*>(srcInst); - dstInst.m_payloadType = PayloadType::String_1; - dstInst.m_payload.m_stringIndices[0] = getStringIndex(stringLit->getStringSlice()); - break; - } - case kIROp_IntLit: - { - dstInst.m_payloadType = PayloadType::Int64; - dstInst.m_payload.m_int64 = irConst->value.intVal; - break; - } - case kIROp_PtrLit: - { - dstInst.m_payloadType = PayloadType::Int64; - dstInst.m_payload.m_int64 = (intptr_t) irConst->value.ptrVal; - break; - } - case kIROp_FloatLit: - { - dstInst.m_payloadType = PayloadType::Float64; - dstInst.m_payload.m_float64 = irConst->value.floatVal; - break; - } - case kIROp_BoolLit: - { - dstInst.m_payloadType = PayloadType::UInt32; - dstInst.m_payload.m_uint32 = irConst->value.intVal ? 1 : 0; - break; - } - default: - { - SLANG_RELEASE_ASSERT(!"Unhandled constant type"); - return SLANG_FAIL; - } - } - continue; - } - - IRTextureTypeBase* textureBase = as<IRTextureTypeBase>(srcInst); - if (textureBase) - { - dstInst.m_payloadType = PayloadType::OperandAndUInt32; - dstInst.m_payload.m_operandAndUInt32.m_uint32 = uint32_t(srcInst->op) >> kIROpMeta_OtherShift; - dstInst.m_payload.m_operandAndUInt32.m_operand = getInstIndex(textureBase->getElementType()); - continue; - } - - // ModuleInst is different, in so far as it holds a pointer to IRModule, but we don't need - // to save that off in a special way, so can just use regular path - - const int numOperands = int(srcInst->operandCount); - Ser::InstIndex* dstOperands = nullptr; - - if (numOperands <= Ser::Inst::kMaxOperands) - { - // Checks the compile below is valid - SLANG_COMPILE_TIME_ASSERT(PayloadType(0) == PayloadType::Empty && PayloadType(1) == PayloadType::Operand_1 && PayloadType(2) == PayloadType::Operand_2); - - dstInst.m_payloadType = PayloadType(numOperands); - dstOperands = dstInst.m_payload.m_operands; - } - else - { - dstInst.m_payloadType = PayloadType::OperandExternal; - - int operandArrayBaseIndex = int(m_serialData->m_externalOperands.getCount()); - m_serialData->m_externalOperands.setCount(operandArrayBaseIndex + numOperands); - - dstOperands = m_serialData->m_externalOperands.begin() + operandArrayBaseIndex; - - auto& externalOperands = dstInst.m_payload.m_externalOperand; - externalOperands.m_arrayIndex = Ser::ArrayIndex(operandArrayBaseIndex); - externalOperands.m_size = Ser::SizeType(numOperands); - } - - for (int j = 0; j < numOperands; ++j) - { - const Ser::InstIndex dstInstIndex = getInstIndex(srcInst->getOperand(j)); - dstOperands[j] = dstInstIndex; - } - } - } - - // Convert strings into a string table - { - SerialStringTableUtil::encodeStringTable(m_stringSlicePool, serialData->m_stringTable); - } - - // If the option to use RawSourceLocations is enabled, serialize out as is - if (options & OptionFlag::RawSourceLocation) - { - const Index numInsts = m_insts.getCount(); - serialData->m_rawSourceLocs.setCount(numInsts); - - Ser::RawSourceLoc* dstLocs = serialData->m_rawSourceLocs.begin(); - // 0 is null, just mark as no location - dstLocs[0] = Ser::RawSourceLoc(0); - for (Index i = 1; i < numInsts; ++i) - { - IRInst* srcInst = m_insts[i]; - dstLocs[i] = Ser::RawSourceLoc(srcInst->sourceLoc.getRaw()); - } - } - - if (options & OptionFlag::DebugInfo) - { - _calcDebugInfo(); - } - - m_serialData = nullptr; - return SLANG_OK; -} - -static Result _writeArrayChunk(IRSerialCompressionType compressionType, FourCC chunkId, const void* data, size_t numEntries, size_t typeSize, RiffContainer* container) -{ - typedef RiffContainer::Chunk Chunk; - typedef RiffContainer::ScopeChunk ScopeChunk; - - typedef IRSerialBinary Bin; - if (numEntries == 0) - { - return SLANG_OK; - } - - // Make compressed fourCC - chunkId = (compressionType != IRSerialCompressionType::None) ? SLANG_MAKE_COMPRESSED_FOUR_CC(chunkId) : chunkId; - - ScopeChunk scope(container, Chunk::Kind::Data, chunkId); - - switch (compressionType) - { - case IRSerialCompressionType::None: - { - Bin::ArrayHeader header; - header.numEntries = uint32_t(numEntries); - - container->write(&header, sizeof(header)); - container->write(data, typeSize * numEntries); - break; - } - case IRSerialCompressionType::VariableByteLite: - { - List<uint8_t> compressedPayload; - - size_t numCompressedEntries = (numEntries * typeSize) / sizeof(uint32_t); - ByteEncodeUtil::encodeLiteUInt32((const uint32_t*)data, numCompressedEntries, compressedPayload); - - Bin::CompressedArrayHeader header; - header.numEntries = uint32_t(numEntries); - header.numCompressedEntries = uint32_t(numCompressedEntries); - - container->write(&header, sizeof(header)); - container->write(compressedPayload.getBuffer(), compressedPayload.getCount()); - break; - } - default: - { - return SLANG_FAIL; - } - } - return SLANG_OK; -} - -template <typename T> -Result _writeArrayChunk(IRSerialCompressionType compressionType, FourCC chunkId, const List<T>& array, RiffContainer* container) -{ - return _writeArrayChunk(compressionType, chunkId, array.begin(), size_t(array.getCount()), sizeof(T), container); -} - -Result _encodeInsts(IRSerialCompressionType compressionType, const List<IRSerialData::Inst>& instsIn, List<uint8_t>& encodeArrayOut) -{ - typedef IRSerialBinary Bin; - typedef IRSerialData::Inst::PayloadType PayloadType; - - if (compressionType != IRSerialCompressionType::VariableByteLite) - { - return SLANG_FAIL; - } - encodeArrayOut.clear(); - - const size_t numInsts = size_t(instsIn.getCount()); - const IRSerialData::Inst* insts = instsIn.begin(); - - uint8_t* encodeOut = encodeArrayOut.begin(); - uint8_t* encodeEnd = encodeArrayOut.end(); - - // Calculate the maximum instruction size with worst case possible encoding - // 2 bytes hold the payload size, and the result type - // Note that if there were some free bits, we could encode some of this stuff into bits, but if we remove payloadType, then there are no free bits - const size_t maxInstSize = 2 + ByteEncodeUtil::kMaxLiteEncodeUInt32 + Math::Max(sizeof(insts->m_payload.m_float64), size_t(2 * ByteEncodeUtil::kMaxLiteEncodeUInt32)); - - for (size_t i = 0; i < numInsts; ++i) - { - const auto& inst = insts[i]; - - // Make sure there is space for the largest possible instruction - if (encodeOut + maxInstSize >= encodeEnd) - { - const size_t offset = size_t(encodeOut - encodeArrayOut.begin()); - - const UInt oldCapacity = encodeArrayOut.getCapacity(); - - encodeArrayOut.reserve(oldCapacity + (oldCapacity >> 1) + maxInstSize); - const UInt capacity = encodeArrayOut.getCapacity(); - encodeArrayOut.setCount(capacity); - - encodeOut = encodeArrayOut.begin() + offset; - encodeEnd = encodeArrayOut.end(); - } - - *encodeOut++ = uint8_t(inst.m_op); - *encodeOut++ = uint8_t(inst.m_payloadType); - - encodeOut += ByteEncodeUtil::encodeLiteUInt32((uint32_t)inst.m_resultTypeIndex, encodeOut); - - switch (inst.m_payloadType) - { - case PayloadType::Empty: - { - break; - } - case PayloadType::Operand_1: - case PayloadType::String_1: - case PayloadType::UInt32: - { - // 1 UInt32 - encodeOut += ByteEncodeUtil::encodeLiteUInt32((uint32_t)inst.m_payload.m_operands[0], encodeOut); - break; - } - case PayloadType::Operand_2: - case PayloadType::OperandAndUInt32: - case PayloadType::OperandExternal: - case PayloadType::String_2: - { - // 2 UInt32 - encodeOut += ByteEncodeUtil::encodeLiteUInt32((uint32_t)inst.m_payload.m_operands[0], encodeOut); - encodeOut += ByteEncodeUtil::encodeLiteUInt32((uint32_t)inst.m_payload.m_operands[1], encodeOut); - break; - } - case PayloadType::Float64: - { - memcpy(encodeOut, &inst.m_payload.m_float64, sizeof(inst.m_payload.m_float64)); - encodeOut += sizeof(inst.m_payload.m_float64); - break; - } - case PayloadType::Int64: - { - memcpy(encodeOut, &inst.m_payload.m_int64, sizeof(inst.m_payload.m_int64)); - encodeOut += sizeof(inst.m_payload.m_int64); - break; - } - } - } - - // Fix the size - encodeArrayOut.setCount(UInt(encodeOut - encodeArrayOut.begin())); - return SLANG_OK; -} - -Result _writeInstArrayChunk(IRSerialCompressionType compressionType, FourCC chunkId, const List<IRSerialData::Inst>& array, RiffContainer* container) -{ - typedef RiffContainer::Chunk Chunk; - typedef RiffContainer::ScopeChunk ScopeChunk; - - typedef IRSerialBinary Bin; - if (array.getCount() == 0) - { - return SLANG_OK; - } - - switch (compressionType) - { - case IRSerialCompressionType::None: - { - return _writeArrayChunk(compressionType, chunkId, array, container); - } - case IRSerialCompressionType::VariableByteLite: - { - List<uint8_t> compressedPayload; - SLANG_RETURN_ON_FAIL(_encodeInsts(compressionType, array, compressedPayload)); - - ScopeChunk scope(container, Chunk::Kind::Data, SLANG_MAKE_COMPRESSED_FOUR_CC(chunkId)); - - Bin::CompressedArrayHeader header; - header.numEntries = uint32_t(array.getCount()); - header.numCompressedEntries = 0; - - container->write(&header, sizeof(header)); - container->write(compressedPayload.getBuffer(), compressedPayload.getCount()); - - return SLANG_OK; - } - default: break; - } - return SLANG_FAIL; -} - -/* static */Result IRSerialWriter::writeContainer(const IRSerialData& data, IRSerialCompressionType compressionType, RiffContainer* container) -{ - typedef RiffContainer::Chunk Chunk; - typedef RiffContainer::ScopeChunk ScopeChunk; - - ScopeChunk scopeModule(container, Chunk::Kind::List, Bin::kSlangModuleFourCc); - - // Write the header - { - Bin::ModuleHeader moduleHeader; - moduleHeader.compressionType = uint32_t(IRSerialCompressionType::VariableByteLite); - - ScopeChunk scopeHeader(container, Chunk::Kind::Data, Bin::kSlangModuleHeaderFourCc); - container->write(&moduleHeader, sizeof(moduleHeader)); - } - - SLANG_RETURN_ON_FAIL(_writeInstArrayChunk(compressionType, Bin::kInstFourCc, data.m_insts, container)); - SLANG_RETURN_ON_FAIL(_writeArrayChunk(compressionType, Bin::kChildRunFourCc, data.m_childRuns, container)); - SLANG_RETURN_ON_FAIL(_writeArrayChunk(compressionType, Bin::kExternalOperandsFourCc, data.m_externalOperands, container)); - SLANG_RETURN_ON_FAIL(_writeArrayChunk(IRSerialCompressionType::None, Bin::kStringFourCc, data.m_stringTable, container)); - - SLANG_RETURN_ON_FAIL(_writeArrayChunk(IRSerialCompressionType::None, Bin::kUInt32SourceLocFourCc, data.m_rawSourceLocs, container)); - - if (data.m_debugSourceInfos.getCount()) - { - _writeArrayChunk(IRSerialCompressionType::None, Bin::kDebugStringFourCc, data.m_debugStringTable, container); - _writeArrayChunk(IRSerialCompressionType::None, Bin::kDebugLineInfoFourCc, data.m_debugLineInfos, container); - _writeArrayChunk(IRSerialCompressionType::None, Bin::kDebugAdjustedLineInfoFourCc, data.m_debugAdjustedLineInfos, container); - _writeArrayChunk(IRSerialCompressionType::None, Bin::kDebugSourceInfoFourCc, data.m_debugSourceInfos, container); - _writeArrayChunk(compressionType, Bin::kDebugSourceLocRunFourCc, data.m_debugSourceLocRuns, container); - } - - return SLANG_OK; -} - -/* static */Result IRSerialWriter::writeStream(const IRSerialData& data, IRSerialCompressionType compressionType, Stream* stream) -{ - RiffContainer container; - SLANG_RETURN_ON_FAIL(writeContainer(data, compressionType, &container)); - return RiffUtil::write(container.getRoot(), true, stream); -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! IRSerialReader !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -class ListResizer -{ - public: - virtual void* setSize(size_t newSize) = 0; - SLANG_FORCE_INLINE size_t getTypeSize() const { return m_typeSize; } - ListResizer(size_t typeSize):m_typeSize(typeSize) {} - - protected: - size_t m_typeSize; -}; - -template <typename T> -class ListResizerForType: public ListResizer -{ - public: - typedef ListResizer Parent; - - SLANG_FORCE_INLINE ListResizerForType(List<T>& list): - Parent(sizeof(T)), - m_list(list) - {} - - virtual void* setSize(size_t newSize) SLANG_OVERRIDE - { - m_list.setCount(UInt(newSize)); - return (void*)m_list.begin(); - } - - protected: - List<T>& m_list; -}; - -static Result _readArrayChunk(IRSerialCompressionType compressionType, RiffContainer::DataChunk* dataChunk, ListResizer& listOut) -{ - typedef IRSerialBinary Bin; - - RiffReadHelper read = dataChunk->asReadHelper(); - const size_t typeSize = listOut.getTypeSize(); - - switch (compressionType) - { - case IRSerialCompressionType::VariableByteLite: - { - Bin::CompressedArrayHeader header; - SLANG_RETURN_ON_FAIL(read.read(header)); - - void* dst = listOut.setSize(header.numEntries); - SLANG_ASSERT(header.numCompressedEntries == uint32_t((header.numEntries * typeSize) / sizeof(uint32_t))); - - // Decode.. - ByteEncodeUtil::decodeLiteUInt32(read.getData(), header.numCompressedEntries, (uint32_t*)dst); - break; - } - case IRSerialCompressionType::None: - { - // Read uncompressed - Bin::ArrayHeader header; - SLANG_RETURN_ON_FAIL(read.read(header)); - const size_t payloadSize = header.numEntries * typeSize; - SLANG_ASSERT(payloadSize == read.getRemainingSize()); - void* dst = listOut.setSize(header.numEntries); - ::memcpy(dst, read.getData(), payloadSize); - break; - } - } - return SLANG_OK; -} - -template <typename T> -static Result _readArrayChunk(const IRSerialBinary::ModuleHeader* header, RiffContainer::DataChunk* dataChunk, List<T>& arrayOut) -{ - typedef IRSerialBinary Bin; - - Bin::CompressionType compressionType = Bin::CompressionType::None; - - if (dataChunk->m_fourCC == SLANG_MAKE_COMPRESSED_FOUR_CC(dataChunk->m_fourCC)) - { - // If it has compression, use the compression type set in the header - compressionType = Bin::CompressionType(header->compressionType); - } - ListResizerForType<T> resizer(arrayOut); - return _readArrayChunk(compressionType, dataChunk, resizer); -} - -template <typename T> -static Result _readArrayUncompressedChunk(const IRSerialBinary::ModuleHeader* header, RiffContainer::DataChunk* chunk, List<T>& arrayOut) -{ - SLANG_UNUSED(header); - ListResizerForType<T> resizer(arrayOut); - return _readArrayChunk(IRSerialBinary::CompressionType::None, chunk, resizer); -} - -static Result _decodeInsts(IRSerialCompressionType compressionType, const uint8_t* encodeCur, size_t encodeInSize, List<IRSerialData::Inst>& instsOut) -{ - const uint8_t* encodeEnd = encodeCur + encodeInSize; - - typedef IRSerialBinary Bin; - typedef IRSerialData::Inst::PayloadType PayloadType; - - if (compressionType != IRSerialCompressionType::VariableByteLite) - { - return SLANG_FAIL; - } - - const size_t numInsts = size_t(instsOut.getCount()); - IRSerialData::Inst* insts = instsOut.begin(); - - for (size_t i = 0; i < numInsts; ++i) - { - if (encodeCur >= encodeEnd) - { - SLANG_ASSERT(!"Invalid decode"); - return SLANG_FAIL; - } - - auto& inst = insts[i]; - - inst.m_op = *encodeCur++; - const PayloadType payloadType = PayloadType(*encodeCur++); - inst.m_payloadType = payloadType; - - // Read the result value - encodeCur += ByteEncodeUtil::decodeLiteUInt32(encodeCur, (uint32_t*)&inst.m_resultTypeIndex); - - switch (inst.m_payloadType) - { - case PayloadType::Empty: - { - break; - } - case PayloadType::Operand_1: - case PayloadType::String_1: - case PayloadType::UInt32: - { - // 1 UInt32 - encodeCur += ByteEncodeUtil::decodeLiteUInt32(encodeCur, (uint32_t*)&inst.m_payload.m_operands[0]); - break; - } - case PayloadType::Operand_2: - case PayloadType::OperandAndUInt32: - case PayloadType::OperandExternal: - case PayloadType::String_2: - { - // 2 UInt32 - encodeCur += ByteEncodeUtil::decodeLiteUInt32(encodeCur, 2, (uint32_t*)&inst.m_payload.m_operands[0]); - break; - } - case PayloadType::Float64: - { - memcpy(&inst.m_payload.m_float64, encodeCur, sizeof(inst.m_payload.m_float64)); - encodeCur += sizeof(inst.m_payload.m_float64); - break; - } - case PayloadType::Int64: - { - memcpy(&inst.m_payload.m_int64, encodeCur, sizeof(inst.m_payload.m_int64)); - encodeCur += sizeof(inst.m_payload.m_int64); - break; - } - } - } - - return SLANG_OK; -} - -static Result _readInstArrayChunk(const IRSerialBinary::ModuleHeader* moduleHeader, RiffContainer::DataChunk* chunk, List<IRSerialData::Inst>& arrayOut) -{ - typedef IRSerialBinary Bin; - - IRSerialCompressionType compressionType = IRSerialCompressionType::None; - if (chunk->m_fourCC == SLANG_MAKE_COMPRESSED_FOUR_CC(chunk->m_fourCC)) - { - compressionType = IRSerialCompressionType(moduleHeader->compressionType); - } - - switch (compressionType) - { - case IRSerialCompressionType::None: - { - ListResizerForType<IRSerialData::Inst> resizer(arrayOut); - return _readArrayChunk(compressionType, chunk, resizer); - } - case IRSerialCompressionType::VariableByteLite: - { - RiffReadHelper read = chunk->asReadHelper(); - - Bin::CompressedArrayHeader header; - SLANG_RETURN_ON_FAIL(read.read(header)); - - arrayOut.setCount(header.numEntries); - - SLANG_RETURN_ON_FAIL(_decodeInsts(compressionType, read.getData(), read.getRemainingSize(), arrayOut)); - break; - } - default: - { - return SLANG_FAIL; - } - } - - return SLANG_OK; -} - -/* static */Result IRSerialReader::readContainer(RiffContainer::ListChunk* module, IRSerialData* outData) -{ - typedef IRSerialBinary Bin; - - outData->clear(); - - Bin::ModuleHeader* header = module->findContainedData<Bin::ModuleHeader>(Bin::kSlangModuleHeaderFourCc); - if (!header) - { - return SLANG_FAIL; - } - - for (RiffContainer::Chunk* chunk = module->m_containedChunks; chunk; chunk = chunk->m_next) - { - RiffContainer::DataChunk* dataChunk = as<RiffContainer::DataChunk>(chunk); - if (!dataChunk) - { - continue; - } - - switch (dataChunk->m_fourCC) - { - case SLANG_MAKE_COMPRESSED_FOUR_CC(Bin::kInstFourCc): - case Bin::kInstFourCc: - { - SLANG_RETURN_ON_FAIL(_readInstArrayChunk(header, dataChunk, outData->m_insts)); - break; - } - case SLANG_MAKE_COMPRESSED_FOUR_CC(Bin::kChildRunFourCc): - case Bin::kChildRunFourCc: - { - SLANG_RETURN_ON_FAIL(_readArrayChunk(header, dataChunk, outData->m_childRuns)); - break; - } - case SLANG_MAKE_COMPRESSED_FOUR_CC(Bin::kExternalOperandsFourCc): - case Bin::kExternalOperandsFourCc: - { - SLANG_RETURN_ON_FAIL(_readArrayChunk(header, dataChunk, outData->m_externalOperands)); - break; - } - case Bin::kStringFourCc: - { - SLANG_RETURN_ON_FAIL(_readArrayUncompressedChunk(header, dataChunk, outData->m_stringTable)); - break; - } - case Bin::kUInt32SourceLocFourCc: - { - SLANG_RETURN_ON_FAIL(_readArrayUncompressedChunk(header, dataChunk, outData->m_rawSourceLocs)); - break; - } - case Bin::kDebugStringFourCc: - { - SLANG_RETURN_ON_FAIL(_readArrayUncompressedChunk(header, dataChunk, outData->m_debugStringTable)); - break; - } - case Bin::kDebugLineInfoFourCc: - { - SLANG_RETURN_ON_FAIL(_readArrayUncompressedChunk(header, dataChunk, outData->m_debugLineInfos)); - break; - } - case Bin::kDebugAdjustedLineInfoFourCc: - { - SLANG_RETURN_ON_FAIL(_readArrayUncompressedChunk(header, dataChunk, outData->m_debugAdjustedLineInfos)); - break; - } - case Bin::kDebugSourceInfoFourCc: - { - SLANG_RETURN_ON_FAIL(_readArrayChunk(header, dataChunk, outData->m_debugSourceInfos)); - break; - } - case SLANG_MAKE_COMPRESSED_FOUR_CC(Bin::kDebugSourceLocRunFourCc): - case Bin::kDebugSourceLocRunFourCc: - { - SLANG_RETURN_ON_FAIL(_readArrayChunk(header, dataChunk, outData->m_debugSourceLocRuns)); - break; - } - default: - { - break; - } - } - } - - return SLANG_OK; -} - -/* static */Result IRSerialReader::readStream(Stream* stream, IRSerialData* outData) -{ - typedef IRSerialBinary Bin; - - outData->clear(); - - // Read the riff file - RiffContainer container; - SLANG_RETURN_ON_FAIL(RiffUtil::read(stream, container)); - - // Find the riff module - RiffContainer::ListChunk* module = container.getRoot()->findListRec(Bin::kSlangModuleFourCc); - if (!module) - { - return SLANG_FAIL; - } - - SLANG_RETURN_ON_FAIL(readContainer(module, outData)); - return SLANG_OK; -} - -static SourceRange _toSourceRange(const IRSerialData::DebugSourceInfo& info) -{ - SourceRange range; - range.begin = SourceLoc::fromRaw(info.m_startSourceLoc); - range.end = SourceLoc::fromRaw(info.m_endSourceLoc); - return range; -} - -static int _findIndex(const List<IRSerialData::DebugSourceInfo>& infos, SourceLoc sourceLoc) -{ - const int numInfos = int(infos.getCount()); - for (int i = 0; i < numInfos; ++i) - { - if (_toSourceRange(infos[i]).contains(sourceLoc)) - { - return i; - } - } - - return -1; -} - -static int _calcFixSourceLoc(const IRSerialData::DebugSourceInfo& info, SourceView* sourceView, SourceRange& rangeOut) -{ - rangeOut = _toSourceRange(info); - return int(sourceView->getRange().begin.getRaw()) - int(info.m_startSourceLoc); -} - -/* static */Result IRSerialReader::readContainerModules(RiffContainer* container, Session* session, SourceManager* sourceManager, List<RefPtr<IRModule>>& outModules, List<FrontEndCompileRequest::ExtraEntryPointInfo>& outEntryPoints) -{ - List<RiffContainer::ListChunk*> moduleChunks; - List<RiffContainer::DataChunk*> entryPointChunks; - // First try to find a list - { - RiffContainer::ListChunk* listChunk = container->getRoot()->findListRec(SerialBinary::kSlangModuleListFourCc); - if (listChunk) - { - listChunk->findContained(IRSerialBinary::kSlangModuleFourCc, moduleChunks); - listChunk->findContained(SerialBinary::kEntryPointFourCc, entryPointChunks); - } - else - { - // Maybe its just a single module - RiffContainer::ListChunk* moduleChunk = container->getRoot()->findListRec(IRSerialBinary::kSlangModuleFourCc); - if (!moduleChunk) - { - // Couldn't find any modules - return SLANG_FAIL; - } - moduleChunks.add(moduleChunk); - } - } - - // Okay, we need to decode into ir modules - for (auto moduleChunk : moduleChunks) - { - IRSerialData serialData; - - SLANG_RETURN_ON_FAIL(IRSerialReader::readContainer(moduleChunk, &serialData)); - - // Construct into a module - RefPtr<IRModule> irModule; - IRSerialReader reader; - SLANG_RETURN_ON_FAIL(reader.read(serialData, session, sourceManager, irModule)); - - outModules.add(irModule); - } - - for (auto entryPointChunk : entryPointChunks) - { - auto reader = entryPointChunk->asReadHelper(); - - auto readString = [&]() - { - uint32_t length = 0; - reader.read(length); - - char* begin = (char*)reader.getData(); - reader.skip(length + 1); - - return UnownedStringSlice(begin, begin + length); - }; - - FrontEndCompileRequest::ExtraEntryPointInfo entryPointInfo; - - entryPointInfo.name = session->getNamePool()->getName(readString()); - reader.read(entryPointInfo.profile); - entryPointInfo.mangledName = readString(); - - outEntryPoints.add(entryPointInfo); - } - - return SLANG_OK; -} - - -// TODO: The following function isn't really part of the IR serialization system, but rather -// a layered "container" format, and as such probably belongs in a higher-level system that -// simply calls into the `IRSerialReader` rather than being part of it... -// -/* static */Result IRSerialReader::readStreamModules(Stream* stream, Session* session, SourceManager* sourceManager, List<RefPtr<IRModule>>& outModules, List<FrontEndCompileRequest::ExtraEntryPointInfo>& outEntryPoints) -{ - // Load up the module - RiffContainer container; - SLANG_RETURN_ON_FAIL(RiffUtil::read(stream, container)); - - SLANG_RETURN_ON_FAIL(readContainerModules(&container, session, sourceManager, outModules, outEntryPoints)); - return SLANG_OK; -} - -/* static */Result IRSerialReader::read(const IRSerialData& data, Session* session, SourceManager* sourceManager, RefPtr<IRModule>& moduleOut) -{ - typedef Ser::Inst::PayloadType PayloadType; - - m_serialData = &data; - - auto module = new IRModule(); - moduleOut = module; - m_module = module; - - module->session = session; - - // Set up the string rep cache - m_stringRepresentationCache.init(&data.m_stringTable); - - // Add all the instructions - - List<IRInst*> insts; - - const Index numInsts = data.m_insts.getCount(); - - SLANG_ASSERT(numInsts > 0); - - insts.setCount(numInsts); - insts[0] = nullptr; - - // 0 holds null - // 1 holds the IRModuleInst - { - // Check that insts[1] is the module inst - const Ser::Inst& srcInst = data.m_insts[1]; - SLANG_RELEASE_ASSERT(srcInst.m_op == kIROp_Module); - SLANG_ASSERT(srcInst.m_payloadType == PayloadType::Empty); - - // Create the module inst - auto moduleInst = static_cast<IRModuleInst*>(createEmptyInstWithSize(module, kIROp_Module, sizeof(IRModuleInst))); - module->moduleInst = moduleInst; - moduleInst->module = module; - - // Set the IRModuleInst - insts[1] = moduleInst; - } - - for (Index i = 2; i < numInsts; ++i) - { - const Ser::Inst& srcInst = data.m_insts[i]; - - const IROp op((IROp)srcInst.m_op); - - if (_isConstant(op)) - { - // Handling of constants - - // Calculate the minimum object size (ie not including the payload of value) - const size_t prefixSize = SLANG_OFFSET_OF(IRConstant, value); - - IRConstant* irConst = nullptr; - switch (op) - { - case kIROp_BoolLit: - { - SLANG_ASSERT(srcInst.m_payloadType == PayloadType::UInt32); - irConst = static_cast<IRConstant*>(createEmptyInstWithSize(module, op, prefixSize + sizeof(IRIntegerValue))); - irConst->value.intVal = srcInst.m_payload.m_uint32 != 0; - break; - } - case kIROp_IntLit: - { - SLANG_ASSERT(srcInst.m_payloadType == PayloadType::Int64); - irConst = static_cast<IRConstant*>(createEmptyInstWithSize(module, op, prefixSize + sizeof(IRIntegerValue))); - irConst->value.intVal = srcInst.m_payload.m_int64; - break; - } - case kIROp_PtrLit: - { - SLANG_ASSERT(srcInst.m_payloadType == PayloadType::Int64); - irConst = static_cast<IRConstant*>(createEmptyInstWithSize(module, op, prefixSize + sizeof(void*))); - irConst->value.ptrVal = (void*) (intptr_t) srcInst.m_payload.m_int64; - break; - } - case kIROp_FloatLit: - { - SLANG_ASSERT(srcInst.m_payloadType == PayloadType::Float64); - irConst = static_cast<IRConstant*>(createEmptyInstWithSize(module, op, prefixSize + sizeof(IRFloatingPointValue))); - irConst->value.floatVal = srcInst.m_payload.m_float64; - break; - } - case kIROp_StringLit: - { - SLANG_ASSERT(srcInst.m_payloadType == PayloadType::String_1); - - const UnownedStringSlice slice = m_stringRepresentationCache.getStringSlice(StringHandle(srcInst.m_payload.m_stringIndices[0])); - - const size_t sliceSize = slice.getLength(); - const size_t instSize = prefixSize + SLANG_OFFSET_OF(IRConstant::StringValue, chars) + sliceSize; - - irConst = static_cast<IRConstant*>(createEmptyInstWithSize(module, op, instSize)); - - IRConstant::StringValue& dstString = irConst->value.stringVal; - - dstString.numChars = uint32_t(sliceSize); - // Turn into pointer to avoid warning of array overrun - char* dstChars = dstString.chars; - // Copy the chars - memcpy(dstChars, slice.begin(), sliceSize); - break; - } - default: - { - SLANG_ASSERT(!"Unknown constant type"); - return SLANG_FAIL; - } - } - - insts[i] = irConst; - } - else if (_isTextureTypeBase(op)) - { - IRTextureTypeBase* inst = static_cast<IRTextureTypeBase*>(createEmptyInst(module, op, 1)); - SLANG_ASSERT(srcInst.m_payloadType == PayloadType::OperandAndUInt32); - - // Reintroduce the texture type bits into the the - const uint32_t other = srcInst.m_payload.m_operandAndUInt32.m_uint32; - inst->op = IROp(uint32_t(inst->op) | (other << kIROpMeta_OtherShift)); - - insts[i] = inst; - } - else - { - int numOperands = srcInst.getNumOperands(); - insts[i] = createEmptyInst(module, op, numOperands); - } - } - - // Patch up the operands - for (Index i = 1; i < numInsts; ++i) - { - const Ser::Inst& srcInst = data.m_insts[i]; - const IROp op((IROp)srcInst.m_op); - - IRInst* dstInst = insts[i]; - - // Set the result type - if (srcInst.m_resultTypeIndex != Ser::InstIndex(0)) - { - IRInst* resultInst = insts[int(srcInst.m_resultTypeIndex)]; - // NOTE! Counter intuitively the IRType* paramter may not be IRType* derived for example - // IRGlobalGenericParam is valid, but isn't IRType* derived - - //SLANG_RELEASE_ASSERT(as<IRType>(resultInst)); - dstInst->setFullType(static_cast<IRType*>(resultInst)); - } - - //if (!isParentDerived(op)) - { - const Ser::InstIndex* srcOperandIndices; - const int numOperands = data.getOperands(srcInst, &srcOperandIndices); - - auto dstOperands = dstInst->getOperands(); - - for (int j = 0; j < numOperands; j++) - { - dstOperands[j].init(dstInst, insts[int(srcOperandIndices[j])]); - } - } - } - - // Patch up the children - { - const Index numChildRuns = data.m_childRuns.getCount(); - for (Index i = 0; i < numChildRuns; i++) - { - const auto& run = data.m_childRuns[i]; - - IRInst* inst = insts[int(run.m_parentIndex)]; - - for (int j = 0; j < int(run.m_numChildren); ++j) - { - IRInst* child = insts[j + int(run.m_startInstIndex)]; - SLANG_ASSERT(child->parent == nullptr); - child->insertAtEnd(inst); - } - } - } - - // Re-add source locations, if they are defined - if (m_serialData->m_rawSourceLocs.getCount() == numInsts) - { - const Ser::RawSourceLoc* srcLocs = m_serialData->m_rawSourceLocs.begin(); - for (Index i = 1; i < numInsts; ++i) - { - IRInst* dstInst = insts[i]; - - dstInst->sourceLoc.setRaw(Slang::SourceLoc::RawValue(srcLocs[i])); - } - } - - if (sourceManager && m_serialData->m_debugSourceInfos.getCount()) - { - List<UnownedStringSlice> debugStringSlices; - SerialStringTableUtil::decodeStringTable(m_serialData->m_debugStringTable, debugStringSlices); - - // All of the strings are placed in the manager (and its StringSlicePool) where the SourceView and SourceFile are constructed from - List<StringSlicePool::Handle> stringMap; - SerialStringTableUtil::calcStringSlicePoolMap(debugStringSlices, sourceManager->getStringSlicePool(), stringMap); - - const List<IRSerialData::DebugSourceInfo>& sourceInfos = m_serialData->m_debugSourceInfos; - - // Construct the source files - Index numSourceFiles = sourceInfos.getCount(); - - // These hold the views (and SourceFile as there is only one SourceFile per view) in the same order as the sourceInfos - List<SourceView*> sourceViews; - sourceViews.setCount(numSourceFiles); - - for (Index i = 0; i < numSourceFiles; ++i) - { - const IRSerialData::DebugSourceInfo& srcSourceInfo = sourceInfos[i]; - - PathInfo pathInfo; - pathInfo.type = PathInfo::Type::FoundPath; - pathInfo.foundPath = debugStringSlices[UInt(srcSourceInfo.m_pathIndex)]; - - SourceFile* sourceFile = sourceManager->createSourceFileWithSize(pathInfo, srcSourceInfo.m_endSourceLoc - srcSourceInfo.m_startSourceLoc); - SourceView* sourceView = sourceManager->createSourceView(sourceFile, nullptr); - - // We need to accumulate all line numbers, for this source file, both adjusted and unadjusted - List<IRSerialData::DebugLineInfo> lineInfos; - // Add the adjusted lines - { - lineInfos.setCount(srcSourceInfo.m_numAdjustedLineInfos); - const IRSerialData::DebugAdjustedLineInfo* srcAdjustedLineInfos = m_serialData->m_debugAdjustedLineInfos.getBuffer() + srcSourceInfo.m_adjustedLineInfosStartIndex; - const int numAdjustedLines = int(srcSourceInfo.m_numAdjustedLineInfos); - for (int j = 0; j < numAdjustedLines; ++j) - { - lineInfos[j] = srcAdjustedLineInfos[j].m_lineInfo; - } - } - // Add regular lines - lineInfos.addRange(m_serialData->m_debugLineInfos.getBuffer() + srcSourceInfo.m_lineInfosStartIndex, srcSourceInfo.m_numLineInfos); - // Put in sourceloc order - lineInfos.sort(); - - List<uint32_t> lineBreakOffsets; - - // We can now set up the line breaks array - const int numLines = int(srcSourceInfo.m_numLines); - lineBreakOffsets.setCount(numLines); - - { - const Index numLineInfos = lineInfos.getCount(); - Index lineIndex = 0; - - // Every line up and including should hold the same offset - for (Index lineInfoIndex = 0; lineInfoIndex < numLineInfos; ++lineInfoIndex) - { - const auto& lineInfo = lineInfos[lineInfoIndex]; - - const uint32_t offset = lineInfo.m_lineStartOffset; - SLANG_ASSERT(offset > 0); - const int finishIndex = int(lineInfo.m_lineIndex); - - SLANG_ASSERT(finishIndex < numLines); - - for (; lineIndex < finishIndex; ++lineIndex) - { - lineBreakOffsets[lineIndex] = offset - 1; - } - lineBreakOffsets[lineIndex] = offset; - lineIndex++; - } - - // Do the remaining lines - const uint32_t offset = uint32_t(srcSourceInfo.m_endSourceLoc - srcSourceInfo.m_startSourceLoc); - for (; lineIndex < numLines; ++lineIndex) - { - lineBreakOffsets[lineIndex] = offset; - } - } - - sourceFile->setLineBreakOffsets(lineBreakOffsets.getBuffer(), lineBreakOffsets.getCount()); - - if (srcSourceInfo.m_numAdjustedLineInfos) - { - List<IRSerialData::DebugAdjustedLineInfo> adjustedLineInfos; - - int numEntries = int(srcSourceInfo.m_numAdjustedLineInfos); - - adjustedLineInfos.addRange(m_serialData->m_debugAdjustedLineInfos.getBuffer() + srcSourceInfo.m_adjustedLineInfosStartIndex, numEntries); - adjustedLineInfos.sort(); - - // Work out the views adjustments, and place in dstEntries - List<SourceView::Entry> dstEntries; - dstEntries.setCount(numEntries); - - const uint32_t sourceLocOffset = uint32_t(sourceView->getRange().begin.getRaw()); - - for (int j = 0; j < numEntries; ++j) - { - const auto& srcEntry = adjustedLineInfos[j]; - auto& dstEntry = dstEntries[j]; - - dstEntry.m_pathHandle = stringMap[int(srcEntry.m_pathStringIndex)]; - dstEntry.m_startLoc = SourceLoc::fromRaw(srcEntry.m_lineInfo.m_lineStartOffset + sourceLocOffset); - dstEntry.m_lineAdjust = int32_t(srcEntry.m_adjustedLineIndex) - int32_t(srcEntry.m_lineInfo.m_lineIndex); - } - - // Set the adjustments on the view - sourceView->setEntries(dstEntries.getBuffer(), dstEntries.getCount()); - } - - sourceViews[i] = sourceView; - } - - // We now need to apply the runs - { - List<IRSerialData::SourceLocRun> sourceRuns(m_serialData->m_debugSourceLocRuns); - // They are now in source location order - sourceRuns.sort(); - - // Just guess initially 0 for the source file that contains the initial run - SourceRange range; - int fixSourceLoc = _calcFixSourceLoc(sourceInfos[0], sourceViews[0], range); - - const Index numRuns = sourceRuns.getCount(); - for (Index i = 0; i < numRuns; ++i) - { - const auto& run = sourceRuns[i]; - const SourceLoc srcSourceLoc = SourceLoc::fromRaw(run.m_sourceLoc); - - if (!range.contains(srcSourceLoc)) - { - int index = _findIndex(sourceInfos, srcSourceLoc); - if (index < 0) - { - // Didn't find the match - continue; - } - fixSourceLoc = _calcFixSourceLoc(sourceInfos[index], sourceViews[index], range); - SLANG_ASSERT(range.contains(srcSourceLoc)); - } - - // Work out the fixed source location - SourceLoc sourceLoc = SourceLoc::fromRaw(int(run.m_sourceLoc) + fixSourceLoc); - - SLANG_ASSERT(Index(uint32_t(run.m_startInstIndex) + run.m_numInst) <= insts.getCount()); - IRInst** dstInsts = insts.getBuffer() + int(run.m_startInstIndex); - - const int runSize = int(run.m_numInst); - for (int j = 0; j < runSize; ++j) - { - dstInsts[j]->sourceLoc = sourceLoc; - } - } - } - } - - return SLANG_OK; -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!! IRSerialUtil !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -/* static */void IRSerialUtil::calcInstructionList(IRModule* module, List<IRInst*>& instsOut) -{ - // We reserve 0 for null - instsOut.setCount(1); - instsOut[0] = nullptr; - - // Stack for parentInst - List<IRInst*> parentInstStack; - - IRModuleInst* moduleInst = module->getModuleInst(); - parentInstStack.add(moduleInst); - - // Add to list - instsOut.add(moduleInst); - - // Traverse all of the instructions - while (parentInstStack.getCount()) - { - // If it's in the stack it is assumed it is already in the inst map - IRInst* parentInst = parentInstStack.getLast(); - parentInstStack.removeLast(); - - IRInstListBase childrenList = parentInst->getDecorationsAndChildren(); - for (IRInst* child : childrenList) - { - instsOut.add(child); - parentInstStack.add(child); - } - } -} - -/* static */SlangResult IRSerialUtil::verifySerialize(IRModule* module, Session* session, SourceManager* sourceManager, IRSerialCompressionType compressionType, IRSerialWriter::OptionFlags optionFlags) -{ - // Verify if we can stream out with debug information - - List<IRInst*> originalInsts; - calcInstructionList(module, originalInsts); - - IRSerialData serialData; - { - // Write IR out to serialData - copying over SourceLoc information directly - IRSerialWriter writer; - SLANG_RETURN_ON_FAIL(writer.write(module, sourceManager, optionFlags, &serialData)); - } - - // Write the data out to stream - OwnedMemoryStream memoryStream(FileAccess::ReadWrite); - SLANG_RETURN_ON_FAIL(IRSerialWriter::writeStream(serialData, compressionType, &memoryStream)); - - // Reset stream - memoryStream.seek(SeekOrigin::Start, 0); - - IRSerialData readData; - - SLANG_RETURN_ON_FAIL(IRSerialReader::readStream(&memoryStream, &readData)); - - // Check the stream read data is the same - if (readData != serialData) - { - SLANG_ASSERT(!"Streamed in data doesn't match"); - return SLANG_FAIL; - } - - RefPtr<IRModule> irReadModule; - - SourceManager workSourceManager; - workSourceManager.initialize(sourceManager, nullptr); - - { - IRSerialReader reader; - SLANG_RETURN_ON_FAIL(reader.read(serialData, session, &workSourceManager, irReadModule)); - } - - List<IRInst*> readInsts; - calcInstructionList(irReadModule, readInsts); - - if (readInsts.getCount() != originalInsts.getCount()) - { - SLANG_ASSERT(!"Instruction counts don't match"); - return SLANG_FAIL; - } - - if (optionFlags & IRSerialWriter::OptionFlag::RawSourceLocation) - { - SLANG_ASSERT(readInsts[0] == originalInsts[0]); - // All the source locs should be identical - for (Index i = 1; i < readInsts.getCount(); ++i) - { - IRInst* origInst = originalInsts[i]; - IRInst* readInst = readInsts[i]; - - if (origInst->sourceLoc.getRaw() != readInst->sourceLoc.getRaw()) - { - SLANG_ASSERT(!"Source locs don't match"); - return SLANG_FAIL; - } - } - } - else if (optionFlags & IRSerialWriter::OptionFlag::DebugInfo) - { - // They should be on the same line nos - for (Index i = 1; i < readInsts.getCount(); ++i) - { - IRInst* origInst = originalInsts[i]; - IRInst* readInst = readInsts[i]; - - if (origInst->sourceLoc.getRaw() == readInst->sourceLoc.getRaw()) - { - continue; - } - - // Work out the - SourceView* origSourceView = sourceManager->findSourceView(origInst->sourceLoc); - SourceView* readSourceView = workSourceManager.findSourceView(readInst->sourceLoc); - - // if both are null we are done - if (origSourceView == nullptr && origSourceView == readSourceView) - { - continue; - } - SLANG_ASSERT(origSourceView && readSourceView); - - { - auto origInfo = origSourceView->getHumaneLoc(origInst->sourceLoc, SourceLocType::Actual); - auto readInfo = readSourceView->getHumaneLoc(readInst->sourceLoc, SourceLocType::Actual); - - if (!(origInfo.line == readInfo.line && origInfo.column == readInfo.column && origInfo.pathInfo.foundPath == readInfo.pathInfo.foundPath)) - { - SLANG_ASSERT(!"Debug data didn't match"); - return SLANG_FAIL; - } - } - - // We may have adjusted line numbers -> but they may not match, because we only reconstruct one view - // So for now disable this test - - if (false) - { - auto origInfo = origSourceView->getHumaneLoc(origInst->sourceLoc, SourceLocType::Nominal); - auto readInfo = readSourceView->getHumaneLoc(readInst->sourceLoc, SourceLocType::Nominal); - - if (!(origInfo.line == readInfo.line && origInfo.column == readInfo.column && origInfo.pathInfo.foundPath == readInfo.pathInfo.foundPath)) - { - SLANG_ASSERT(!"Debug data didn't match"); - return SLANG_FAIL; - } - } - } - } - - return SLANG_OK; -} - -} // namespace Slang diff --git a/source/slang/slang-ir-serialize.h b/source/slang/slang-ir-serialize.h deleted file mode 100644 index ee99c08dc..000000000 --- a/source/slang/slang-ir-serialize.h +++ /dev/null @@ -1,163 +0,0 @@ -// slang-ir-serialize.h -#ifndef SLANG_IR_SERIALIZE_H_INCLUDED -#define SLANG_IR_SERIALIZE_H_INCLUDED - -#include "slang-ir-serialize-types.h" - -#include "../core/slang-riff.h" - -#include "slang-ir.h" - -#include "slang-ir-serialize-types.h" - -// For TranslationUnitRequest -// and FrontEndCompileRequest::ExtraEntryPointInfo -#include "slang-compiler.h" - -namespace Slang { - -struct IRSerialWriter -{ - typedef IRSerialData Ser; - typedef IRSerialBinary Bin; - - struct OptionFlag - { - typedef uint32_t Type; - enum Enum: Type - { - RawSourceLocation = 0x01, - DebugInfo = 0x02, - }; - }; - typedef OptionFlag::Type OptionFlags; - - Result write(IRModule* module, SourceManager* sourceManager, OptionFlags options, IRSerialData* serialData); - - static Result writeStream(const IRSerialData& data, IRSerialCompressionType compressionType, Stream* stream); - - /// Write to a container - static Result writeContainer(const IRSerialData& data, IRSerialCompressionType compressionType, RiffContainer* container); - - /// Get an instruction index from an instruction - Ser::InstIndex getInstIndex(IRInst* inst) const { return inst ? Ser::InstIndex(m_instMap[inst]) : Ser::InstIndex(0); } - - /// Get a slice from an index - UnownedStringSlice getStringSlice(Ser::StringIndex index) const { return m_stringSlicePool.getSlice(StringSlicePool::Handle(index)); } - /// Get index from string representations - Ser::StringIndex getStringIndex(StringRepresentation* string) { return Ser::StringIndex(m_stringSlicePool.add(string)); } - Ser::StringIndex getStringIndex(const UnownedStringSlice& slice) { return Ser::StringIndex(m_stringSlicePool.add(slice)); } - Ser::StringIndex getStringIndex(Name* name) { return name ? getStringIndex(name->text) : Ser::kNullStringIndex; } - Ser::StringIndex getStringIndex(const char* chars) { return Ser::StringIndex(m_stringSlicePool.add(chars)); } - Ser::StringIndex getStringIndex(const String& string) { return Ser::StringIndex(m_stringSlicePool.add(string.getUnownedSlice())); } - - StringSlicePool& getStringPool() { return m_stringSlicePool; } - StringSlicePool& getDebugStringPool() { return m_debugStringSlicePool; } - - IRSerialWriter() : - m_serialData(nullptr), - m_stringSlicePool(StringSlicePool::Style::Default), - m_debugStringSlicePool(StringSlicePool::Style::Default) - {} - -protected: - class DebugSourceFile : public RefObject - { - public: - DebugSourceFile(SourceFile* sourceFile, SourceLoc::RawValue baseSourceLoc): - m_sourceFile(sourceFile), - m_baseSourceLoc(baseSourceLoc) - { - // Need to know how many lines there are - const List<uint32_t>& lineOffsets = sourceFile->getLineBreakOffsets(); - - const auto numLineIndices = lineOffsets.getCount(); - - // Set none as being used initially - m_lineIndexUsed.setCount(numLineIndices); - ::memset(m_lineIndexUsed.begin(), 0, numLineIndices * sizeof(uint8_t)); - } - /// True if we have information on that line index - bool hasLineIndex(int lineIndex) const { return m_lineIndexUsed[lineIndex] != 0; } - void setHasLineIndex(int lineIndex) { m_lineIndexUsed[lineIndex] = 1; } - - SourceLoc::RawValue m_baseSourceLoc; ///< The base source location - - SourceFile* m_sourceFile; ///< The source file - List<uint8_t> m_lineIndexUsed; ///< Has 1 if the line is used - List<uint32_t> m_usedLineIndices; ///< Holds the lines that have been hit - - List<IRSerialData::DebugLineInfo> m_lineInfos; ///< The line infos - List<IRSerialData::DebugAdjustedLineInfo> m_adjustedLineInfos; ///< The adjusted line infos - }; - - void _addInstruction(IRInst* inst); - Result _calcDebugInfo(); - /// Returns the remapped sourceLoc, or 0 if sourceLoc couldn't be added - void _addDebugSourceLocRun(SourceLoc sourceLoc, uint32_t startInstIndex, uint32_t numInst); - - List<IRInst*> m_insts; ///< Instructions in same order as stored in the - - List<IRDecoration*> m_decorations; ///< Holds all decorations in order of the instructions as found - List<IRInst*> m_instWithFirstDecoration; ///< All decorations are held in this order after all the regular instructions - - Dictionary<IRInst*, Ser::InstIndex> m_instMap; ///< Map an instruction to an instruction index - - StringSlicePool m_stringSlicePool; - IRSerialData* m_serialData; ///< Where the data is stored - - StringSlicePool m_debugStringSlicePool; ///< Slices held just for debug usage - - SourceLoc::RawValue m_debugFreeSourceLoc; /// Locations greater than this are free - Dictionary<SourceFile*, RefPtr<DebugSourceFile> > m_debugSourceFileMap; - - SourceManager* m_sourceManager; ///< The source manager -}; - -struct IRSerialReader -{ - typedef IRSerialData Ser; - typedef StringRepresentationCache::Handle StringHandle; - - /// Read a stream to fill in dataOut IRSerialData - static Result readStream(Stream* stream, IRSerialData* dataOut); - - /// Read potentially multiple modules from a stream - static Result readStreamModules(Stream* stream, Session* session, SourceManager* manager, List<RefPtr<IRModule>>& outModules, List<FrontEndCompileRequest::ExtraEntryPointInfo>& outEntryPoints); - - /// Read potentially multiple modules from a stream - static Result readContainerModules(RiffContainer* container, Session* session, SourceManager* manager, List<RefPtr<IRModule>>& outModules, List<FrontEndCompileRequest::ExtraEntryPointInfo>& outEntryPoints); - - /// Read a stream to fill in dataOut IRSerialData - static Result readContainer(RiffContainer::ListChunk* module, IRSerialData* outData); - - /// Read a module from serial data - Result read(const IRSerialData& data, Session* session, SourceManager* sourceManager, RefPtr<IRModule>& moduleOut); - - IRSerialReader(): - m_serialData(nullptr), - m_module(nullptr) - { - } - - protected: - - StringRepresentationCache m_stringRepresentationCache; - - const IRSerialData* m_serialData; - IRModule* m_module; -}; - -struct IRSerialUtil -{ - /// Produces an instruction list which is in same order as written through IRSerialWriter - static void calcInstructionList(IRModule* module, List<IRInst*>& instsOut); - - /// Verify serialization - static SlangResult verifySerialize(IRModule* module, Session* session, SourceManager* sourceManager, IRSerialCompressionType compressionType, IRSerialWriter::OptionFlags optionFlags); -}; - - -} // namespace Slang - -#endif diff --git a/source/slang/slang-options.cpp b/source/slang/slang-options.cpp index 1fe78977a..42003fb62 100644 --- a/source/slang/slang-options.cpp +++ b/source/slang/slang-options.cpp @@ -13,7 +13,7 @@ #include "slang-file-system.h" #include "slang-repro.h" -#include "slang-ir-serialize.h" +#include "slang-serialize-ir.h" #include "../core/slang-type-text-util.h" @@ -592,7 +592,7 @@ struct OptionsParser { String name; SLANG_RETURN_ON_FAIL(tryReadCommandLineArgument(sink, arg, &argCursor, argEnd, name)); - SLANG_RETURN_ON_FAIL(IRSerialTypeUtil::parseCompressionType(name.getUnownedSlice(), requestImpl->getLinkage()->irCompressionType)); + SLANG_RETURN_ON_FAIL(SerialParseUtil::parseCompressionType(name.getUnownedSlice(), requestImpl->getLinkage()->serialCompressionType)); } else if (argStr == "-target") { diff --git a/source/slang/slang-ast-serialize.cpp b/source/slang/slang-serialize-ast.cpp index 22d5c3de2..82cd7d6b4 100644 --- a/source/slang/slang-ast-serialize.cpp +++ b/source/slang/slang-serialize-ast.cpp @@ -1,5 +1,5 @@ -// slang-ast-serialize.cpp -#include "slang-ast-serialize.h" +// slang-serialize-ast.cpp +#include "slang-serialize-ast.h" #include "slang-ast-generated.h" #include "slang-ast-generated-macro.h" @@ -1087,10 +1087,11 @@ ASTSerialClasses::ASTSerialClasses(): // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ASTSerialWriter !!!!!!!!!!!!!!!!!!!!!!!!!!!! -ASTSerialWriter::ASTSerialWriter(ASTSerialClasses* classes, ASTSerialFilter* filter) : +ASTSerialWriter::ASTSerialWriter(ASTSerialClasses* classes, ASTSerialFilter* filter, DebugSerialWriter* debugWriter) : m_arena(2048), m_classes(classes), - m_filter(filter) + m_filter(filter), + m_debugWriter(debugWriter) { // 0 is always the null pointer m_entries.add(nullptr); @@ -1292,8 +1293,14 @@ ASTSerialIndex ASTSerialWriter::addName(const Name* name) ASTSerialSourceLoc ASTSerialWriter::addSourceLoc(SourceLoc sourceLoc) { - SLANG_UNUSED(sourceLoc); - return 0; + if (sourceLoc.isValid() && m_debugWriter) + { + return m_debugWriter->addSourceLoc(sourceLoc); + } + else + { + return 0; + } } ASTSerialIndex ASTSerialWriter::_addArray(size_t elementSize, size_t alignment, const void* elements, Index elementCount) @@ -1706,8 +1713,7 @@ UnownedStringSlice ASTSerialReader::getStringSlice(ASTSerialIndex index) SourceLoc ASTSerialReader::getSourceLoc(ASTSerialSourceLoc loc) { - SLANG_UNUSED(loc); - return SourceLoc(); + return (loc && m_debugReader) ? m_debugReader->getSourceLoc(loc) : SourceLoc(); } SlangResult ASTSerialReader::loadEntries(const uint8_t* data, size_t dataCount, List<const ASTSerialInfo::Entry*>& outEntries) @@ -1865,63 +1871,6 @@ SlangResult ASTSerialReader::load(const uint8_t* data, size_t dataCount, ASTBuil return SLANG_OK; } - -/* static */Result ASTSerialReader::readContainerModules(RiffContainer* container, Linkage* linkage, List<RefPtr<Module>>& outModules) -{ - List<RiffContainer::ListChunk*> moduleChunks; - // First try to find a list - { - RiffContainer::ListChunk* listChunk = container->getRoot()->findListRec(SerialBinary::kSlangModuleListFourCc); - if (listChunk) - { - listChunk->findContained(ASTSerialBinary::kSlangASTModuleFourCC, moduleChunks); - } - else - { - // Maybe its just a single module - RiffContainer::ListChunk* moduleChunk = container->getRoot()->findListRec(ASTSerialBinary::kSlangASTModuleFourCC); - if (!moduleChunk) - { - // Couldn't find any modules - return SLANG_FAIL; - } - moduleChunks.add(moduleChunk); - } - } - - RefPtr<ASTSerialClasses> serialClasses(new ASTSerialClasses); - - // Okay, deserialize the each of the module chunks - for (RiffContainer::ListChunk* listChunk : moduleChunks) - { - // Look for the module data - auto data = listChunk->findContainedData(ASTSerialBinary::kSlangASTModuleDataFourCC); - - if (!data) - { - return SLANG_FAIL; - } - - ASTSerialReader reader(serialClasses); - - RefPtr<Module> module(new Module(linkage)); - SLANG_RETURN_ON_FAIL(reader.load((uint8_t*)data->getPayload(), data->getSize(), module->getASTBuilder(), linkage->getNamePool())); - - ModuleDecl* moduleDecl = reader.getPointer(ASTSerialIndex(1)).dynamicCast<ModuleDecl>(); - if (!moduleDecl) - { - return SLANG_FAIL; - } - - // Set on the module - module->setModuleDecl(moduleDecl); - - outModules.add(module); - } - - return SLANG_OK; -} - // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ASTSerializeUtil !!!!!!!!!!!!!!!!!!!!!!!!!!!! /* static */SlangResult ASTSerialTestUtil::selfTest() @@ -1976,7 +1925,7 @@ SlangResult ASTSerialReader::load(const uint8_t* data, size_t dataCount, ASTBuil ASTSerialFilter* filter = moduleDecl ? &filterStorage : nullptr; - ASTSerialWriter writer(classes, filter); + ASTSerialWriter writer(classes, filter, nullptr); // Lets serialize it all writer.addPointer(node); @@ -1988,7 +1937,7 @@ SlangResult ASTSerialReader::load(const uint8_t* data, size_t dataCount, ASTBuil NamePool namePool; namePool.setRootNamePool(rootNamePool); - ASTSerialReader reader(classes); + ASTSerialReader reader(classes, nullptr); ASTBuilder builder(sharedASTBuilder, "Serialize Check"); diff --git a/source/slang/slang-ast-serialize.h b/source/slang/slang-serialize-ast.h index a9fa9f605..76bbc9682 100644 --- a/source/slang/slang-ast-serialize.h +++ b/source/slang/slang-serialize-ast.h @@ -1,12 +1,14 @@ -// slang-ast-serialize.h -#ifndef SLANG_AST_SERIALIZE_H -#define SLANG_AST_SERIALIZE_H +// slang-serialize-ast.h +#ifndef SLANG_SERIALIZE_AST_H +#define SLANG_SERIALIZE_AST_H #include <type_traits> #include "slang-ast-support-types.h" #include "slang-ast-all.h" +#include "slang-serialize-debug.h" + #include "../core/slang-riff.h" #include "slang-ast-builder.h" @@ -53,7 +55,7 @@ struct ASTSerialTypeInfo; and specialize it for each native type. The specialization holds SerialType - The type that will be used to represent the native type -NativeType - The native typs +NativeType - The native type SerialAlignment - A value that holds what kind of alignment the SerialType needs to be serializable (it may be different from SLANG_ALIGN_OF(SerialType)!) toSerial - A function that with the help of ASTSerialWriter convert the NativeType into the SerialType toNative - A function that with the help of ASTSerialReader convert the SerialType into the NativeType @@ -70,7 +72,7 @@ which just holds the information held in the ASTSerialTypeInfo template, but add So we need to define a specialized ASTSerialTypeInfo for each type that can be a field in a NodeBase derived type. We don't need to define anything explicitly for the NodeBase derived types, as we will just generate the layout from the fields. How do we know the fields? We just -used the macros generated from the C++ extactor. +used the macros generated from the C++ extractor. So first a few things to observe... @@ -301,7 +303,7 @@ struct ASTSerialInfo typedef uint32_t ASTSerialIndexRaw; enum class ASTSerialIndex : ASTSerialIndexRaw; -typedef uint32_t ASTSerialSourceLoc; +typedef DebugSerialData::SourceLoc ASTSerialSourceLoc; /* A type to convert pointers into types such that they can be passed around to readers/writers without having to know the specific type. If there was a base class that all the serialized types derived from, @@ -369,7 +371,6 @@ public: UnownedStringSlice getStringSlice(ASTSerialIndex index); SourceLoc getSourceLoc(ASTSerialSourceLoc loc); - /// Load the entries table (without deserializing anything) /// NOTE! data must stay ins scope for outEntries to be valid SlangResult loadEntries(const uint8_t* data, size_t dataCount, List<const ASTSerialInfo::Entry*>& outEntries); @@ -377,11 +378,9 @@ public: /// NOTE! data must stay ins scope when reading takes place SlangResult load(const uint8_t* data, size_t dataCount, ASTBuilder* builder, NamePool* namePool); - /// Read the modules from the container - static Result readContainerModules(RiffContainer* container, Linkage* linkage, List<RefPtr<Module>>& outModules); - - ASTSerialReader(ASTSerialClasses* classes): - m_classes(classes) + ASTSerialReader(ASTSerialClasses* classes, DebugSerialReader* debugReader): + m_classes(classes), + m_debugReader(debugReader) { } @@ -391,6 +390,8 @@ protected: List<RefPtr<RefObject>> m_scope; ///< Objects to keep in scope during construction + DebugSerialReader* m_debugReader; + NamePool* m_namePool; ASTSerialClasses* m_classes; ///< Used to deserialize @@ -482,7 +483,7 @@ public: /// Write the state into the container SlangResult writeIntoContainer(RiffContainer* container); - ASTSerialWriter(ASTSerialClasses* classes, ASTSerialFilter* filter); + ASTSerialWriter(ASTSerialClasses* classes, ASTSerialFilter* filter, DebugSerialWriter* debugWriter); protected: @@ -498,6 +499,8 @@ protected: return index; } + DebugSerialWriter* m_debugWriter; //< For writing/mapping serialized source locs + Dictionary<const void*, Index> m_ptrMap; // Maps a pointer to an entry index // NOTE! Assumes the content stays in scope! diff --git a/source/slang/slang-serialize-container.cpp b/source/slang/slang-serialize-container.cpp new file mode 100644 index 000000000..6f6438947 --- /dev/null +++ b/source/slang/slang-serialize-container.cpp @@ -0,0 +1,553 @@ +// slang-serialize-container.cpp +#include "slang-serialize-container.h" + +#include "../core/slang-text-io.h" +#include "../core/slang-byte-encode-util.h" + +#include "../core/slang-math.h" + +#include "slang-compiler.h" +#include "slang-serialize-ast.h" +#include "slang-serialize-ir.h" +#include "slang-serialize-debug.h" + +namespace Slang { + +/* static */SlangResult SerialContainerUtil::requestToData(EndToEndCompileRequest* request, const WriteOptions& options, SerialContainerData& out) +{ + SLANG_UNUSED(options); + + out.clear(); + + auto linkage = request->getLinkage(); + auto sink = request->getSink(); + auto frontEndReq = request->getFrontEndReq(); + + for (TranslationUnitRequest* translationUnit : frontEndReq->translationUnits) + { + auto module = translationUnit->module; + auto irModule = module->getIRModule(); + + // Root AST node + auto moduleDecl = translationUnit->getModuleDecl(); + + SLANG_ASSERT(irModule || moduleDecl); + + SerialContainerData::TranslationUnit dstTranslationUnit; + dstTranslationUnit.astRootNode = moduleDecl; + dstTranslationUnit.irModule = irModule; + + out.translationUnits.add(dstTranslationUnit); + } + + auto program = request->getSpecializedGlobalAndEntryPointsComponentType(); + + // Add all the target modules + { + + for (auto target : linkage->targets) + { + auto targetProgram = program->getTargetProgram(target); + auto irModule = targetProgram->getOrCreateIRModuleForLayout(sink); + + SerialContainerData::TargetModule targetModule; + + targetModule.irModule = irModule; + + auto& dstTarget = targetModule.target; + + dstTarget.floatingPointMode = target->floatingPointMode; + dstTarget.profile = target->targetProfile; + dstTarget.flags = target->targetFlags; + dstTarget.codeGenTarget = target->target; + + out.targetModules.add(targetModule); + } + } + + // Entry points + { + auto entryPointCount = program->getEntryPointCount(); + for (Index ii = 0; ii < entryPointCount; ++ii) + { + auto entryPoint = program->getEntryPoint(ii); + auto entryPointMangledName = program->getEntryPointMangledName(ii); + + SerialContainerData::EntryPoint dstEntryPoint; + + dstEntryPoint.name = entryPoint->getName(); + dstEntryPoint.mangledName = entryPointMangledName; + dstEntryPoint.profile = entryPoint->getProfile(); + + out.entryPoints.add(dstEntryPoint); + } + } + + return SLANG_OK; +} + +/* static */SlangResult SerialContainerUtil::write(const SerialContainerData& data, const WriteOptions& options, RiffContainer* container) +{ + //auto linkage = request->getLinkage(); + //auto sink = request->getSink(); + //auto frontEndReq = request->getFrontEndReq(); + //SourceManager* sourceManager = frontEndReq->getSourceManager(); + + RefPtr<DebugSerialWriter> debugWriter; + + // The string pool used across the whole of the container + StringSlicePool containerStringPool(StringSlicePool::Style::Default); + + RiffContainer::ScopeChunk scopeModule(container, RiffContainer::Chunk::Kind::List, SerialBinary::kContainerFourCc); + + // Write the header + { + SerialBinary::ContainerHeader containerHeader; + // Save the compression type if used - as can only serialize with a single compression type + containerHeader.compressionType = uint32_t(options.compressionType); + + RiffContainer::ScopeChunk scopeHeader(container, RiffContainer::Chunk::Kind::Data, SerialBinary::kContainerHeaderFourCc); + container->write(&containerHeader, sizeof(containerHeader)); + } + + if (data.translationUnits.getCount()) + { + // Module list + RiffContainer::ScopeChunk moduleListScope(container, RiffContainer::Chunk::Kind::List, SerialBinary::kTranslationUnitListFourCc); + + if (options.optionFlags & SerialOptionFlag::DebugInfo) + { + debugWriter = new DebugSerialWriter(options.sourceManager); + } + + RefPtr<ASTSerialClasses> astClasses = new ASTSerialClasses; + + for (const auto& translationUnit : data.translationUnits) + { + // Okay, we need to serialize this module to our container file. + // We currently don't serialize it's name..., but support for that could be added. + + // Write the IR information + if (translationUnit.irModule) + { + IRSerialData serialData; + IRSerialWriter writer; + SLANG_RETURN_ON_FAIL(writer.write(translationUnit.irModule, debugWriter, options.optionFlags, &serialData)); + SLANG_RETURN_ON_FAIL(IRSerialWriter::writeContainer(serialData, options.compressionType, container)); + } + + // Write the AST information + + if (ModuleDecl* moduleDecl = as<ModuleDecl>(translationUnit.astRootNode)) + { + ModuleASTSerialFilter filter(moduleDecl); + ASTSerialWriter writer(astClasses, &filter, debugWriter); + + // Add the module and everything that isn't filtered out in the filter. + writer.addPointer(moduleDecl); + + // We can now serialize it into the riff container. + SLANG_RETURN_ON_FAIL(writer.writeIntoContainer(container)); + } + } + + if (data.targetModules.getCount()) + { + // TODO: in the case where we have specialization, we might need + // to serialize IR related to `program`... + + for (const auto& targetModule : data.targetModules) + { + IRModule* irModule = targetModule.irModule; + + // Okay, we need to serialize this target program and its IR too... + IRSerialData serialData; + IRSerialWriter writer; + + SLANG_RETURN_ON_FAIL(writer.write(irModule, debugWriter, options.optionFlags, &serialData)); + SLANG_RETURN_ON_FAIL(IRSerialWriter::writeContainer(serialData, options.compressionType, container)); + } + } + } + + if (data.entryPoints.getCount()) + { + for (const auto& entryPoint : data.entryPoints) + { + RiffContainer::ScopeChunk entryPointScope(container, RiffContainer::Chunk::Kind::Data, SerialBinary::kEntryPointFourCc); + + SerialContainerBinary::EntryPoint dst; + + dst.name = uint32_t(containerStringPool.add(entryPoint.name->text)); + dst.profile = entryPoint.profile.raw; + dst.mangledName = uint32_t(containerStringPool.add(entryPoint.mangledName)); + + container->write(&dst, sizeof(dst)); + } + } + + // We can now output the debug information. This is for all IR and AST + if (debugWriter) + { + // Write out the debug info + DebugSerialData debugData; + debugWriter->write(&debugData); + + debugData.writeContainer(options.compressionType, container); + } + + // Write the container string table + if (containerStringPool.getAdded().getCount() > 0) + { + RiffContainer::ScopeChunk stringTableScope(container, RiffContainer::Chunk::Kind::Data, SerialBinary::kStringTableFourCc); + + List<char> encodedTable; + SerialStringTableUtil::encodeStringTable(containerStringPool, encodedTable); + + container->write(encodedTable.getBuffer(), encodedTable.getCount()); + } + + return SLANG_OK; +} + +/* static */Result SerialContainerUtil::read(RiffContainer* container, const ReadOptions& options, SerialContainerData& out) +{ + out.clear(); + + RiffContainer::ListChunk* containerChunk = container->getRoot()->findListRec(SerialBinary::kContainerFourCc); + if (!containerChunk) + { + // Must be a container + return SLANG_FAIL; + } + + SerialBinary::ContainerHeader* containerHeader = containerChunk->findContainedData<SerialBinary::ContainerHeader>(SerialBinary::kContainerHeaderFourCc); + if (!containerHeader) + { + // Didn't find the header + return SLANG_FAIL; + } + + const SerialCompressionType containerCompressionType = SerialCompressionType(containerHeader->compressionType); + + StringSlicePool containerStringPool(StringSlicePool::Style::Default); + + if (RiffContainer::Data* stringTableData = containerChunk->findContainedData(SerialBinary::kStringTableFourCc)) + { + SerialStringTableUtil::decodeStringTable((const char*)stringTableData->getPayload(), stringTableData->getSize(), containerStringPool); + } + + RefPtr<DebugSerialReader> debugReader; + RefPtr<ASTSerialClasses> astClasses; + + // Debug information + if (auto debugChunk = containerChunk->findContainedList(DebugSerialData::kDebugFourCc)) + { + // Read into data + DebugSerialData debugData; + SLANG_RETURN_ON_FAIL(debugData.readContainer(containerCompressionType, debugChunk)); + + // Turn into DebugReader + debugReader = new DebugSerialReader; + SLANG_RETURN_ON_FAIL(debugReader->read(&debugData, options.sourceManager)); + } + + // Add translation units + if (RiffContainer::ListChunk* translationUnits = containerChunk->findContainedList(SerialBinary::kTranslationUnitListFourCc)) + { + RiffContainer::Chunk* chunk = translationUnits->getFirstContainedChunk(); + while (chunk) + { + auto startChunk = chunk; + + RefPtr<ASTBuilder> astBuilder; + NodeBase* astRootNode = nullptr; + RefPtr<IRModule> irModule; + + if (auto irChunk = as<RiffContainer::ListChunk>(chunk, IRSerialBinary::kIRModuleFourCc)) + { + IRSerialData serialData; + + SLANG_RETURN_ON_FAIL(IRSerialReader::readContainer(irChunk, containerCompressionType, &serialData)); + + // Read IR back from serialData + IRSerialReader reader; + SLANG_RETURN_ON_FAIL(reader.read(serialData, options.session, debugReader, irModule)); + + // Onto next chunk + chunk = chunk->m_next; + } + + if (auto astChunk = as<RiffContainer::ListChunk>(chunk, ASTSerialBinary::kSlangASTModuleFourCC)) + { + RiffContainer::Data* astData = astChunk->findContainedData(ASTSerialBinary::kSlangASTModuleDataFourCC); + + if (astData) + { + if (!astClasses) + { + astClasses = new ASTSerialClasses; + } + + // TODO(JS): We probably want to store off better information about each of the translation unit + // including some kind of 'name'. + // For now we just generate a name. + + StringBuilder buf; + buf << "tu" << out.translationUnits.getCount(); + + astBuilder = new ASTBuilder(options.sharedASTBuilder, buf.ProduceString()); + + ASTSerialReader reader(astClasses, debugReader); + + SLANG_RETURN_ON_FAIL(reader.load((const uint8_t*)astData->getPayload(), astData->getSize(), astBuilder, options.namePool)); + + // Get the root node. It's at index 1 (0 is the null value). + astRootNode = reader.getPointer(ASTSerialIndex(1)).dynamicCast<NodeBase>(); + } + + // Onto next chunk + chunk = chunk->m_next; + } + + if (astBuilder || irModule) + { + SerialContainerData::TranslationUnit translationUnit; + + translationUnit.astBuilder = astBuilder; + translationUnit.astRootNode = astRootNode; + translationUnit.irModule = irModule; + + out.translationUnits.add(translationUnit); + } + + // If no progress, step to next chunk + chunk = (chunk == startChunk) ? chunk->m_next : chunk; + } + } + + // Add all the entry points + { + List<RiffContainer::DataChunk*> entryPointChunks; + containerChunk->findContained(SerialBinary::kEntryPointFourCc, entryPointChunks); + + for (auto entryPointChunk : entryPointChunks) + { + auto reader = entryPointChunk->asReadHelper(); + + auto readString = [&]() + { + uint32_t length = 0; + reader.read(length); + + char* begin = (char*)reader.getData(); + reader.skip(length + 1); + + return UnownedStringSlice(begin, begin + length); + }; + + SerialContainerBinary::EntryPoint srcEntryPoint; + SLANG_RETURN_ON_FAIL(reader.read(srcEntryPoint)); + + SerialContainerData::EntryPoint dstEntryPoint; + + dstEntryPoint.name = options.namePool->getName(containerStringPool.getSlice(StringSlicePool::Handle(srcEntryPoint.name))); + dstEntryPoint.profile.raw = srcEntryPoint.profile; + dstEntryPoint.mangledName = containerStringPool.getSlice(StringSlicePool::Handle(srcEntryPoint.mangledName)); + + out.entryPoints.add(dstEntryPoint); + } + } + + return SLANG_OK; +} + +/* static */SlangResult SerialContainerUtil::verifyIRSerialize(IRModule* module, Session* session, const WriteOptions& options) +{ + // Verify if we can stream out with raw source locs + + List<IRInst*> originalInsts; + IRSerialWriter::calcInstructionList(module, originalInsts); + + IRSerialData irData; + + OwnedMemoryStream memoryStream(FileAccess::ReadWrite); + + { + RiffContainer riffContainer; + + // Need to put all of this in a container + RiffContainer::ScopeChunk containerScope(&riffContainer, RiffContainer::Chunk::Kind::List, SerialBinary::kContainerFourCc); + + RefPtr<DebugSerialWriter> debugWriter; + + if (options.optionFlags & SerialOptionFlag::DebugInfo) + { + debugWriter = new DebugSerialWriter(options.sourceManager); + } + + { + // Write IR out to serialData - copying over SourceLoc information directly + IRSerialWriter writer; + SLANG_RETURN_ON_FAIL(writer.write(module, debugWriter, options.optionFlags, &irData)); + } + SLANG_RETURN_ON_FAIL(IRSerialWriter::writeContainer(irData, options.compressionType, &riffContainer)); + + // Write the debug info Riff container + if (debugWriter) + { + DebugSerialData serialData; + debugWriter->write(&serialData); + + SLANG_RETURN_ON_FAIL(serialData.writeContainer(options.compressionType, &riffContainer)); + } + + SLANG_RETURN_ON_FAIL(RiffUtil::write(&riffContainer, &memoryStream)); + } + + // Reset stream + memoryStream.seek(SeekOrigin::Start, 0); + + SourceManager workSourceManager; + workSourceManager.initialize(options.sourceManager, nullptr); + + // The read ir module + RefPtr<IRModule> irReadModule; + { + RiffContainer riffContainer; + SLANG_RETURN_ON_FAIL(RiffUtil::read(&memoryStream, riffContainer)); + + RiffContainer::ListChunk* rootList = riffContainer.getRoot(); + + RefPtr<DebugSerialReader> debugReader; + + // If we have debug info then find and read it + if (options.optionFlags & SerialOptionFlag::DebugInfo) + { + RiffContainer::ListChunk* debugList = rootList->findContainedList(DebugSerialData::kDebugFourCc); + if (!debugList) + { + return SLANG_FAIL; + } + DebugSerialData debugData; + SLANG_RETURN_ON_FAIL(debugData.readContainer(options.compressionType, debugList)); + + debugReader = new DebugSerialReader; + SLANG_RETURN_ON_FAIL(debugReader->read(&debugData, &workSourceManager)); + } + + { + RiffContainer::ListChunk* irList = rootList->findContainedList(IRSerialBinary::kIRModuleFourCc); + if (!irList) + { + return SLANG_FAIL; + } + + { + IRSerialData irReadData; + IRSerialReader reader; + SLANG_RETURN_ON_FAIL(reader.readContainer(irList, options.compressionType, &irReadData)); + + // Check the stream read data is the same + if (irData != irReadData) + { + SLANG_ASSERT(!"Streamed in data doesn't match"); + return SLANG_FAIL; + } + + SLANG_RETURN_ON_FAIL(reader.read(irData, session, debugReader, irReadModule)); + } + } + } + + List<IRInst*> readInsts; + IRSerialWriter::calcInstructionList(irReadModule, readInsts); + + if (readInsts.getCount() != originalInsts.getCount()) + { + SLANG_ASSERT(!"Instruction counts don't match"); + return SLANG_FAIL; + } + + if (options.optionFlags & SerialOptionFlag::RawSourceLocation) + { + SLANG_ASSERT(readInsts[0] == originalInsts[0]); + // All the source locs should be identical + for (Index i = 1; i < readInsts.getCount(); ++i) + { + IRInst* origInst = originalInsts[i]; + IRInst* readInst = readInsts[i]; + + if (origInst->sourceLoc.getRaw() != readInst->sourceLoc.getRaw()) + { + SLANG_ASSERT(!"Source locs don't match"); + return SLANG_FAIL; + } + } + } + else if (options.optionFlags & SerialOptionFlag::DebugInfo) + { + // They should be on the same line nos + for (Index i = 1; i < readInsts.getCount(); ++i) + { + IRInst* origInst = originalInsts[i]; + IRInst* readInst = readInsts[i]; + + if (origInst->sourceLoc.getRaw() == readInst->sourceLoc.getRaw()) + { + continue; + } + + // Work out the + SourceView* origSourceView = options.sourceManager->findSourceView(origInst->sourceLoc); + SourceView* readSourceView = workSourceManager.findSourceView(readInst->sourceLoc); + + // if both are null we are done + if (origSourceView == nullptr && origSourceView == readSourceView) + { + continue; + } + SLANG_ASSERT(origSourceView && readSourceView); + + // The offset should be the same + Index origOffset = origInst->sourceLoc.getRaw() - origSourceView->getRange().begin.getRaw(); + Index readOffset = readInst->sourceLoc.getRaw() - readSourceView->getRange().begin.getRaw(); + + if (origOffset != readOffset) + { + SLANG_ASSERT(!"SourceLoc offset debug data didn't match"); + return SLANG_FAIL; + } + + { + auto origInfo = origSourceView->getHumaneLoc(origInst->sourceLoc, SourceLocType::Actual); + auto readInfo = readSourceView->getHumaneLoc(readInst->sourceLoc, SourceLocType::Actual); + + if (!(origInfo.line == readInfo.line && origInfo.column == readInfo.column && origInfo.pathInfo.foundPath == readInfo.pathInfo.foundPath)) + { + SLANG_ASSERT(!"Debug data didn't match"); + return SLANG_FAIL; + } + } + + // We may have adjusted line numbers -> but they may not match, because we only reconstruct one view + // So for now disable this test + + if (false) + { + auto origInfo = origSourceView->getHumaneLoc(origInst->sourceLoc, SourceLocType::Nominal); + auto readInfo = readSourceView->getHumaneLoc(readInst->sourceLoc, SourceLocType::Nominal); + + if (!(origInfo.line == readInfo.line && origInfo.column == readInfo.column && origInfo.pathInfo.foundPath == readInfo.pathInfo.foundPath)) + { + SLANG_ASSERT(!"Debug data didn't match"); + return SLANG_FAIL; + } + } + } + } + + return SLANG_OK; +} + +} // namespace Slang diff --git a/source/slang/slang-serialize-container.h b/source/slang/slang-serialize-container.h new file mode 100644 index 000000000..2f2f509b8 --- /dev/null +++ b/source/slang/slang-serialize-container.h @@ -0,0 +1,107 @@ +// slang-serialize-container.h +#ifndef SLANG_SERIALIZE_CONTAINER_H +#define SLANG_SERIALIZE_CONTAINER_H + +#include "../core/slang-riff.h" +#include "slang-serialize-types.h" +#include "slang-ir-insts.h" +#include "slang-profile.h" + +namespace Slang { + +class EndToEndCompileRequest; + +/* The binary representation actually held in riff/file format*/ +struct SerialContainerBinary +{ + struct Target + { + uint32_t target; + uint32_t flags; + uint32_t profile; + uint32_t floatingPointMode; + }; + + struct EntryPoint + { + uint32_t name; + uint32_t profile; + uint32_t mangledName; + }; +}; + +/* Struct that holds all the data that can be held in a 'container' */ +struct SerialContainerData +{ + struct Target + { + CodeGenTarget codeGenTarget = CodeGenTarget::Unknown; + SlangTargetFlags flags = 0; + Profile profile; + FloatingPointMode floatingPointMode = FloatingPointMode::Default; + }; + + struct TargetModule + { + // IR module for a specific compilation target + Target target; + RefPtr<IRModule> irModule; + }; + + struct TranslationUnit + { + RefPtr<IRModule> irModule; + RefPtr<ASTBuilder> astBuilder; + NodeBase* astRootNode = nullptr; + }; + + struct EntryPoint + { + Name* name = nullptr; + Profile profile; + String mangledName; + }; + + void clear() + { + entryPoints.clear(); + translationUnits.clear(); + targetModules.clear(); + } + + List<TranslationUnit> translationUnits; + List<TargetModule> targetModules; + List<EntryPoint> entryPoints; +}; + +struct SerialContainerUtil +{ + struct WriteOptions + { + SerialCompressionType compressionType = SerialCompressionType::VariableByteLite; + SerialOptionFlags optionFlags = 0; + SourceManager* sourceManager = nullptr; + }; + + struct ReadOptions + { + Session* session = nullptr; + SourceManager* sourceManager = nullptr; + NamePool* namePool = nullptr; + SharedASTBuilder* sharedASTBuilder = nullptr; + }; + + /// Get the serializable contents of the request as data + static SlangResult requestToData(EndToEndCompileRequest* request, const WriteOptions& options, SerialContainerData& outData); + + static SlangResult write(const SerialContainerData& data, const WriteOptions& options, RiffContainer* container); + + static SlangResult read(RiffContainer* container, const ReadOptions& options, SerialContainerData& out); + + /// Verify IR serialization + static SlangResult verifyIRSerialize(IRModule* module, Session* session, const WriteOptions& options); +}; + +} // namespace Slang + +#endif diff --git a/source/slang/slang-serialize-debug.cpp b/source/slang/slang-serialize-debug.cpp new file mode 100644 index 000000000..8f549e0f0 --- /dev/null +++ b/source/slang/slang-serialize-debug.cpp @@ -0,0 +1,404 @@ +// slang-serialize-debug.cpp +#include "slang-serialize-debug.h" + +#include "../core/slang-text-io.h" +#include "../core/slang-byte-encode-util.h" + +#include "../core/slang-math.h" + +namespace Slang { + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DebugSerialData !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +size_t DebugSerialData::calcSizeInBytes() const +{ + return SerialListUtil::calcArraySize(m_debugStringTable) + + SerialListUtil::calcArraySize(m_debugLineInfos) + + SerialListUtil::calcArraySize(m_debugSourceInfos) + + SerialListUtil::calcArraySize(m_debugAdjustedLineInfos); +} + +void DebugSerialData::clear() +{ + m_debugLineInfos.clear(); + m_debugAdjustedLineInfos.clear(); + m_debugSourceInfos.clear(); + m_debugStringTable.clear(); +} + + +bool DebugSerialData::operator==(const ThisType& rhs) const +{ + return (this == &rhs) || + ( SerialListUtil::isEqual(m_debugStringTable, rhs.m_debugStringTable) && + SerialListUtil::isEqual(m_debugLineInfos, rhs.m_debugLineInfos) && + SerialListUtil::isEqual(m_debugAdjustedLineInfos, rhs.m_debugAdjustedLineInfos) && + SerialListUtil::isEqual(m_debugSourceInfos, rhs.m_debugSourceInfos)); +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DebugSerialWriter !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +DebugSerialData::SourceLoc DebugSerialWriter::addSourceLoc(SourceLoc sourceLoc) +{ + // If it's not set we can ignore + if (!sourceLoc.isValid()) + { + return DebugSerialData::SourceLoc(0); + } + + // Look up the view it's from + SourceView* sourceView = m_sourceManager->findSourceView(sourceLoc); + if (!sourceView) + { + // If not found we just ingore + return DebugSerialData::SourceLoc(0); + } + + SourceFile* sourceFile = sourceView->getSourceFile(); + DebugSourceFile* debugSourceFile; + { + RefPtr<DebugSourceFile>* ptrDebugSourceFile = m_debugSourceFileMap.TryGetValue(sourceFile); + if (ptrDebugSourceFile == nullptr) + { + const SourceLoc::RawValue baseSourceLoc = m_debugFreeSourceLoc; + m_debugFreeSourceLoc += SourceLoc::RawValue(sourceView->getRange().getSize() + 1); + + debugSourceFile = new DebugSourceFile(sourceFile, baseSourceLoc); + m_debugSourceFileMap.Add(sourceFile, debugSourceFile); + } + else + { + debugSourceFile = *ptrDebugSourceFile; + } + } + + // We need to work out the line index + + int offset = sourceView->getRange().getOffset(sourceLoc); + int lineIndex = sourceFile->calcLineIndexFromOffset(offset); + + DebugSerialData::DebugLineInfo lineInfo; + lineInfo.m_lineStartOffset = sourceFile->getLineBreakOffsets()[lineIndex]; + lineInfo.m_lineIndex = lineIndex; + + if (!debugSourceFile->hasLineIndex(lineIndex)) + { + // Add the information about the line + int entryIndex = sourceView->findEntryIndex(sourceLoc); + if (entryIndex < 0) + { + debugSourceFile->m_lineInfos.add(lineInfo); + } + else + { + const auto& entry = sourceView->getEntries()[entryIndex]; + + DebugSerialData::DebugAdjustedLineInfo adjustedLineInfo; + adjustedLineInfo.m_lineInfo = lineInfo; + adjustedLineInfo.m_pathStringIndex = SerialStringData::kNullStringIndex; + + const auto& pool = sourceView->getSourceManager()->getStringSlicePool(); + SLANG_ASSERT(pool.getStyle() == StringSlicePool::Style::Default); + + if (!pool.isDefaultHandle(entry.m_pathHandle)) + { + UnownedStringSlice slice = pool.getSlice(entry.m_pathHandle); + SLANG_ASSERT(slice.getLength() > 0); + adjustedLineInfo.m_pathStringIndex = DebugSerialData::StringIndex(m_debugStringSlicePool.add(slice)); + } + + adjustedLineInfo.m_adjustedLineIndex = lineIndex + entry.m_lineAdjust; + + debugSourceFile->m_adjustedLineInfos.add(adjustedLineInfo); + } + + debugSourceFile->setHasLineIndex(lineIndex); + } + + return DebugSerialData::SourceLoc(debugSourceFile->m_baseSourceLoc + offset); +} + +void DebugSerialWriter::write(DebugSerialData* outDebugData) +{ + outDebugData->clear(); + + // Okay we can now calculate the final source information + + for (auto& pair : m_debugSourceFileMap) + { + DebugSourceFile* debugSourceFile = pair.Value; + SourceFile* sourceFile = debugSourceFile->m_sourceFile; + + DebugSerialData::DebugSourceInfo sourceInfo; + + sourceInfo.m_numLines = uint32_t(debugSourceFile->m_sourceFile->getLineBreakOffsets().getCount()); + + sourceInfo.m_range.m_start = uint32_t(debugSourceFile->m_baseSourceLoc); + sourceInfo.m_range.m_end = uint32_t(debugSourceFile->m_baseSourceLoc + sourceFile->getContentSize()); + + sourceInfo.m_pathIndex = DebugSerialData::StringIndex(m_debugStringSlicePool.add(sourceFile->getPathInfo().foundPath)); + + sourceInfo.m_lineInfosStartIndex = uint32_t(outDebugData->m_debugLineInfos.getCount()); + sourceInfo.m_adjustedLineInfosStartIndex = uint32_t(outDebugData->m_debugAdjustedLineInfos.getCount()); + + sourceInfo.m_numLineInfos = uint32_t(debugSourceFile->m_lineInfos.getCount()); + sourceInfo.m_numAdjustedLineInfos = uint32_t(debugSourceFile->m_adjustedLineInfos.getCount()); + + // Add the line infos + outDebugData->m_debugLineInfos.addRange(debugSourceFile->m_lineInfos.begin(), debugSourceFile->m_lineInfos.getCount()); + outDebugData->m_debugAdjustedLineInfos.addRange(debugSourceFile->m_adjustedLineInfos.begin(), debugSourceFile->m_adjustedLineInfos.getCount()); + + // Add the source info + outDebugData->m_debugSourceInfos.add(sourceInfo); + } + + // Convert the string pool + SerialStringTableUtil::encodeStringTable(m_debugStringSlicePool, outDebugData->m_debugStringTable); +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DebugSerialReader !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +Index DebugSerialReader::findViewIndex(DebugSerialData::SourceLoc loc) +{ + for (Index i = 0; i < m_views.getCount(); ++i) + { + if (m_views[i].m_range.contains(loc)) + { + return i; + } + } + return -1; +} + + +int DebugSerialReader::calcFixSourceLoc(DebugSerialData::SourceLoc loc, DebugSerialData::SourceRange& outRange) +{ + if (m_lastViewIndex < 0 || !m_views[m_lastViewIndex].m_range.contains(loc)) + { + m_lastViewIndex = findViewIndex(loc); + } + + if (m_lastViewIndex < 0) + { + // Set an invalid range, as couldn't find + outRange = DebugSerialData::SourceRange::getInvalid(); + return 0; + } + + const auto& view = m_views[m_lastViewIndex]; + + SLANG_ASSERT(view.m_range.contains(loc)); + + outRange = view.m_range; + return view.m_sourceView->getRange().begin.getRaw() - view.m_range.m_start; +} + +SourceLoc DebugSerialReader::getSourceLoc(DebugSerialData::SourceLoc loc) +{ + if (loc != 0) + { + if (m_lastViewIndex >= 0) + { + const auto& view = m_views[m_lastViewIndex]; + if (view.m_range.contains(loc)) + { + return view.m_range.getSourceLoc(loc, view.m_sourceView); + } + } + + m_lastViewIndex = findViewIndex(loc); + if (m_lastViewIndex >= 0) + { + const auto& view = m_views[m_lastViewIndex]; + return view.m_range.getSourceLoc(loc, view.m_sourceView); + } + } + return SourceLoc(); +} + +SlangResult DebugSerialReader::read(const DebugSerialData* serialData, SourceManager* sourceManager) +{ + m_views.setCount(0); + + if (!sourceManager || serialData->m_debugSourceInfos.getCount() == 0) + { + return SLANG_OK; + } + + List<UnownedStringSlice> debugStringSlices; + SerialStringTableUtil::decodeStringTable(serialData->m_debugStringTable.getBuffer(), serialData->m_debugStringTable.getCount(), debugStringSlices); + + // All of the strings are placed in the manager (and its StringSlicePool) where the SourceView and SourceFile are constructed from + List<StringSlicePool::Handle> stringMap; + SerialStringTableUtil::calcStringSlicePoolMap(debugStringSlices, sourceManager->getStringSlicePool(), stringMap); + + // Construct the source files + const Index numSourceFiles = serialData->m_debugSourceInfos.getCount(); + + // These hold the views (and SourceFile as there is only one SourceFile per view) in the same order as the sourceInfos + m_views.setCount(numSourceFiles); + + for (Index i = 0; i < numSourceFiles; ++i) + { + const auto& srcSourceInfo = serialData->m_debugSourceInfos[i]; + + PathInfo pathInfo; + pathInfo.type = PathInfo::Type::FoundPath; + pathInfo.foundPath = debugStringSlices[UInt(srcSourceInfo.m_pathIndex)]; + + SourceFile* sourceFile = sourceManager->createSourceFileWithSize(pathInfo, srcSourceInfo.m_range.getCount()); + SourceView* sourceView = sourceManager->createSourceView(sourceFile, nullptr); + + // We need to accumulate all line numbers, for this source file, both adjusted and unadjusted + List<DebugSerialData::DebugLineInfo> lineInfos; + // Add the adjusted lines + { + lineInfos.setCount(srcSourceInfo.m_numAdjustedLineInfos); + const DebugSerialData::DebugAdjustedLineInfo* srcAdjustedLineInfos = serialData->m_debugAdjustedLineInfos.getBuffer() + srcSourceInfo.m_adjustedLineInfosStartIndex; + const int numAdjustedLines = int(srcSourceInfo.m_numAdjustedLineInfos); + for (int j = 0; j < numAdjustedLines; ++j) + { + lineInfos[j] = srcAdjustedLineInfos[j].m_lineInfo; + } + } + // Add regular lines + lineInfos.addRange(serialData->m_debugLineInfos.getBuffer() + srcSourceInfo.m_lineInfosStartIndex, srcSourceInfo.m_numLineInfos); + + // Put in sourceloc order + lineInfos.sort(); + + List<uint32_t> lineBreakOffsets; + + // We can now set up the line breaks array + const int numLines = int(srcSourceInfo.m_numLines); + lineBreakOffsets.setCount(numLines); + + { + const Index numLineInfos = lineInfos.getCount(); + Index lineIndex = 0; + + // Every line up and including should hold the same offset + for (Index lineInfoIndex = 0; lineInfoIndex < numLineInfos; ++lineInfoIndex) + { + const auto& lineInfo = lineInfos[lineInfoIndex]; + + const uint32_t offset = lineInfo.m_lineStartOffset; + SLANG_ASSERT(offset > 0); + const int finishIndex = int(lineInfo.m_lineIndex); + + SLANG_ASSERT(finishIndex < numLines); + + for (; lineIndex < finishIndex; ++lineIndex) + { + lineBreakOffsets[lineIndex] = offset - 1; + } + lineBreakOffsets[lineIndex] = offset; + lineIndex++; + } + + // Do the remaining lines + { + const uint32_t endOffset = srcSourceInfo.m_range.getCount(); + for (; lineIndex < numLines; ++lineIndex) + { + lineBreakOffsets[lineIndex] = endOffset; + } + } + } + + sourceFile->setLineBreakOffsets(lineBreakOffsets.getBuffer(), lineBreakOffsets.getCount()); + + if (srcSourceInfo.m_numAdjustedLineInfos) + { + List<DebugSerialData::DebugAdjustedLineInfo> adjustedLineInfos; + + int numEntries = int(srcSourceInfo.m_numAdjustedLineInfos); + + adjustedLineInfos.addRange(serialData->m_debugAdjustedLineInfos.getBuffer() + srcSourceInfo.m_adjustedLineInfosStartIndex, numEntries); + adjustedLineInfos.sort(); + + // Work out the views adjustments, and place in dstEntries + List<SourceView::Entry> dstEntries; + dstEntries.setCount(numEntries); + + const uint32_t sourceLocOffset = uint32_t(sourceView->getRange().begin.getRaw()); + + for (int j = 0; j < numEntries; ++j) + { + const auto& srcEntry = adjustedLineInfos[j]; + auto& dstEntry = dstEntries[j]; + + dstEntry.m_pathHandle = stringMap[int(srcEntry.m_pathStringIndex)]; + dstEntry.m_startLoc = SourceLoc::fromRaw(srcEntry.m_lineInfo.m_lineStartOffset + sourceLocOffset); + dstEntry.m_lineAdjust = int32_t(srcEntry.m_adjustedLineIndex) - int32_t(srcEntry.m_lineInfo.m_lineIndex); + } + + // Set the adjustments on the view + sourceView->setEntries(dstEntries.getBuffer(), dstEntries.getCount()); + } + + // Set the view and the source range + View& view = m_views[i]; + view.m_sourceView = sourceView; + view.m_range = srcSourceInfo.m_range; + } + + return SLANG_OK; +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DebugSerialData !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +/* static */Result DebugSerialData::writeContainer(SerialCompressionType moduleCompressionType, RiffContainer* container) +{ + RiffContainer::ScopeChunk debugChunkScope(container, RiffContainer::Chunk::Kind::List, DebugSerialData::kDebugFourCc); + + SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayUncompressedChunk(DebugSerialData::kDebugStringFourCc, m_debugStringTable, container)); + SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayUncompressedChunk(DebugSerialData::kDebugLineInfoFourCc, m_debugLineInfos, container)); + SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayUncompressedChunk(DebugSerialData::kDebugAdjustedLineInfoFourCc, m_debugAdjustedLineInfos, container)); + SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayChunk(moduleCompressionType, DebugSerialData::kDebugSourceInfoFourCc, m_debugSourceInfos, container)); + + return SLANG_OK; +} + +/* static */Result DebugSerialData::readContainer(SerialCompressionType moduleCompressionType, RiffContainer::ListChunk* listChunk) +{ + SLANG_ASSERT(listChunk->getSubType() == DebugSerialData::kDebugFourCc); + + clear(); + for (RiffContainer::Chunk* chunk = listChunk->m_containedChunks; chunk; chunk = chunk->m_next) + { + RiffContainer::DataChunk* dataChunk = as<RiffContainer::DataChunk>(chunk); + if (!dataChunk) + { + continue; + } + + switch (dataChunk->m_fourCC) + { + case DebugSerialData::kDebugStringFourCc: + { + SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayUncompressedChunk(dataChunk, m_debugStringTable)); + break; + } + case DebugSerialData::kDebugLineInfoFourCc: + { + SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayUncompressedChunk(dataChunk, m_debugLineInfos)); + break; + } + case DebugSerialData::kDebugAdjustedLineInfoFourCc: + { + SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayUncompressedChunk(dataChunk, m_debugAdjustedLineInfos)); + break; + } + case DebugSerialData::kDebugSourceInfoFourCc: + { + SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayChunk(moduleCompressionType, dataChunk, m_debugSourceInfos)); + break; + } + } + } + + return SLANG_OK; +} + +} // namespace Slang diff --git a/source/slang/slang-serialize-debug.h b/source/slang/slang-serialize-debug.h new file mode 100644 index 000000000..ce39a058d --- /dev/null +++ b/source/slang/slang-serialize-debug.h @@ -0,0 +1,225 @@ +// slang-serialize-debug.h +#ifndef SLANG_SERIALIZE_DEBUG_H +#define SLANG_SERIALIZE_DEBUG_H + +#include "../core/slang-riff.h" +#include "../core/slang-string-slice-pool.h" +#include "../core/slang-array-view.h" + +#include "slang-serialize-types.h" + +#include "slang-name.h" +#include "slang-source-loc.h" + +namespace Slang { + +class DebugSerialData +{ +public: + typedef DebugSerialData ThisType; + + typedef uint32_t SourceLoc; + typedef SerialStringData::StringIndex StringIndex; + + // The list that contains all the subsequent modules + static const FourCC kDebugFourCc = SLANG_FOUR_CC('S', 'd', 'e', 'b'); + + static const FourCC kDebugStringFourCc = SLANG_FOUR_CC('S', 'd', 's', 't'); + static const FourCC kDebugLineInfoFourCc = SLANG_FOUR_CC('S', 'd', 'l', 'n'); + static const FourCC kDebugAdjustedLineInfoFourCc = SLANG_FOUR_CC('S', 'd', 'a', 'l'); + static const FourCC kDebugSourceInfoFourCc = SLANG_FOUR_CC('S', 'd', 's', 'o'); + + struct SourceRange + { + typedef SourceRange ThisType; + + bool operator==(const ThisType& rhs) const { return m_start == rhs.m_start && m_end == rhs.m_end; } + bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } + + Slang::SourceLoc getSourceLoc(SourceLoc loc, SourceView* view) const + { + return (loc && contains(loc)) ? Slang::SourceLoc::fromRaw((loc - m_start) + SourceLoc(view->getRange().begin.getRaw())) : Slang::SourceLoc(); + } + + SourceLoc getCount() const { return m_end - m_start; } + + bool contains(SourceLoc loc) const { return loc >= m_start && loc <= m_end; } + + /// Set up a range that can't occur in practice + static SourceRange getInvalid() { return SourceRange{ ~SourceLoc(0), ~SourceLoc(0) }; } + + SourceLoc m_start; ///< The offset to the source + SourceLoc m_end; ///< The number of bytes in the source + }; + + struct DebugSourceInfo + { + typedef DebugSourceInfo ThisType; + + bool operator==(const ThisType& rhs) const + { + return m_pathIndex == rhs.m_pathIndex && + m_range == rhs.m_range && + m_numLineInfos == rhs.m_numLineInfos && + m_lineInfosStartIndex == rhs.m_lineInfosStartIndex && + m_numLineInfos == rhs.m_numLineInfos; + } + bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } + + StringIndex m_pathIndex; ///< Index to the string table + + SourceRange m_range; ///< The range of locations + + uint32_t m_numLines; ///< Total number of lines in source file + + uint32_t m_lineInfosStartIndex; ///< Index into m_debugLineInfos + uint32_t m_numLineInfos; ///< The number of line infos + + uint32_t m_adjustedLineInfosStartIndex; ///< Adjusted start index + uint32_t m_numAdjustedLineInfos; ///< The number of line infos + }; + + struct DebugLineInfo + { + typedef DebugLineInfo ThisType; + bool operator<(const ThisType& rhs) const { return m_lineStartOffset < rhs.m_lineStartOffset; } + bool operator==(const ThisType& rhs) const + { + return m_lineStartOffset == rhs.m_lineStartOffset && + m_lineIndex == rhs.m_lineIndex; + } + bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } + + uint32_t m_lineStartOffset; ///< The offset into the source file + uint32_t m_lineIndex; ///< Original line index + }; + + struct DebugAdjustedLineInfo + { + typedef DebugAdjustedLineInfo ThisType; + bool operator==(const ThisType& rhs) const + { + return m_lineInfo == rhs.m_lineInfo && + m_adjustedLineIndex == rhs.m_adjustedLineIndex && + m_pathStringIndex == rhs.m_pathStringIndex; + } + bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } + bool operator<(const ThisType& rhs) const { return m_lineInfo < rhs.m_lineInfo; } + + DebugLineInfo m_lineInfo; + uint32_t m_adjustedLineIndex; ///< The line index with the adjustment (if there is any). Is 0 if m_pathStringIndex is 0. + StringIndex m_pathStringIndex; ///< The path as an index + }; + + size_t calcSizeInBytes() const; + void clear(); + + Index findSourceInfoIndex(SourceLoc sourceLoc) const + { + const Index numInfos = m_debugSourceInfos.getCount(); + for (Index i = 0; i < numInfos; ++i) + { + if (m_debugSourceInfos[i].m_range.contains(sourceLoc)) + { + return i; + } + } + return -1; + } + + bool operator==(const ThisType& rhs) const; + + Result writeContainer(SerialCompressionType moduleCompressionType, RiffContainer* container); + Result readContainer(SerialCompressionType moduleCompressionType, RiffContainer::ListChunk* listChunk); + + // Data only set if we have debug information + + List<char> m_debugStringTable; ///< String table for debug use only + List<DebugLineInfo> m_debugLineInfos; ///< Debug line information + List<DebugAdjustedLineInfo> m_debugAdjustedLineInfos; ///< Adjusted line infos + List<DebugSourceInfo> m_debugSourceInfos; ///< Debug source information +}; + +class DebugSerialReader : public RefObject +{ +public: + Index findViewIndex(DebugSerialData::SourceLoc loc); + + SourceLoc getSourceLoc(DebugSerialData::SourceLoc loc); + + /// Works out the amount to fix an input source loc to get a regular Slang::SourceLoc + int calcFixSourceLoc(DebugSerialData::SourceLoc loc, DebugSerialData::SourceRange& outRange); + + /// Calc the loc + static SourceLoc calcFixedLoc(DebugSerialData::SourceLoc loc, int fix, const DebugSerialData::SourceRange& range) { SLANG_ASSERT(range.contains(loc)); SLANG_UNUSED(range); return SourceLoc::fromRaw(SourceLoc::RawValue(loc + fix)); } + + SlangResult read(const DebugSerialData* serialData, SourceManager* sourceManager); + +protected: + struct View + { + DebugSerialData::SourceRange m_range; + SourceView* m_sourceView; + }; + + List<View> m_views; ///< All the views + Index m_lastViewIndex = -1; ///< Caches last lookup +}; + +/// Used to write serialized Debug information +class DebugSerialWriter : public RefObject +{ +public: + + class DebugSourceFile : public RefObject + { + public: + DebugSourceFile(SourceFile* sourceFile, SourceLoc::RawValue baseSourceLoc) : + m_sourceFile(sourceFile), + m_baseSourceLoc(baseSourceLoc) + { + // Need to know how many lines there are + const List<uint32_t>& lineOffsets = sourceFile->getLineBreakOffsets(); + + const auto numLineIndices = lineOffsets.getCount(); + + // Set none as being used initially + m_lineIndexUsed.setCount(numLineIndices); + ::memset(m_lineIndexUsed.begin(), 0, numLineIndices * sizeof(uint8_t)); + } + /// True if we have information on that line index + bool hasLineIndex(int lineIndex) const { return m_lineIndexUsed[lineIndex] != 0; } + void setHasLineIndex(int lineIndex) { m_lineIndexUsed[lineIndex] = 1; } + + SourceLoc::RawValue m_baseSourceLoc; ///< The base source location + + SourceFile* m_sourceFile; ///< The source file + List<uint8_t> m_lineIndexUsed; ///< Has 1 if the line is used + List<uint32_t> m_usedLineIndices; ///< Holds the lines that have been hit + + List<DebugSerialData::DebugLineInfo> m_lineInfos; ///< The line infos + List<DebugSerialData::DebugAdjustedLineInfo> m_adjustedLineInfos; ///< The adjusted line infos + }; + + /// Add a source location. Returns the location that can be serialized. + DebugSerialData::SourceLoc addSourceLoc(SourceLoc sourceLoc); + + /// Write into outDebugData + void write(DebugSerialData* outDebugData); + + DebugSerialWriter(SourceManager* sourceManager): + m_sourceManager(sourceManager), + m_debugStringSlicePool(StringSlicePool::Style::Default), + m_debugFreeSourceLoc(1) + { + } + + SourceManager* m_sourceManager; + StringSlicePool m_debugStringSlicePool; ///< Slices held just for debug usage + SourceLoc::RawValue m_debugFreeSourceLoc; /// Locations greater than this are free + Dictionary<SourceFile*, RefPtr<DebugSourceFile> > m_debugSourceFileMap; +}; + +} // namespace Slang + +#endif diff --git a/source/slang/slang-serialize-ir-types.cpp b/source/slang/slang-serialize-ir-types.cpp new file mode 100644 index 000000000..cceb72463 --- /dev/null +++ b/source/slang/slang-serialize-ir-types.cpp @@ -0,0 +1,102 @@ +// slang-serialize-ir-types.cpp +#include "slang-serialize-ir-types.h" + +#include "../core/slang-text-io.h" +#include "../core/slang-byte-encode-util.h" + +#include "slang-ir-insts.h" + +#include "../core/slang-math.h" + +namespace Slang { + +/* Note that an IRInst can be derived from, but when it derived from it's new members are IRUse variables, and they in +effect alias over the operands - and reflected in the operand count. There _could_ be other members after these IRUse +variables, but only a few types include extra data, and these do not have any operands: + +* IRConstant - Needs special-case handling +* IRModuleInst - Presumably we can just set to the module pointer on reconstruction + +Note! That on an IRInst there is an IRType* variable (accessed as getFullType()). As it stands it may NOT actually point +to an IRType derived type. Its 'ok' as long as it's an instruction that can be used in the place of the type. So this code does not +bother to check if it's correct, and just casts it. +*/ + +/* static */const IRSerialData::PayloadInfo IRSerialData::s_payloadInfos[int(Inst::PayloadType::CountOf)] = +{ + { 0, 0 }, // Empty + { 1, 0 }, // Operand_1 + { 2, 0 }, // Operand_2 + { 1, 0 }, // OperandAndUInt32, + { 0, 0 }, // OperandExternal - This isn't correct, Operand has to be specially handled + { 0, 1 }, // String_1, + { 0, 2 }, // String_2, + { 0, 0 }, // UInt32, + { 0, 0 }, // Float64, + { 0, 0 } // Int64, +}; + +// Check all compressible chunk ids, start with upper case 'S' +SLANG_COMPILE_TIME_ASSERT(SLANG_FOUR_CC_GET_FIRST_CHAR(IRSerialBinary::kInstFourCc) == 'S'); +SLANG_COMPILE_TIME_ASSERT(SLANG_FOUR_CC_GET_FIRST_CHAR(IRSerialBinary::kChildRunFourCc) == 'S'); +SLANG_COMPILE_TIME_ASSERT(SLANG_FOUR_CC_GET_FIRST_CHAR(IRSerialBinary::kExternalOperandsFourCc) == 'S'); + +// Compressed version starts with 's' +SLANG_COMPILE_TIME_ASSERT(SLANG_FOUR_CC_GET_FIRST_CHAR(SLANG_MAKE_COMPRESSED_FOUR_CC(IRSerialBinary::kInstFourCc)) == 's'); + +struct PrefixString; + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! IRSerialData !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +template<typename T> +static size_t _calcArraySize(const List<T>& list) +{ + return list.getCount() * sizeof(T); +} + +size_t IRSerialData::calcSizeInBytes() const +{ + return + _calcArraySize(m_insts) + + _calcArraySize(m_childRuns) + + _calcArraySize(m_externalOperands) + + _calcArraySize(m_stringTable) + + /* Raw source locs */ + _calcArraySize(m_rawSourceLocs) + + /* Debug */ + _calcArraySize(m_debugSourceLocRuns); +} + +IRSerialData::IRSerialData() +{ + clear(); +} + +void IRSerialData::clear() +{ + // First Instruction is null + m_insts.setCount(1); + memset(&m_insts[0], 0, sizeof(Inst)); + + m_childRuns.clear(); + m_externalOperands.clear(); + m_rawSourceLocs.clear(); + + m_stringTable.clear(); + + m_debugSourceLocRuns.clear(); +} + +bool IRSerialData::operator==(const ThisType& rhs) const +{ + return (this == &rhs) || + (SerialListUtil::isEqual(m_insts, rhs.m_insts) && + SerialListUtil::isEqual(m_childRuns, rhs.m_childRuns) && + SerialListUtil::isEqual(m_externalOperands, rhs.m_externalOperands) && + SerialListUtil::isEqual(m_rawSourceLocs, rhs.m_rawSourceLocs) && + SerialListUtil::isEqual(m_stringTable, rhs.m_stringTable) && + /* Debug */ + SerialListUtil::isEqual(m_debugSourceLocRuns, rhs.m_debugSourceLocRuns)); +} + +} // namespace Slang diff --git a/source/slang/slang-ir-serialize-types.h b/source/slang/slang-serialize-ir-types.h index 3d006b86e..513212ad6 100644 --- a/source/slang/slang-ir-serialize-types.h +++ b/source/slang/slang-serialize-ir-types.h @@ -1,11 +1,14 @@ -// slang-ir-serialize-types.h -#ifndef SLANG_IR_SERIALIZE_TYPES_H_INCLUDED -#define SLANG_IR_SERIALIZE_TYPES_H_INCLUDED +// slang-serialize-ir-types.h +#ifndef SLANG_SERIALIZE_IR_TYPES_H_INCLUDED +#define SLANG_SERIALIZE_IR_TYPES_H_INCLUDED #include "../core/slang-riff.h" #include "../core/slang-string-slice-pool.h" #include "../core/slang-array-view.h" +#include "slang-serialize-types.h" +#include "slang-serialize-debug.h" + #include "slang-name.h" #include "slang-source-loc.h" @@ -13,74 +16,44 @@ namespace Slang { -enum class IRSerialCompressionType : uint8_t -{ - None, - VariableByteLite, -}; +// Pre-declare +class Name; -class StringRepresentationCache +struct IRSerialBinary { - // TODO: The name of this type is no longer accurate to its function. - // It doesn't *cache* anything, and instead just provides convenient - // access to the contents of a serialized string table. + /// IR module list + static const FourCC kIRModuleFourCc = SLANG_FOUR_CC('S', 'i', 'm', 'd'); - public: - typedef StringSlicePool::Handle Handle; - - struct Entry - { - uint32_t m_startIndex; - uint32_t m_numChars; - }; + /* NOTE! All FourCC that can be compressed must start with capital 'S', because compressed version is the same FourCC + with the 'S' replaced with 's' */ - /// Get as a string slice - UnownedStringSlice getStringSlice(Handle handle) const; + static const FourCC kInstFourCc = SLANG_FOUR_CC('S', 'L', 'i', 'n'); + static const FourCC kChildRunFourCc = SLANG_FOUR_CC('S', 'L', 'c', 'r'); + static const FourCC kExternalOperandsFourCc = SLANG_FOUR_CC('S', 'L', 'e', 'o'); - /// Initialize a cache to use a string table - void init(const List<char>* stringTable); + static const FourCC kCompressedInstFourCc = SLANG_MAKE_COMPRESSED_FOUR_CC(kInstFourCc); + static const FourCC kCompressedChildRunFourCc = SLANG_MAKE_COMPRESSED_FOUR_CC(kChildRunFourCc); + static const FourCC kCompressedExternalOperandsFourCc = SLANG_MAKE_COMPRESSED_FOUR_CC(kExternalOperandsFourCc); - /// Ctor - StringRepresentationCache(); - - protected: - const List<char>* m_stringTable; - List<Entry> m_entries; -}; + static const FourCC kUInt32RawSourceLocFourCc = SLANG_FOUR_CC('S', 'r', 's', '4'); -struct SerialStringTableUtil -{ - /// Convert a pool into a string table - static void encodeStringTable(const StringSlicePool& pool, List<char>& stringTable); - static void encodeStringTable(const ConstArrayView<UnownedStringSlice>& slices, List<char>& stringTable); - /// Appends the decoded strings into slicesOut - static void appendDecodedStringTable(const List<char>& stringTable, List<UnownedStringSlice>& slicesOut); - /// Decodes a string table (and does so such that the indices are compatible with StringSlicePool) - static void decodeStringTable(const List<char>& stringTable, List<UnownedStringSlice>& slicesOut); - - /// Produces an index map, from slices to indices in pool - static void calcStringSlicePoolMap(const List<UnownedStringSlice>& slices, StringSlicePool& pool, List<StringSlicePool::Handle>& indexMap); + /// Debug information is held elsewhere, but if this optional section exists, it maps instructions to locs + static const FourCC kDebugSourceLocRunFourCc = SLANG_FOUR_CC('S', 'd', 's', 'r'); }; -// Pre-declare -class Name; - struct IRSerialData { typedef IRSerialData ThisType; + typedef SerialStringData::StringIndex StringIndex; + enum class InstIndex : uint32_t; - enum class StringIndex : uint32_t; enum class ArrayIndex : uint32_t; enum class RawSourceLoc : SourceLoc::RawValue; ///< This is just to copy over source loc data (ie not strictly serialize) - enum class StringOffset : uint32_t; ///< Offset into the m_stringsBuffer typedef uint32_t SizeType; - static const StringIndex kNullStringIndex = StringIndex(StringSlicePool::kNullHandle); - static const StringIndex kEmptyStringIndex = StringIndex(StringSlicePool::kEmptyHandle); - /// A run of instructions struct InstRun { @@ -106,7 +79,7 @@ struct IRSerialData bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } bool operator<(const ThisType& rhs) const { return m_sourceLoc < rhs.m_sourceLoc; } - uint32_t m_sourceLoc; ///< The source location + DebugSerialData::SourceLoc m_sourceLoc; ///< The source location InstIndex m_startInstIndex; ///< The index to the first instruction SizeType m_numInst; ///< The number of children }; @@ -117,68 +90,7 @@ struct IRSerialData uint8_t m_numStrings; }; - struct DebugSourceInfo - { - typedef DebugSourceInfo ThisType; - - bool operator==(const ThisType& rhs) const - { - return m_pathIndex == rhs.m_pathIndex && - m_startSourceLoc == rhs.m_startSourceLoc && - m_endSourceLoc == rhs.m_endSourceLoc && - m_numLineInfos == rhs.m_numLineInfos && - m_lineInfosStartIndex == rhs.m_lineInfosStartIndex && - m_numLineInfos == rhs.m_numLineInfos; - } - bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } - - bool isSourceLocInRange(uint32_t sourceLoc) const { return sourceLoc >= m_startSourceLoc && sourceLoc <= m_endSourceLoc; } - - StringIndex m_pathIndex; ///< Index to the string table - uint32_t m_startSourceLoc; ///< The offset to the source - uint32_t m_endSourceLoc; ///< The number of bytes in the source - - uint32_t m_numLines; ///< Total number of lines in source file - - uint32_t m_lineInfosStartIndex; ///< Index into m_debugLineInfos - uint32_t m_numLineInfos; ///< The number of line infos - - uint32_t m_adjustedLineInfosStartIndex; ///< Adjusted start index - uint32_t m_numAdjustedLineInfos; ///< The number of line infos - }; - - struct DebugLineInfo - { - typedef DebugLineInfo ThisType; - bool operator<(const ThisType& rhs) const { return m_lineStartOffset < rhs.m_lineStartOffset; } - bool operator==(const ThisType& rhs) const - { - return m_lineStartOffset == rhs.m_lineStartOffset && - m_lineIndex == rhs.m_lineIndex; - } - bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } - - uint32_t m_lineStartOffset; ///< The offset into the source file - uint32_t m_lineIndex; ///< Original line index - }; - - struct DebugAdjustedLineInfo - { - typedef DebugAdjustedLineInfo ThisType; - bool operator==(const ThisType& rhs) const - { - return m_lineInfo == rhs.m_lineInfo && - m_adjustedLineIndex == rhs.m_adjustedLineIndex && - m_pathStringIndex == rhs.m_pathStringIndex; - } - bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } - bool operator<(const ThisType& rhs) const { return m_lineInfo < rhs.m_lineInfo; } - - DebugLineInfo m_lineInfo; - uint32_t m_adjustedLineIndex; ///< The line index with the adjustment (if there is any). Is 0 if m_pathStringIndex is 0. - StringIndex m_pathStringIndex; ///< The path as an index - }; - + // Instruction... // We can store SourceLoc values separately. Just store per index information. // Parent information is stored in m_childRuns @@ -268,20 +180,14 @@ struct IRSerialData List<Inst> m_insts; ///< The instructions + List<RawSourceLoc> m_rawSourceLocs; ///< A source location per instruction (saved without modification from IRInst) + List<InstRun> m_childRuns; ///< Holds the information about children that belong to an instruction List<InstIndex> m_externalOperands; ///< Holds external operands (for instructions with more than kNumOperands) List<char> m_stringTable; ///< All strings. Indexed into by StringIndex - List<RawSourceLoc> m_rawSourceLocs; ///< A source location per instruction (saved without modification from IRInst)s - - // Data only set if we have debug information - - List<char> m_debugStringTable; ///< String table for debug use only - List<DebugLineInfo> m_debugLineInfos; ///< Debug line information - List<DebugAdjustedLineInfo> m_debugAdjustedLineInfos; ///< Adjusted line infos - List<DebugSourceInfo> m_debugSourceInfos; ///< Debug source information List<SourceLocRun> m_debugSourceLocRuns; ///< Runs of instructions that use a source loc static const PayloadInfo s_payloadInfos[int(Inst::PayloadType::CountOf)]; @@ -346,72 +252,6 @@ SLANG_FORCE_INLINE int IRSerialData::getOperands(const Inst& inst, const InstInd } } -// For types/FourCC that work for serializing in general (not just IR). Really this should be placed in some other header -struct SerialBinary -{ - static const FourCC kRiffFourCc = RiffFourCC::kRiff; - - static const FourCC kSlangModuleListFourCc = SLANG_FOUR_CC('S', 'L', 'm', 'l'); - - static const FourCC kEntryPointFourCc = SLANG_FOUR_CC('E', 'P', 'n', 't'); -}; - -// Replace first char with 's' -#define SLANG_MAKE_COMPRESSED_FOUR_CC(fourCc) SLANG_FOUR_CC_REPLACE_FIRST_CHAR(fourCc, 's') - -struct IRSerialBinary -{ - static const FourCC kSlangModuleFourCc = SLANG_FOUR_CC('S', 'L', 'm', 'd'); ///< Holds all the slang specific chunks - - static const FourCC kSlangModuleHeaderFourCc = SLANG_FOUR_CC('S', 'L', 'h', 'd'); - - /* NOTE! All FourCC that can be compressed must start with capital 'S', because compressed version is the same FourCC - with the 'S' replaced with 's' */ - - static const FourCC kInstFourCc = SLANG_FOUR_CC('S', 'L', 'i', 'n'); - static const FourCC kChildRunFourCc = SLANG_FOUR_CC('S', 'L', 'c', 'r'); - static const FourCC kExternalOperandsFourCc = SLANG_FOUR_CC('S', 'L', 'e', 'o'); - - static const FourCC kCompressedInstFourCc = SLANG_MAKE_COMPRESSED_FOUR_CC(kInstFourCc); - static const FourCC kCompressedChildRunFourCc = SLANG_MAKE_COMPRESSED_FOUR_CC(kChildRunFourCc); - static const FourCC kCompressedExternalOperandsFourCc = SLANG_MAKE_COMPRESSED_FOUR_CC(kExternalOperandsFourCc); - - static const FourCC kStringFourCc = SLANG_FOUR_CC('S', 'L', 's', 't'); - - static const FourCC kUInt32SourceLocFourCc = SLANG_FOUR_CC('S', 'r', 's', '4'); - - static const FourCC kDebugStringFourCc = SLANG_FOUR_CC('S', 'd', 's', 't'); - static const FourCC kDebugLineInfoFourCc = SLANG_FOUR_CC('S', 'd', 'l', 'n'); - static const FourCC kDebugAdjustedLineInfoFourCc = SLANG_FOUR_CC('S', 'd', 'a', 'l'); - static const FourCC kDebugSourceInfoFourCc = SLANG_FOUR_CC('S', 'd', 's', 'o'); - static const FourCC kDebugSourceLocRunFourCc = SLANG_FOUR_CC('S', 'd', 's', 'r'); - - typedef IRSerialCompressionType CompressionType; - - struct ModuleHeader - { - uint32_t compressionType; ///< Holds the compression type used (if used at all) - }; - struct ArrayHeader - { - uint32_t numEntries; - }; - struct CompressedArrayHeader - { - uint32_t numEntries; ///< The number of entries - uint32_t numCompressedEntries; ///< The amount of compressed entries - }; - -}; - -struct IRSerialTypeUtil -{ - /// Given text, finds the compression type - static SlangResult parseCompressionType(const UnownedStringSlice& text, IRSerialCompressionType& outType); - /// Given a compression type, return text - static UnownedStringSlice getText(IRSerialCompressionType type); -}; - } // namespace Slang diff --git a/source/slang/slang-serialize-ir.cpp b/source/slang/slang-serialize-ir.cpp new file mode 100644 index 000000000..ac8085d51 --- /dev/null +++ b/source/slang/slang-serialize-ir.cpp @@ -0,0 +1,910 @@ +// slang-serialize-ir.cpp +#include "slang-serialize-ir.h" + +#include "../core/slang-text-io.h" +#include "../core/slang-byte-encode-util.h" + +#include "slang-ir-insts.h" + +#include "../core/slang-math.h" + +namespace Slang { + +static bool _isTextureTypeBase(IROp opIn) +{ + const int op = (kIROpMeta_OpMask & opIn); + return op >= kIROp_FirstTextureTypeBase && op <= kIROp_LastTextureTypeBase; +} + +static bool _isConstant(IROp opIn) +{ + const int op = (kIROpMeta_OpMask & opIn); + return op >= kIROp_FirstConstant && op <= kIROp_LastConstant; +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! IRSerialWriter !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +void IRSerialWriter::_addInstruction(IRInst* inst) +{ + // It cannot already be in the map + SLANG_ASSERT(!m_instMap.ContainsKey(inst)); + + // Add to the map + m_instMap.Add(inst, Ser::InstIndex(m_insts.getCount())); + m_insts.add(inst); +} + +Result IRSerialWriter::_calcDebugInfo(DebugSerialWriter* debugWriter) +{ + // We need to find the unique source Locs + // We are not going to store SourceLocs directly, because there may be multiple views mapping down to + // the same underlying source file + + // First find all the unique locs + struct InstLoc + { + typedef InstLoc ThisType; + + SLANG_FORCE_INLINE bool operator<(const ThisType& rhs) const { return sourceLoc < rhs.sourceLoc || (sourceLoc == rhs.sourceLoc && instIndex < rhs.instIndex); } + + uint32_t instIndex; + uint32_t sourceLoc; + }; + + // Find all of the source locations and their associated instructions + List<InstLoc> instLocs; + const Index numInsts = m_insts.getCount(); + for (Index i = 1; i < numInsts; i++) + { + IRInst* srcInst = m_insts[i]; + if (!srcInst->sourceLoc.isValid()) + { + continue; + } + InstLoc instLoc; + instLoc.instIndex = uint32_t(i); + instLoc.sourceLoc = uint32_t(srcInst->sourceLoc.getRaw()); + instLocs.add(instLoc); + } + + // Sort them + instLocs.sort(); + + // Look for runs + const InstLoc* startInstLoc = instLocs.begin(); + const InstLoc* endInstLoc = instLocs.end(); + + while (startInstLoc < endInstLoc) + { + const uint32_t startSourceLoc = startInstLoc->sourceLoc; + + // Find the run with the same source loc + + const InstLoc* curInstLoc = startInstLoc + 1; + uint32_t curInstIndex = startInstLoc->instIndex + 1; + + // Find the run size with same source loc and run of instruction indices + for (; curInstLoc < endInstLoc && curInstLoc->sourceLoc == startSourceLoc && curInstLoc->instIndex == curInstIndex; ++curInstLoc, ++curInstIndex) + { + } + + // Add the run + + IRSerialData::SourceLocRun sourceLocRun; + sourceLocRun.m_numInst = curInstIndex - startInstLoc->instIndex;; + sourceLocRun.m_startInstIndex = IRSerialData::InstIndex(startInstLoc->instIndex); + sourceLocRun.m_sourceLoc = debugWriter->addSourceLoc(SourceLoc::fromRaw(startSourceLoc)); + + m_serialData->m_debugSourceLocRuns.add(sourceLocRun); + + // Next + startInstLoc = curInstLoc; + } + + return SLANG_OK; +} + +Result IRSerialWriter::write(IRModule* module, DebugSerialWriter* debugWriter, SerialOptionFlags options, IRSerialData* serialData) +{ + typedef Ser::Inst::PayloadType PayloadType; + + m_serialData = serialData; + + serialData->clear(); + + // We reserve 0 for null + m_insts.clear(); + m_insts.add(nullptr); + + // Reset + m_instMap.Clear(); + m_decorations.clear(); + + // Stack for parentInst + List<IRInst*> parentInstStack; + + IRModuleInst* moduleInst = module->getModuleInst(); + parentInstStack.add(moduleInst); + + // Add to the map + _addInstruction(moduleInst); + + // Traverse all of the instructions + while (parentInstStack.getCount()) + { + // If it's in the stack it is assumed it is already in the inst map + IRInst* parentInst = parentInstStack.getLast(); + parentInstStack.removeLast(); + SLANG_ASSERT(m_instMap.ContainsKey(parentInst)); + + // Okay we go through each of the children in order. If they are IRInstParent derived, we add to stack to process later + // cos we want breadth first so the order of children is the same as their index order, meaning we don't need to store explicit indices + const Ser::InstIndex startChildInstIndex = Ser::InstIndex(m_insts.getCount()); + + IRInstListBase childrenList = parentInst->getDecorationsAndChildren(); + for (IRInst* child : childrenList) + { + // This instruction can't be in the map... + SLANG_ASSERT(!m_instMap.ContainsKey(child)); + + _addInstruction(child); + + parentInstStack.add(child); + } + + // If it had any children, then store the information about it + if (Ser::InstIndex(m_insts.getCount()) != startChildInstIndex) + { + Ser::InstRun run; + run.m_parentIndex = m_instMap[parentInst]; + run.m_startInstIndex = startChildInstIndex; + run.m_numChildren = Ser::SizeType(m_insts.getCount() - int(startChildInstIndex)); + + m_serialData->m_childRuns.add(run); + } + } + +#if 0 + { + List<IRInst*> workInsts; + calcInstructionList(module, workInsts); + SLANG_ASSERT(workInsts.Count() == m_insts.Count()); + for (UInt i = 0; i < workInsts.Count(); ++i) + { + SLANG_ASSERT(workInsts[i] == m_insts[i]); + } + } +#endif + + // Set to the right size + m_serialData->m_insts.setCount(m_insts.getCount()); + // Clear all instructions + memset(m_serialData->m_insts.begin(), 0, sizeof(Ser::Inst) * m_serialData->m_insts.getCount()); + + // Need to set up the actual instructions + { + const Index numInsts = m_insts.getCount(); + + for (Index i = 1; i < numInsts; ++i) + { + IRInst* srcInst = m_insts[i]; + Ser::Inst& dstInst = m_serialData->m_insts[i]; + + dstInst.m_op = uint8_t(srcInst->op & kIROpMeta_OpMask); + dstInst.m_payloadType = PayloadType::Empty; + + dstInst.m_resultTypeIndex = getInstIndex(srcInst->getFullType()); + + IRConstant* irConst = as<IRConstant>(srcInst); + if (irConst) + { + switch (srcInst->op) + { + // Special handling for the ir const derived types + case kIROp_StringLit: + { + auto stringLit = static_cast<IRStringLit*>(srcInst); + dstInst.m_payloadType = PayloadType::String_1; + dstInst.m_payload.m_stringIndices[0] = getStringIndex(stringLit->getStringSlice()); + break; + } + case kIROp_IntLit: + { + dstInst.m_payloadType = PayloadType::Int64; + dstInst.m_payload.m_int64 = irConst->value.intVal; + break; + } + case kIROp_PtrLit: + { + dstInst.m_payloadType = PayloadType::Int64; + dstInst.m_payload.m_int64 = (intptr_t) irConst->value.ptrVal; + break; + } + case kIROp_FloatLit: + { + dstInst.m_payloadType = PayloadType::Float64; + dstInst.m_payload.m_float64 = irConst->value.floatVal; + break; + } + case kIROp_BoolLit: + { + dstInst.m_payloadType = PayloadType::UInt32; + dstInst.m_payload.m_uint32 = irConst->value.intVal ? 1 : 0; + break; + } + default: + { + SLANG_RELEASE_ASSERT(!"Unhandled constant type"); + return SLANG_FAIL; + } + } + continue; + } + + IRTextureTypeBase* textureBase = as<IRTextureTypeBase>(srcInst); + if (textureBase) + { + dstInst.m_payloadType = PayloadType::OperandAndUInt32; + dstInst.m_payload.m_operandAndUInt32.m_uint32 = uint32_t(srcInst->op) >> kIROpMeta_OtherShift; + dstInst.m_payload.m_operandAndUInt32.m_operand = getInstIndex(textureBase->getElementType()); + continue; + } + + // ModuleInst is different, in so far as it holds a pointer to IRModule, but we don't need + // to save that off in a special way, so can just use regular path + + const int numOperands = int(srcInst->operandCount); + Ser::InstIndex* dstOperands = nullptr; + + if (numOperands <= Ser::Inst::kMaxOperands) + { + // Checks the compile below is valid + SLANG_COMPILE_TIME_ASSERT(PayloadType(0) == PayloadType::Empty && PayloadType(1) == PayloadType::Operand_1 && PayloadType(2) == PayloadType::Operand_2); + + dstInst.m_payloadType = PayloadType(numOperands); + dstOperands = dstInst.m_payload.m_operands; + } + else + { + dstInst.m_payloadType = PayloadType::OperandExternal; + + int operandArrayBaseIndex = int(m_serialData->m_externalOperands.getCount()); + m_serialData->m_externalOperands.setCount(operandArrayBaseIndex + numOperands); + + dstOperands = m_serialData->m_externalOperands.begin() + operandArrayBaseIndex; + + auto& externalOperands = dstInst.m_payload.m_externalOperand; + externalOperands.m_arrayIndex = Ser::ArrayIndex(operandArrayBaseIndex); + externalOperands.m_size = Ser::SizeType(numOperands); + } + + for (int j = 0; j < numOperands; ++j) + { + const Ser::InstIndex dstInstIndex = getInstIndex(srcInst->getOperand(j)); + dstOperands[j] = dstInstIndex; + } + } + } + + // Convert strings into a string table + { + SerialStringTableUtil::encodeStringTable(m_stringSlicePool, serialData->m_stringTable); + } + + // If the option to use RawSourceLocations is enabled, serialize out as is + if (options & SerialOptionFlag::RawSourceLocation) + { + const Index numInsts = m_insts.getCount(); + serialData->m_rawSourceLocs.setCount(numInsts); + + Ser::RawSourceLoc* dstLocs = serialData->m_rawSourceLocs.begin(); + // 0 is null, just mark as no location + dstLocs[0] = Ser::RawSourceLoc(0); + for (Index i = 1; i < numInsts; ++i) + { + IRInst* srcInst = m_insts[i]; + dstLocs[i] = Ser::RawSourceLoc(srcInst->sourceLoc.getRaw()); + } + } + + if ((options & SerialOptionFlag::DebugInfo) && debugWriter) + { + _calcDebugInfo(debugWriter); + } + + m_serialData = nullptr; + return SLANG_OK; +} + +Result _encodeInsts(SerialCompressionType compressionType, const List<IRSerialData::Inst>& instsIn, List<uint8_t>& encodeArrayOut) +{ + typedef IRSerialBinary Bin; + typedef IRSerialData::Inst::PayloadType PayloadType; + + if (compressionType != SerialCompressionType::VariableByteLite) + { + return SLANG_FAIL; + } + + encodeArrayOut.clear(); + + const size_t numInsts = size_t(instsIn.getCount()); + const IRSerialData::Inst* insts = instsIn.begin(); + + uint8_t* encodeOut = encodeArrayOut.begin(); + uint8_t* encodeEnd = encodeArrayOut.end(); + + // Calculate the maximum instruction size with worst case possible encoding + // 2 bytes hold the payload size, and the result type + // Note that if there were some free bits, we could encode some of this stuff into bits, but if we remove payloadType, then there are no free bits + const size_t maxInstSize = 2 + ByteEncodeUtil::kMaxLiteEncodeUInt32 + Math::Max(sizeof(insts->m_payload.m_float64), size_t(2 * ByteEncodeUtil::kMaxLiteEncodeUInt32)); + + for (size_t i = 0; i < numInsts; ++i) + { + const auto& inst = insts[i]; + + // Make sure there is space for the largest possible instruction + if (encodeOut + maxInstSize >= encodeEnd) + { + const size_t offset = size_t(encodeOut - encodeArrayOut.begin()); + + const UInt oldCapacity = encodeArrayOut.getCapacity(); + + encodeArrayOut.reserve(oldCapacity + (oldCapacity >> 1) + maxInstSize); + const UInt capacity = encodeArrayOut.getCapacity(); + encodeArrayOut.setCount(capacity); + + encodeOut = encodeArrayOut.begin() + offset; + encodeEnd = encodeArrayOut.end(); + } + + *encodeOut++ = uint8_t(inst.m_op); + *encodeOut++ = uint8_t(inst.m_payloadType); + + encodeOut += ByteEncodeUtil::encodeLiteUInt32((uint32_t)inst.m_resultTypeIndex, encodeOut); + + switch (inst.m_payloadType) + { + case PayloadType::Empty: + { + break; + } + case PayloadType::Operand_1: + case PayloadType::String_1: + case PayloadType::UInt32: + { + // 1 UInt32 + encodeOut += ByteEncodeUtil::encodeLiteUInt32((uint32_t)inst.m_payload.m_operands[0], encodeOut); + break; + } + case PayloadType::Operand_2: + case PayloadType::OperandAndUInt32: + case PayloadType::OperandExternal: + case PayloadType::String_2: + { + // 2 UInt32 + encodeOut += ByteEncodeUtil::encodeLiteUInt32((uint32_t)inst.m_payload.m_operands[0], encodeOut); + encodeOut += ByteEncodeUtil::encodeLiteUInt32((uint32_t)inst.m_payload.m_operands[1], encodeOut); + break; + } + case PayloadType::Float64: + { + memcpy(encodeOut, &inst.m_payload.m_float64, sizeof(inst.m_payload.m_float64)); + encodeOut += sizeof(inst.m_payload.m_float64); + break; + } + case PayloadType::Int64: + { + memcpy(encodeOut, &inst.m_payload.m_int64, sizeof(inst.m_payload.m_int64)); + encodeOut += sizeof(inst.m_payload.m_int64); + break; + } + } + } + + // Fix the size + encodeArrayOut.setCount(UInt(encodeOut - encodeArrayOut.begin())); + return SLANG_OK; +} + +Result _writeInstArrayChunk(SerialCompressionType compressionType, FourCC chunkId, const List<IRSerialData::Inst>& array, RiffContainer* container) +{ + typedef RiffContainer::Chunk Chunk; + typedef RiffContainer::ScopeChunk ScopeChunk; + + typedef IRSerialBinary Bin; + if (array.getCount() == 0) + { + return SLANG_OK; + } + + switch (compressionType) + { + case SerialCompressionType::None: + { + return SerialRiffUtil::writeArrayChunk(compressionType, chunkId, array, container); + } + case SerialCompressionType::VariableByteLite: + { + List<uint8_t> compressedPayload; + SLANG_RETURN_ON_FAIL(_encodeInsts(compressionType, array, compressedPayload)); + + ScopeChunk scope(container, Chunk::Kind::Data, SLANG_MAKE_COMPRESSED_FOUR_CC(chunkId)); + + SerialBinary::CompressedArrayHeader header; + header.numEntries = uint32_t(array.getCount()); + header.numCompressedEntries = 0; + + container->write(&header, sizeof(header)); + container->write(compressedPayload.getBuffer(), compressedPayload.getCount()); + + return SLANG_OK; + } + default: break; + } + return SLANG_FAIL; +} + +/* static */Result IRSerialWriter::writeContainer(const IRSerialData& data, SerialCompressionType compressionType, RiffContainer* container) +{ + typedef RiffContainer::Chunk Chunk; + typedef RiffContainer::ScopeChunk ScopeChunk; + + ScopeChunk scopeModule(container, Chunk::Kind::List, Bin::kIRModuleFourCc); + + SLANG_RETURN_ON_FAIL(_writeInstArrayChunk(compressionType, Bin::kInstFourCc, data.m_insts, container)); + SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayChunk(compressionType, Bin::kChildRunFourCc, data.m_childRuns, container)); + SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayChunk(compressionType, Bin::kExternalOperandsFourCc, data.m_externalOperands, container)); + SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayChunk(SerialCompressionType::None, SerialBinary::kStringTableFourCc, data.m_stringTable, container)); + + SLANG_RETURN_ON_FAIL(SerialRiffUtil::writeArrayChunk(SerialCompressionType::None, Bin::kUInt32RawSourceLocFourCc, data.m_rawSourceLocs, container)); + + if (data.m_debugSourceLocRuns.getCount()) + { + SerialRiffUtil::writeArrayChunk(compressionType, Bin::kDebugSourceLocRunFourCc, data.m_debugSourceLocRuns, container); + } + + return SLANG_OK; +} + +/* static */void IRSerialWriter::calcInstructionList(IRModule* module, List<IRInst*>& instsOut) +{ + // We reserve 0 for null + instsOut.setCount(1); + instsOut[0] = nullptr; + + // Stack for parentInst + List<IRInst*> parentInstStack; + + IRModuleInst* moduleInst = module->getModuleInst(); + parentInstStack.add(moduleInst); + + // Add to list + instsOut.add(moduleInst); + + // Traverse all of the instructions + while (parentInstStack.getCount()) + { + // If it's in the stack it is assumed it is already in the inst map + IRInst* parentInst = parentInstStack.getLast(); + parentInstStack.removeLast(); + + IRInstListBase childrenList = parentInst->getDecorationsAndChildren(); + for (IRInst* child : childrenList) + { + instsOut.add(child); + parentInstStack.add(child); + } + } +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! IRSerialReader !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +static Result _decodeInsts(SerialCompressionType compressionType, const uint8_t* encodeCur, size_t encodeInSize, List<IRSerialData::Inst>& instsOut) +{ + const uint8_t* encodeEnd = encodeCur + encodeInSize; + + typedef IRSerialBinary Bin; + typedef IRSerialData::Inst::PayloadType PayloadType; + + if (compressionType != SerialCompressionType::VariableByteLite) + { + return SLANG_FAIL; + } + + const size_t numInsts = size_t(instsOut.getCount()); + IRSerialData::Inst* insts = instsOut.begin(); + + for (size_t i = 0; i < numInsts; ++i) + { + if (encodeCur >= encodeEnd) + { + SLANG_ASSERT(!"Invalid decode"); + return SLANG_FAIL; + } + + auto& inst = insts[i]; + + inst.m_op = *encodeCur++; + const PayloadType payloadType = PayloadType(*encodeCur++); + inst.m_payloadType = payloadType; + + // Read the result value + encodeCur += ByteEncodeUtil::decodeLiteUInt32(encodeCur, (uint32_t*)&inst.m_resultTypeIndex); + + switch (inst.m_payloadType) + { + case PayloadType::Empty: + { + break; + } + case PayloadType::Operand_1: + case PayloadType::String_1: + case PayloadType::UInt32: + { + // 1 UInt32 + encodeCur += ByteEncodeUtil::decodeLiteUInt32(encodeCur, (uint32_t*)&inst.m_payload.m_operands[0]); + break; + } + case PayloadType::Operand_2: + case PayloadType::OperandAndUInt32: + case PayloadType::OperandExternal: + case PayloadType::String_2: + { + // 2 UInt32 + encodeCur += ByteEncodeUtil::decodeLiteUInt32(encodeCur, 2, (uint32_t*)&inst.m_payload.m_operands[0]); + break; + } + case PayloadType::Float64: + { + memcpy(&inst.m_payload.m_float64, encodeCur, sizeof(inst.m_payload.m_float64)); + encodeCur += sizeof(inst.m_payload.m_float64); + break; + } + case PayloadType::Int64: + { + memcpy(&inst.m_payload.m_int64, encodeCur, sizeof(inst.m_payload.m_int64)); + encodeCur += sizeof(inst.m_payload.m_int64); + break; + } + } + } + + return SLANG_OK; +} + +static Result _readInstArrayChunk(SerialCompressionType containerCompressionType, RiffContainer::DataChunk* chunk, List<IRSerialData::Inst>& arrayOut) +{ + SerialCompressionType compressionType = SerialCompressionType::None; + if (chunk->m_fourCC == SLANG_MAKE_COMPRESSED_FOUR_CC(chunk->m_fourCC)) + { + compressionType = SerialCompressionType(containerCompressionType); + } + + switch (compressionType) + { + case SerialCompressionType::None: + { + SerialRiffUtil::ListResizerForType<IRSerialData::Inst> resizer(arrayOut); + return SerialRiffUtil::readArrayChunk(compressionType, chunk, resizer); + } + case SerialCompressionType::VariableByteLite: + { + RiffReadHelper read = chunk->asReadHelper(); + + SerialBinary::CompressedArrayHeader header; + SLANG_RETURN_ON_FAIL(read.read(header)); + + arrayOut.setCount(header.numEntries); + + SLANG_RETURN_ON_FAIL(_decodeInsts(compressionType, read.getData(), read.getRemainingSize(), arrayOut)); + break; + } + default: + { + return SLANG_FAIL; + } + } + + return SLANG_OK; +} + +/* static */Result IRSerialReader::readContainer(RiffContainer::ListChunk* module, SerialCompressionType containerCompressionType, IRSerialData* outData) +{ + typedef IRSerialBinary Bin; + + outData->clear(); + + for (RiffContainer::Chunk* chunk = module->m_containedChunks; chunk; chunk = chunk->m_next) + { + RiffContainer::DataChunk* dataChunk = as<RiffContainer::DataChunk>(chunk); + if (!dataChunk) + { + continue; + } + + switch (dataChunk->m_fourCC) + { + case SLANG_MAKE_COMPRESSED_FOUR_CC(Bin::kInstFourCc): + case Bin::kInstFourCc: + { + SLANG_RETURN_ON_FAIL(_readInstArrayChunk(containerCompressionType, dataChunk, outData->m_insts)); + break; + } + case SLANG_MAKE_COMPRESSED_FOUR_CC(Bin::kChildRunFourCc): + case Bin::kChildRunFourCc: + { + SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayChunk(containerCompressionType, dataChunk, outData->m_childRuns)); + break; + } + case SLANG_MAKE_COMPRESSED_FOUR_CC(Bin::kExternalOperandsFourCc): + case Bin::kExternalOperandsFourCc: + { + SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayChunk(containerCompressionType, dataChunk, outData->m_externalOperands)); + break; + } + case SerialBinary::kStringTableFourCc: + { + SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayUncompressedChunk(dataChunk, outData->m_stringTable)); + break; + } + case Bin::kUInt32RawSourceLocFourCc: + { + SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayUncompressedChunk(dataChunk, outData->m_rawSourceLocs)); + break; + } + case SLANG_MAKE_COMPRESSED_FOUR_CC(Bin::kDebugSourceLocRunFourCc): + case Bin::kDebugSourceLocRunFourCc: + { + SLANG_RETURN_ON_FAIL(SerialRiffUtil::readArrayChunk(containerCompressionType, dataChunk, outData->m_debugSourceLocRuns)); + break; + } + default: + { + break; + } + } + } + + return SLANG_OK; +} + +Result IRSerialReader::read(const IRSerialData& data, Session* session, DebugSerialReader* debugReader, RefPtr<IRModule>& outModule) +{ + typedef Ser::Inst::PayloadType PayloadType; + + m_serialData = &data; + + auto module = new IRModule(); + outModule = module; + m_module = module; + + module->session = session; + + // Convert m_stringTable into StringSlicePool. + SerialStringTableUtil::decodeStringTable(data.m_stringTable.getBuffer(), data.m_stringTable.getCount(), m_stringTable); + + // Add all the instructions + List<IRInst*> insts; + + const Index numInsts = data.m_insts.getCount(); + + SLANG_ASSERT(numInsts > 0); + + insts.setCount(numInsts); + insts[0] = nullptr; + + // 0 holds null + // 1 holds the IRModuleInst + { + // Check that insts[1] is the module inst + const Ser::Inst& srcInst = data.m_insts[1]; + SLANG_RELEASE_ASSERT(srcInst.m_op == kIROp_Module); + SLANG_ASSERT(srcInst.m_payloadType == PayloadType::Empty); + + // Create the module inst + auto moduleInst = static_cast<IRModuleInst*>(createEmptyInstWithSize(module, kIROp_Module, sizeof(IRModuleInst))); + module->moduleInst = moduleInst; + moduleInst->module = module; + + // Set the IRModuleInst + insts[1] = moduleInst; + } + + for (Index i = 2; i < numInsts; ++i) + { + const Ser::Inst& srcInst = data.m_insts[i]; + + const IROp op((IROp)srcInst.m_op); + + if (_isConstant(op)) + { + // Handling of constants + + // Calculate the minimum object size (ie not including the payload of value) + const size_t prefixSize = SLANG_OFFSET_OF(IRConstant, value); + + IRConstant* irConst = nullptr; + switch (op) + { + case kIROp_BoolLit: + { + SLANG_ASSERT(srcInst.m_payloadType == PayloadType::UInt32); + irConst = static_cast<IRConstant*>(createEmptyInstWithSize(module, op, prefixSize + sizeof(IRIntegerValue))); + irConst->value.intVal = srcInst.m_payload.m_uint32 != 0; + break; + } + case kIROp_IntLit: + { + SLANG_ASSERT(srcInst.m_payloadType == PayloadType::Int64); + irConst = static_cast<IRConstant*>(createEmptyInstWithSize(module, op, prefixSize + sizeof(IRIntegerValue))); + irConst->value.intVal = srcInst.m_payload.m_int64; + break; + } + case kIROp_PtrLit: + { + SLANG_ASSERT(srcInst.m_payloadType == PayloadType::Int64); + irConst = static_cast<IRConstant*>(createEmptyInstWithSize(module, op, prefixSize + sizeof(void*))); + irConst->value.ptrVal = (void*) (intptr_t) srcInst.m_payload.m_int64; + break; + } + case kIROp_FloatLit: + { + SLANG_ASSERT(srcInst.m_payloadType == PayloadType::Float64); + irConst = static_cast<IRConstant*>(createEmptyInstWithSize(module, op, prefixSize + sizeof(IRFloatingPointValue))); + irConst->value.floatVal = srcInst.m_payload.m_float64; + break; + } + case kIROp_StringLit: + { + SLANG_ASSERT(srcInst.m_payloadType == PayloadType::String_1); + + const UnownedStringSlice slice = m_stringTable.getSlice(StringSlicePool::Handle(srcInst.m_payload.m_stringIndices[0])); + + const size_t sliceSize = slice.getLength(); + const size_t instSize = prefixSize + SLANG_OFFSET_OF(IRConstant::StringValue, chars) + sliceSize; + + irConst = static_cast<IRConstant*>(createEmptyInstWithSize(module, op, instSize)); + + IRConstant::StringValue& dstString = irConst->value.stringVal; + + dstString.numChars = uint32_t(sliceSize); + // Turn into pointer to avoid warning of array overrun + char* dstChars = dstString.chars; + // Copy the chars + memcpy(dstChars, slice.begin(), sliceSize); + break; + } + default: + { + SLANG_ASSERT(!"Unknown constant type"); + return SLANG_FAIL; + } + } + + insts[i] = irConst; + } + else if (_isTextureTypeBase(op)) + { + IRTextureTypeBase* inst = static_cast<IRTextureTypeBase*>(createEmptyInst(module, op, 1)); + SLANG_ASSERT(srcInst.m_payloadType == PayloadType::OperandAndUInt32); + + // Reintroduce the texture type bits into the the + const uint32_t other = srcInst.m_payload.m_operandAndUInt32.m_uint32; + inst->op = IROp(uint32_t(inst->op) | (other << kIROpMeta_OtherShift)); + + insts[i] = inst; + } + else + { + int numOperands = srcInst.getNumOperands(); + insts[i] = createEmptyInst(module, op, numOperands); + } + } + + // Patch up the operands + for (Index i = 1; i < numInsts; ++i) + { + const Ser::Inst& srcInst = data.m_insts[i]; + const IROp op((IROp)srcInst.m_op); + + IRInst* dstInst = insts[i]; + + // Set the result type + if (srcInst.m_resultTypeIndex != Ser::InstIndex(0)) + { + IRInst* resultInst = insts[int(srcInst.m_resultTypeIndex)]; + // NOTE! Counter intuitively the IRType* paramter may not be IRType* derived for example + // IRGlobalGenericParam is valid, but isn't IRType* derived + + //SLANG_RELEASE_ASSERT(as<IRType>(resultInst)); + dstInst->setFullType(static_cast<IRType*>(resultInst)); + } + + //if (!isParentDerived(op)) + { + const Ser::InstIndex* srcOperandIndices; + const int numOperands = data.getOperands(srcInst, &srcOperandIndices); + + auto dstOperands = dstInst->getOperands(); + + for (int j = 0; j < numOperands; j++) + { + dstOperands[j].init(dstInst, insts[int(srcOperandIndices[j])]); + } + } + } + + // Patch up the children + { + const Index numChildRuns = data.m_childRuns.getCount(); + for (Index i = 0; i < numChildRuns; i++) + { + const auto& run = data.m_childRuns[i]; + + IRInst* inst = insts[int(run.m_parentIndex)]; + + for (int j = 0; j < int(run.m_numChildren); ++j) + { + IRInst* child = insts[j + int(run.m_startInstIndex)]; + SLANG_ASSERT(child->parent == nullptr); + child->insertAtEnd(inst); + } + } + } + + // Re-add source locations, if they are defined + if (m_serialData->m_rawSourceLocs.getCount() == numInsts) + { + const Ser::RawSourceLoc* srcLocs = m_serialData->m_rawSourceLocs.begin(); + for (Index i = 1; i < numInsts; ++i) + { + IRInst* dstInst = insts[i]; + + dstInst->sourceLoc.setRaw(Slang::SourceLoc::RawValue(srcLocs[i])); + } + } + + // We now need to apply the runs + if (debugReader && m_serialData->m_debugSourceLocRuns.getCount()) + { + List<IRSerialData::SourceLocRun> sourceRuns(m_serialData->m_debugSourceLocRuns); + // They are now in source location order + sourceRuns.sort(); + + // Just guess initially 0 for the source file that contains the initial run + DebugSerialData::SourceRange range = DebugSerialData::SourceRange::getInvalid(); + int fix = 0; + + const Index numRuns = sourceRuns.getCount(); + for (Index i = 0; i < numRuns; ++i) + { + const auto& run = sourceRuns[i]; + + // Work out the fixed source location + SourceLoc sourceLoc; + if (run.m_sourceLoc) + { + if (!range.contains(run.m_sourceLoc)) + { + fix = debugReader->calcFixSourceLoc(run.m_sourceLoc, range); + } + sourceLoc = debugReader->calcFixedLoc(run.m_sourceLoc, fix, range); + } + + // Write to all the instructions + SLANG_ASSERT(Index(uint32_t(run.m_startInstIndex) + run.m_numInst) <= insts.getCount()); + IRInst** dstInsts = insts.getBuffer() + int(run.m_startInstIndex); + + const int runSize = int(run.m_numInst); + for (int j = 0; j < runSize; ++j) + { + dstInsts[j]->sourceLoc = sourceLoc; + } + } + } + + return SLANG_OK; +} + +} // namespace Slang diff --git a/source/slang/slang-serialize-ir.h b/source/slang/slang-serialize-ir.h new file mode 100644 index 000000000..368930c24 --- /dev/null +++ b/source/slang/slang-serialize-ir.h @@ -0,0 +1,97 @@ +// slang-serialize-ir.h +#ifndef SLANG_SERIALIZE_IR_H_INCLUDED +#define SLANG_SERIALIZE_IR_H_INCLUDED + +#include "slang-serialize-ir-types.h" + +#include "../core/slang-riff.h" + +#include "slang-ir.h" +#include "slang-serialize-debug.h" + +// For TranslationUnitRequest +// and FrontEndCompileRequest::ExtraEntryPointInfo +#include "slang-compiler.h" + +namespace Slang { + +struct IRSerialWriter +{ + typedef IRSerialData Ser; + typedef IRSerialBinary Bin; + + Result write(IRModule* module, DebugSerialWriter* debugWriter, SerialOptionFlags flags, IRSerialData* serialData); + + /// Write to a container + static Result writeContainer(const IRSerialData& data, SerialCompressionType compressionType, RiffContainer* container); + + /// Get an instruction index from an instruction + Ser::InstIndex getInstIndex(IRInst* inst) const { return inst ? Ser::InstIndex(m_instMap[inst]) : Ser::InstIndex(0); } + + /// Get a slice from an index + UnownedStringSlice getStringSlice(Ser::StringIndex index) const { return m_stringSlicePool.getSlice(StringSlicePool::Handle(index)); } + /// Get index from string representations + Ser::StringIndex getStringIndex(StringRepresentation* string) { return Ser::StringIndex(m_stringSlicePool.add(string)); } + Ser::StringIndex getStringIndex(const UnownedStringSlice& slice) { return Ser::StringIndex(m_stringSlicePool.add(slice)); } + Ser::StringIndex getStringIndex(Name* name) { return name ? getStringIndex(name->text) : SerialStringData::kNullStringIndex; } + Ser::StringIndex getStringIndex(const char* chars) { return Ser::StringIndex(m_stringSlicePool.add(chars)); } + Ser::StringIndex getStringIndex(const String& string) { return Ser::StringIndex(m_stringSlicePool.add(string.getUnownedSlice())); } + + StringSlicePool& getStringPool() { return m_stringSlicePool; } + + IRSerialWriter() : + m_serialData(nullptr), + m_stringSlicePool(StringSlicePool::Style::Default) + { + } + + /// Produces an instruction list which is in same order as written through IRSerialWriter + static void calcInstructionList(IRModule* module, List<IRInst*>& instsOut); + +protected: + + void _addInstruction(IRInst* inst); + Result _calcDebugInfo(DebugSerialWriter* debugWriter); + + List<IRInst*> m_insts; ///< Instructions in same order as stored in the + + List<IRDecoration*> m_decorations; ///< Holds all decorations in order of the instructions as found + List<IRInst*> m_instWithFirstDecoration; ///< All decorations are held in this order after all the regular instructions + + Dictionary<IRInst*, Ser::InstIndex> m_instMap; ///< Map an instruction to an instruction index + + StringSlicePool m_stringSlicePool; + IRSerialData* m_serialData; ///< Where the data is stored +}; + +struct IRSerialReader +{ + typedef IRSerialData Ser; + + /// Read potentially multiple modules from a stream + static Result readStreamModules(Stream* stream, Session* session, SourceManager* manager, List<RefPtr<IRModule>>& outModules, List<FrontEndCompileRequest::ExtraEntryPointInfo>& outEntryPoints); + + /// Read a stream to fill in dataOut IRSerialData + static Result readContainer(RiffContainer::ListChunk* module, SerialCompressionType containerCompressionType, IRSerialData* outData); + + /// Read a module from serial data + Result read(const IRSerialData& data, Session* session, DebugSerialReader* debugReader, RefPtr<IRModule>& outModule); + + IRSerialReader(): + m_serialData(nullptr), + m_module(nullptr), + m_stringTable(StringSlicePool::Style::Default) + { + } + + protected: + + StringSlicePool m_stringTable; + + const IRSerialData* m_serialData; + IRModule* m_module; +}; + +} // namespace Slang + +#endif diff --git a/source/slang/slang-serialize-types.cpp b/source/slang/slang-serialize-types.cpp new file mode 100644 index 000000000..fc839afc1 --- /dev/null +++ b/source/slang/slang-serialize-types.cpp @@ -0,0 +1,249 @@ +// slang-serialize-types.cpp +#include "slang-serialize-types.h" + +#include "../core/slang-text-io.h" +#include "../core/slang-byte-encode-util.h" + +#include "../core/slang-math.h" + +namespace Slang { + +// Needed for linkage with some compilers +/* static */ const SerialStringData::StringIndex SerialStringData::kNullStringIndex; +/* static */ const SerialStringData::StringIndex SerialStringData::kEmptyStringIndex; + +namespace { // anonymous + +struct CharReader +{ + char operator()(int pos) const { SLANG_UNUSED(pos); return *m_pos++; } + CharReader(const char* pos) :m_pos(pos) {} + mutable const char* m_pos; +}; + +} // anonymous + + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! SerialStringTableUtil !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +/* static */void SerialStringTableUtil::encodeStringTable(const StringSlicePool& pool, List<char>& stringTable) +{ + // Skip the default handles -> nothing is encoded via them + return encodeStringTable(pool.getAdded(), stringTable); +} + +/* static */void SerialStringTableUtil::encodeStringTable(const ConstArrayView<UnownedStringSlice>& slices, List<char>& stringTable) +{ + stringTable.clear(); + for (const auto& slice : slices) + { + const int len = int(slice.getLength()); + + // We need to write into the the string array + char prefixBytes[6]; + const int numPrefixBytes = EncodeUnicodePointToUTF8(prefixBytes, len); + const Index baseIndex = stringTable.getCount(); + + stringTable.setCount(baseIndex + numPrefixBytes + len); + + char* dst = stringTable.begin() + baseIndex; + + memcpy(dst, prefixBytes, numPrefixBytes); + memcpy(dst + numPrefixBytes, slice.begin(), len); + } +} + +/* static */void SerialStringTableUtil::appendDecodedStringTable(const char* table, size_t tableSize, List<UnownedStringSlice>& slicesOut) +{ + const char* start = table; + const char* cur = start; + const char* end = table + tableSize; + + while (cur < end) + { + CharReader reader(cur); + const int len = GetUnicodePointFromUTF8(reader); + slicesOut.add(UnownedStringSlice(reader.m_pos, len)); + cur = reader.m_pos + len; + } +} + +/* static */void SerialStringTableUtil::decodeStringTable(const char* table, size_t tableSize, List<UnownedStringSlice>& slicesOut) +{ + slicesOut.setCount(2); + slicesOut[0] = UnownedStringSlice(nullptr, size_t(0)); + slicesOut[1] = UnownedStringSlice("", size_t(0)); + + appendDecodedStringTable(table, tableSize, slicesOut); +} + +/* static */void SerialStringTableUtil::decodeStringTable(const char* table, size_t tableSize, StringSlicePool& outPool) +{ + outPool.clear(); + + const char* start = table; + const char* cur = start; + const char* end = table + tableSize; + + while (cur < end) + { + CharReader reader(cur); + const int len = GetUnicodePointFromUTF8(reader); + outPool.add(UnownedStringSlice(reader.m_pos, len)); + cur = reader.m_pos + len; + } +} + +/* static */void SerialStringTableUtil::calcStringSlicePoolMap(const List<UnownedStringSlice>& slices, StringSlicePool& pool, List<StringSlicePool::Handle>& indexMapOut) +{ + SLANG_ASSERT(slices.getCount() >= StringSlicePool::kDefaultHandlesCount); + SLANG_ASSERT(slices[int(StringSlicePool::kNullHandle)] == "" && slices[int(StringSlicePool::kNullHandle)].begin() == nullptr); + SLANG_ASSERT(slices[int(StringSlicePool::kEmptyHandle)] == ""); + + indexMapOut.setCount(slices.getCount()); + // Set up all of the defaults + for (int i = 0; i < StringSlicePool::kDefaultHandlesCount; ++i) + { + indexMapOut[i] = StringSlicePool::Handle(i); + } + + const Index numSlices = slices.getCount(); + for (Index i = StringSlicePool::kDefaultHandlesCount; i < numSlices ; ++i) + { + indexMapOut[i] = pool.add(slices[i]); + } +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!! SerialRiffUtil !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +/* static */ Result SerialRiffUtil::writeArrayChunk(SerialCompressionType compressionType, FourCC chunkId, const void* data, size_t numEntries, size_t typeSize, RiffContainer* container) +{ + typedef RiffContainer::Chunk Chunk; + typedef RiffContainer::ScopeChunk ScopeChunk; + + if (numEntries == 0) + { + return SLANG_OK; + } + + // Make compressed fourCC + chunkId = (compressionType != SerialCompressionType::None) ? SLANG_MAKE_COMPRESSED_FOUR_CC(chunkId) : chunkId; + + ScopeChunk scope(container, Chunk::Kind::Data, chunkId); + + switch (compressionType) + { + case SerialCompressionType::None: + { + SerialBinary::ArrayHeader header; + header.numEntries = uint32_t(numEntries); + + container->write(&header, sizeof(header)); + container->write(data, typeSize * numEntries); + break; + } + case SerialCompressionType::VariableByteLite: + { + List<uint8_t> compressedPayload; + + size_t numCompressedEntries = (numEntries * typeSize) / sizeof(uint32_t); + ByteEncodeUtil::encodeLiteUInt32((const uint32_t*)data, numCompressedEntries, compressedPayload); + + SerialBinary::CompressedArrayHeader header; + header.numEntries = uint32_t(numEntries); + header.numCompressedEntries = uint32_t(numCompressedEntries); + + container->write(&header, sizeof(header)); + container->write(compressedPayload.getBuffer(), compressedPayload.getCount()); + break; + } + default: + { + return SLANG_FAIL; + } + } + return SLANG_OK; +} + +/* static */Result SerialRiffUtil::readArrayChunk(SerialCompressionType compressionType, RiffContainer::DataChunk* dataChunk, ListResizer& listOut) +{ + typedef SerialBinary Bin; + + RiffReadHelper read = dataChunk->asReadHelper(); + const size_t typeSize = listOut.getTypeSize(); + + switch (compressionType) + { + case SerialCompressionType::VariableByteLite: + { + Bin::CompressedArrayHeader header; + SLANG_RETURN_ON_FAIL(read.read(header)); + + void* dst = listOut.setSize(header.numEntries); + SLANG_ASSERT(header.numCompressedEntries == uint32_t((header.numEntries * typeSize) / sizeof(uint32_t))); + + // Decode.. + ByteEncodeUtil::decodeLiteUInt32(read.getData(), header.numCompressedEntries, (uint32_t*)dst); + break; + } + case SerialCompressionType::None: + { + // Read uncompressed + Bin::ArrayHeader header; + SLANG_RETURN_ON_FAIL(read.read(header)); + const size_t payloadSize = header.numEntries * typeSize; + SLANG_ASSERT(payloadSize == read.getRemainingSize()); + void* dst = listOut.setSize(header.numEntries); + ::memcpy(dst, read.getData(), payloadSize); + break; + } + } + return SLANG_OK; +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!! SerialParseUtil !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +#define SLANG_SERIAL_BINARY_COMPRESSION_TYPE(x) \ + x(None, none) \ + x(VariableByteLite, lite) + +/* static */SlangResult SerialParseUtil::parseCompressionType(const UnownedStringSlice& text, SerialCompressionType& outType) +{ + struct Pair + { + UnownedStringSlice name; + SerialCompressionType type; + }; + +#define SLANG_SERIAL_BINARY_PAIR(type, name) { UnownedStringSlice::fromLiteral(#name), SerialCompressionType::type}, + + static const Pair s_pairs[] = + { + SLANG_SERIAL_BINARY_COMPRESSION_TYPE(SLANG_SERIAL_BINARY_PAIR) + }; + + for (const auto& pair : s_pairs) + { + if (pair.name == text) + { + outType = pair.type; + return SLANG_OK; + } + } + return SLANG_FAIL; +} + +/* static */UnownedStringSlice SerialParseUtil::getText(SerialCompressionType type) +{ +#define SLANG_SERIAL_BINARY_CASE(type, name) case SerialCompressionType::type: return UnownedStringSlice::fromLiteral(#name); + switch (type) + { + SLANG_SERIAL_BINARY_COMPRESSION_TYPE(SLANG_SERIAL_BINARY_CASE) + default: break; + } + SLANG_ASSERT(!"Unknown compression type"); + return UnownedStringSlice::fromLiteral("unknown"); +} + + +} // namespace Slang diff --git a/source/slang/slang-serialize-types.h b/source/slang/slang-serialize-types.h new file mode 100644 index 000000000..f799dd607 --- /dev/null +++ b/source/slang/slang-serialize-types.h @@ -0,0 +1,230 @@ +// slang-serialize-types.h +#ifndef SLANG_SERIALIZE_TYPES_H +#define SLANG_SERIALIZE_TYPES_H + +#include "../core/slang-riff.h" +#include "../core/slang-string-slice-pool.h" +#include "../core/slang-array-view.h" + +#include "slang-name.h" +#include "slang-source-loc.h" + +namespace Slang { + +// Options for IR/AST/Debug serialization + +struct SerialOptionFlag +{ + typedef uint32_t Type; + enum Enum : Type + { + RawSourceLocation = 0x01, + DebugInfo = 0x02, + }; +}; +typedef SerialOptionFlag::Type SerialOptionFlags; + + +// Compression styles + +enum class SerialCompressionType : uint8_t +{ + None, + VariableByteLite, +}; + + +struct SerialStringData +{ + enum class StringIndex : uint32_t; + + ///enum class StringOffset : uint32_t; ///< Offset into the m_stringsBuffer + + typedef uint32_t SizeType; + + static const StringIndex kNullStringIndex = StringIndex(StringSlicePool::kNullHandle); + static const StringIndex kEmptyStringIndex = StringIndex(StringSlicePool::kEmptyHandle); +}; + +struct SerialStringTableUtil +{ + /// Convert a pool into a string table + static void encodeStringTable(const StringSlicePool& pool, List<char>& stringTable); + static void encodeStringTable(const ConstArrayView<UnownedStringSlice>& slices, List<char>& stringTable); + + /// Appends the decoded strings into slicesOut + static void appendDecodedStringTable(const char* table, size_t tableSize, List<UnownedStringSlice>& slicesOut); + + /// Decodes a string table (and does so such that the indices are compatible with StringSlicePool) + static void decodeStringTable(const char* table, size_t tableSize, List<UnownedStringSlice>& slicesOut); + + /// Decodes a string table + static void decodeStringTable(const char* table, size_t tableSize, StringSlicePool& outPool); + + /// Produces an index map, from slices to indices in pool + static void calcStringSlicePoolMap(const List<UnownedStringSlice>& slices, StringSlicePool& pool, List<StringSlicePool::Handle>& indexMap); +}; + +struct SerialParseUtil +{ + /// Given text, finds the compression type + static SlangResult parseCompressionType(const UnownedStringSlice& text, SerialCompressionType& outType); + /// Given a compression type, return text + static UnownedStringSlice getText(SerialCompressionType type); +}; + +struct SerialListUtil +{ + template <typename T> + static size_t calcArraySize(const List<T>& list) + { + return list.getCount() * sizeof(T); + } + + template <typename T> + static bool isEqual(const List<T>& aIn, const List<T>& bIn) + { + if (&aIn == &bIn) + { + return true; + } + const Index size = aIn.getCount(); + + if (size != bIn.getCount()) + { + return false; + } + + const T* a = aIn.begin(); + const T* b = bIn.begin(); + + if (a != b) + { + for (Index i = 0; i < size; ++i) + { + if (a[i] != b[i]) + { + return false; + } + } + } + + return true; + } +}; + +// For types/FourCC that work for serializing in general (not just IR). +struct SerialBinary +{ + static const FourCC kRiffFourCc = RiffFourCC::kRiff; + + /// Container + static const FourCC kContainerFourCc = SLANG_FOUR_CC('S', 'L', 'm', 'c'); + + /// A string table + static const FourCC kStringTableFourCc = SLANG_FOUR_CC('S', 'L', 's', 't'); + + /// TranslationUnitList + static const FourCC kTranslationUnitListFourCc = SLANG_FOUR_CC('S', 'L', 'm', 'l'); + + /// An entry point + static const FourCC kEntryPointFourCc = SLANG_FOUR_CC('E', 'P', 'n', 't'); + + /// Container + static const FourCC kContainerHeaderFourCc = SLANG_FOUR_CC('S', 'c', 'h', 'd'); + + struct ContainerHeader + { + uint32_t compressionType; ///< Holds the compression type used (if used at all) + }; + + struct ArrayHeader + { + uint32_t numEntries; + }; + struct CompressedArrayHeader + { + uint32_t numEntries; ///< The number of entries + uint32_t numCompressedEntries; ///< The amount of compressed entries + }; +}; + +// Replace first char with 's' +#define SLANG_MAKE_COMPRESSED_FOUR_CC(fourCc) SLANG_FOUR_CC_REPLACE_FIRST_CHAR(fourCc, 's') + +struct SerialRiffUtil +{ + class ListResizer + { + public: + virtual void* setSize(size_t newSize) = 0; + SLANG_FORCE_INLINE size_t getTypeSize() const { return m_typeSize; } + ListResizer(size_t typeSize) :m_typeSize(typeSize) {} + + protected: + size_t m_typeSize; + }; + + template <typename T> + class ListResizerForType : public ListResizer + { + public: + typedef ListResizer Parent; + + SLANG_FORCE_INLINE ListResizerForType(List<T>& list) : + Parent(sizeof(T)), + m_list(list) + {} + + virtual void* setSize(size_t newSize) SLANG_OVERRIDE + { + m_list.setCount(UInt(newSize)); + return (void*)m_list.begin(); + } + + protected: + List<T>& m_list; + }; + + static Result writeArrayChunk(SerialCompressionType compressionType, FourCC chunkId, const void* data, size_t numEntries, size_t typeSize, RiffContainer* container); + + template <typename T> + static Result writeArrayChunk(SerialCompressionType compressionType, FourCC chunkId, const List<T>& array, RiffContainer* container) + { + return writeArrayChunk(compressionType, chunkId, array.begin(), size_t(array.getCount()), sizeof(T), container); + } + + template <typename T> + static Result writeArrayUncompressedChunk(FourCC chunkId, const List<T>& array, RiffContainer* container) + { + return writeArrayChunk(SerialCompressionType::None, chunkId, array.begin(), size_t(array.getCount()), sizeof(T), container); + } + + static Result readArrayChunk(SerialCompressionType compressionType, RiffContainer::DataChunk* dataChunk, ListResizer& listOut); + + template <typename T> + static Result readArrayChunk(SerialCompressionType moduleCompressionType, RiffContainer::DataChunk* dataChunk, List<T>& arrayOut) + { + SerialCompressionType compressionType = SerialCompressionType::None; + if (dataChunk->m_fourCC == SLANG_MAKE_COMPRESSED_FOUR_CC(dataChunk->m_fourCC)) + { + // If it has compression, use the compression type set in the header + compressionType = moduleCompressionType; + } + ListResizerForType<T> resizer(arrayOut); + return readArrayChunk(compressionType, dataChunk, resizer); + } + + template <typename T> + static Result readArrayUncompressedChunk(RiffContainer::DataChunk* chunk, List<T>& arrayOut) + { + ListResizerForType<T> resizer(arrayOut); + return readArrayChunk(SerialCompressionType::None, chunk, resizer); + } + + +}; + +} // namespace Slang + +#endif diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp index 4c83095c2..c9dccbdbf 100644 --- a/source/slang/slang.cpp +++ b/source/slang/slang.cpp @@ -14,9 +14,6 @@ #include "slang-reflection.h" #include "slang-type-layout.h" -#include "slang-ast-dump.h" -#include "slang-ast-serialize.h" - #include "slang-repro.h" #include "slang-file-system.h" @@ -25,7 +22,11 @@ #include "slang-source-loc.h" -#include "slang-ir-serialize.h" +#include "slang-ast-dump.h" + +#include "slang-serialize-ast.h" +#include "slang-serialize-ir.h" +#include "slang-serialize-container.h" #include "slang-check-impl.h" @@ -1074,20 +1075,25 @@ void FrontEndCompileRequest::generateIR() if (verifyDebugSerialization) { + SerialContainerUtil::WriteOptions options; + options.compressionType = SerialCompressionType::None; + options.sourceManager = getSourceManager(); + options.optionFlags = SerialOptionFlag::DebugInfo; + // Verify debug information - if (SLANG_FAILED(IRSerialUtil::verifySerialize(irModule, getSession(), getSourceManager(), IRSerialBinary::CompressionType::None, IRSerialWriter::OptionFlag::DebugInfo))) + if (SLANG_FAILED(SerialContainerUtil::verifyIRSerialize(irModule, getSession(), options))) { getSink()->diagnose(irModule->moduleInst->sourceLoc, Diagnostics::serialDebugVerificationFailed); } } if (useSerialIRBottleneck) - { + { IRSerialData serialData; { // Write IR out to serialData - copying over SourceLoc information directly IRSerialWriter writer; - writer.write(irModule, getSourceManager(), IRSerialWriter::OptionFlag::RawSourceLocation, &serialData); + writer.write(irModule, nullptr, SerialOptionFlag::RawSourceLocation, &serialData); // Destroy irModule such that memory can be used for newly constructed read irReadModule irModule = nullptr; @@ -3013,33 +3019,47 @@ namespace Slang SlangResult _addLibraryReference(EndToEndCompileRequest* req, Stream* stream) { // Load up the module - RiffContainer container; - SLANG_RETURN_ON_FAIL(RiffUtil::read(stream, container)); + RiffContainer riffContainer; + SLANG_RETURN_ON_FAIL(RiffUtil::read(stream, riffContainer)); + + auto linkage = req->getLinkage(); - List<RefPtr<Module>> modules; + // TODO(JS): May be better to have a ITypeComponent that encapsulates a collection of modules + // For now just add to the linkage - if (SLANG_FAILED(ASTSerialReader::readContainerModules(&container, req->getLinkage(), modules))) { - req->getSink()->diagnose(SourceLoc(), Diagnostics::unableToAddReferenceToModuleContainer); - return SLANG_FAIL; - } + SerialContainerData containerData; - // Read all of the contained modules - List<RefPtr<IRModule>> irModules; - List<FrontEndCompileRequest::ExtraEntryPointInfo> entryPointMangledNames; + SerialContainerUtil::ReadOptions options; + options.namePool = req->getNamePool(); + options.session = req->getSession(); + options.sharedASTBuilder = linkage->getASTBuilder()->getSharedASTBuilder(); + options.sourceManager = linkage->getSourceManager(); - if (SLANG_FAILED(IRSerialReader::readContainerModules(&container, req->getSession(), req->getFrontEndReq()->getSourceManager(), irModules, entryPointMangledNames))) - { - req->getSink()->diagnose(SourceLoc(), Diagnostics::unableToAddReferenceToModuleContainer); - return SLANG_FAIL; - } + SLANG_RETURN_ON_FAIL(SerialContainerUtil::read(&riffContainer, options, containerData)); - // TODO(JS): May be better to have a ITypeComponent that encapsulates a collection of modules - // For now just add to the linkage - auto linkage = req->getLinkage(); - linkage->m_libModules.addRange(irModules); + for (const auto& translationUnit : containerData.translationUnits) + { + // If the irModule is set, add it + if (translationUnit.irModule) + { + linkage->m_libModules.add(translationUnit.irModule); + } + } + + FrontEndCompileRequest* frontEndRequest = req->getFrontEndReq(); + + for (const auto& entryPoint : containerData.entryPoints) + { + FrontEndCompileRequest::ExtraEntryPointInfo dst; + dst.mangledName = entryPoint.mangledName; + dst.name = entryPoint.name; + dst.profile = entryPoint.profile; - req->getFrontEndReq()->m_extraEntryPoints.addRange(entryPointMangledNames); + // Add entry point + frontEndRequest->m_extraEntryPoints.add(dst); + } + } return SLANG_OK; } diff --git a/source/slang/slang.vcxproj b/source/slang/slang.vcxproj index 696873756..a0865d16c 100644 --- a/source/slang/slang.vcxproj +++ b/source/slang/slang.vcxproj @@ -199,7 +199,6 @@ <ClInclude Include="slang-ast-generated.h" /> <ClInclude Include="slang-ast-modifier.h" /> <ClInclude Include="slang-ast-reflect.h" /> - <ClInclude Include="slang-ast-serialize.h" /> <ClInclude Include="slang-ast-stmt.h" /> <ClInclude Include="slang-ast-support-types.h" /> <ClInclude Include="slang-ast-type.h" /> @@ -254,8 +253,6 @@ <ClInclude Include="slang-ir-restructure-scoping.h" /> <ClInclude Include="slang-ir-restructure.h" /> <ClInclude Include="slang-ir-sccp.h" /> - <ClInclude Include="slang-ir-serialize-types.h" /> - <ClInclude Include="slang-ir-serialize.h" /> <ClInclude Include="slang-ir-specialize-arrays.h" /> <ClInclude Include="slang-ir-specialize-function-call.h" /> <ClInclude Include="slang-ir-specialize-resources.h" /> @@ -286,6 +283,12 @@ <ClInclude Include="slang-profile.h" /> <ClInclude Include="slang-reflection.h" /> <ClInclude Include="slang-repro.h" /> + <ClInclude Include="slang-serialize-ast.h" /> + <ClInclude Include="slang-serialize-container.h" /> + <ClInclude Include="slang-serialize-debug.h" /> + <ClInclude Include="slang-serialize-ir-types.h" /> + <ClInclude Include="slang-serialize-ir.h" /> + <ClInclude Include="slang-serialize-types.h" /> <ClInclude Include="slang-source-loc.h" /> <ClInclude Include="slang-syntax.h" /> <ClInclude Include="slang-token-defs.h" /> @@ -303,7 +306,6 @@ <ClCompile Include="slang-ast-decl.cpp" /> <ClCompile Include="slang-ast-dump.cpp" /> <ClCompile Include="slang-ast-reflect.cpp" /> - <ClCompile Include="slang-ast-serialize.cpp" /> <ClCompile Include="slang-ast-substitutions.cpp" /> <ClCompile Include="slang-ast-type.cpp" /> <ClCompile Include="slang-ast-val.cpp" /> @@ -366,8 +368,6 @@ <ClCompile Include="slang-ir-restructure-scoping.cpp" /> <ClCompile Include="slang-ir-restructure.cpp" /> <ClCompile Include="slang-ir-sccp.cpp" /> - <ClCompile Include="slang-ir-serialize-types.cpp" /> - <ClCompile Include="slang-ir-serialize.cpp" /> <ClCompile Include="slang-ir-specialize-arrays.cpp" /> <ClCompile Include="slang-ir-specialize-function-call.cpp" /> <ClCompile Include="slang-ir-specialize-resources.cpp" /> @@ -397,6 +397,12 @@ <ClCompile Include="slang-profile.cpp" /> <ClCompile Include="slang-reflection.cpp" /> <ClCompile Include="slang-repro.cpp" /> + <ClCompile Include="slang-serialize-ast.cpp" /> + <ClCompile Include="slang-serialize-container.cpp" /> + <ClCompile Include="slang-serialize-debug.cpp" /> + <ClCompile Include="slang-serialize-ir-types.cpp" /> + <ClCompile Include="slang-serialize-ir.cpp" /> + <ClCompile Include="slang-serialize-types.cpp" /> <ClCompile Include="slang-source-loc.cpp" /> <ClCompile Include="slang-stdlib.cpp" /> <ClCompile Include="slang-syntax.cpp" /> diff --git a/source/slang/slang.vcxproj.filters b/source/slang/slang.vcxproj.filters index 3fd006f77..5edd4d31e 100644 --- a/source/slang/slang.vcxproj.filters +++ b/source/slang/slang.vcxproj.filters @@ -48,9 +48,6 @@ <ClInclude Include="slang-ast-reflect.h"> <Filter>Header Files</Filter> </ClInclude> - <ClInclude Include="slang-ast-serialize.h"> - <Filter>Header Files</Filter> - </ClInclude> <ClInclude Include="slang-ast-stmt.h"> <Filter>Header Files</Filter> </ClInclude> @@ -213,12 +210,6 @@ <ClInclude Include="slang-ir-sccp.h"> <Filter>Header Files</Filter> </ClInclude> - <ClInclude Include="slang-ir-serialize-types.h"> - <Filter>Header Files</Filter> - </ClInclude> - <ClInclude Include="slang-ir-serialize.h"> - <Filter>Header Files</Filter> - </ClInclude> <ClInclude Include="slang-ir-specialize-arrays.h"> <Filter>Header Files</Filter> </ClInclude> @@ -309,6 +300,24 @@ <ClInclude Include="slang-repro.h"> <Filter>Header Files</Filter> </ClInclude> + <ClInclude Include="slang-serialize-ast.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="slang-serialize-container.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="slang-serialize-debug.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="slang-serialize-ir-types.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="slang-serialize-ir.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="slang-serialize-types.h"> + <Filter>Header Files</Filter> + </ClInclude> <ClInclude Include="slang-source-loc.h"> <Filter>Header Files</Filter> </ClInclude> @@ -356,9 +365,6 @@ <ClCompile Include="slang-ast-reflect.cpp"> <Filter>Source Files</Filter> </ClCompile> - <ClCompile Include="slang-ast-serialize.cpp"> - <Filter>Source Files</Filter> - </ClCompile> <ClCompile Include="slang-ast-substitutions.cpp"> <Filter>Source Files</Filter> </ClCompile> @@ -545,12 +551,6 @@ <ClCompile Include="slang-ir-sccp.cpp"> <Filter>Source Files</Filter> </ClCompile> - <ClCompile Include="slang-ir-serialize-types.cpp"> - <Filter>Source Files</Filter> - </ClCompile> - <ClCompile Include="slang-ir-serialize.cpp"> - <Filter>Source Files</Filter> - </ClCompile> <ClCompile Include="slang-ir-specialize-arrays.cpp"> <Filter>Source Files</Filter> </ClCompile> @@ -638,6 +638,24 @@ <ClCompile Include="slang-repro.cpp"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="slang-serialize-ast.cpp"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="slang-serialize-container.cpp"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="slang-serialize-debug.cpp"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="slang-serialize-ir-types.cpp"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="slang-serialize-ir.cpp"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="slang-serialize-types.cpp"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="slang-source-loc.cpp"> <Filter>Source Files</Filter> </ClCompile> |