diff options
| author | jsmall-nvidia <jsmall@nvidia.com> | 2019-01-07 09:31:31 -0500 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2019-01-07 09:31:31 -0500 |
| commit | eb331446e3bee812d1df19cf59eb2d23d287ac74 (patch) | |
| tree | 34e2bc99746606cf53e775c423871496e9ee302d | |
| parent | d155eaa92d56a4ec00109d25c8c70fe12fb96c2e (diff) | |
Feature/serialization debug info (#767)
* Remove AppContext. Use StdChannels to hold writers, and TestToolUtil to hold test tool specific functionality.
* StdChannels -> StdWriters
* getStdOut -> getOut, getStdError -> getError
* Renamed main.cpp files of tools to try and stop visual studio getting confused between files - such that clicking on an error takes editor to the right location.
* Work in progress on being able to serialize debug information.
* * Added MemoryStream
* First pass converting to IRSerialData
* Able to read and write IRSerialData with debug data
* Start at reconstruting IR serialized data.
* First pass of generation debug SourceLocs from debug data. Works for test set for line nos.
* Bug fixes.
Moved testing of serialization into IRSerialUtil
* Work around problem with irModule = generateIRForTranslationUnit(translationUnit); two times in a row produces different output(!). Fix by just creating once.
* Remove problem with use of ternary op in slang.cpp on gcc/clang.
* Added -verify-debug-serial-ir option that makes IR modules go through full serialization with debug information and verification.
* Add a test that does serial debug verification that is run by default on linux.
| -rw-r--r-- | source/core/slang-string-slice-pool.h | 2 | ||||
| -rw-r--r-- | source/core/stream.cpp | 76 | ||||
| -rw-r--r-- | source/core/stream.h | 28 | ||||
| -rw-r--r-- | source/slang/compiler.cpp | 6 | ||||
| -rw-r--r-- | source/slang/compiler.h | 3 | ||||
| -rw-r--r-- | source/slang/diagnostic-defs.h | 1 | ||||
| -rw-r--r-- | source/slang/ir-serialize.cpp | 687 | ||||
| -rw-r--r-- | source/slang/ir-serialize.h | 198 | ||||
| -rw-r--r-- | source/slang/options.cpp | 4 | ||||
| -rw-r--r-- | source/slang/preprocessor.cpp | 10 | ||||
| -rw-r--r-- | source/slang/slang.cpp | 50 | ||||
| -rw-r--r-- | source/slang/source-loc.cpp | 74 | ||||
| -rw-r--r-- | source/slang/source-loc.h | 50 | ||||
| -rw-r--r-- | tests/cross-compile/non-uniform-indexing.slang | 1 | ||||
| -rw-r--r-- | tests/hlsl/simple/allow-uav-conditional.hlsl | 1 |
15 files changed, 1070 insertions, 121 deletions
diff --git a/source/core/slang-string-slice-pool.h b/source/core/slang-string-slice-pool.h index c9c8b8db9..08cd819ab 100644 --- a/source/core/slang-string-slice-pool.h +++ b/source/core/slang-string-slice-pool.h @@ -50,6 +50,8 @@ public: /// Convert a handle to and index. (A handle is just an index!) static int asIndex(Handle handle) { return int(handle); } + /// Returns true if the handle is to a slice that contains characters (ie not null or empty) + static bool hasContents(Handle handle) { return int(handle) >= kNumDefaultHandles; } /// Ctor StringSlicePool(); diff --git a/source/core/stream.cpp b/source/core/stream.cpp index 949ce718c..19ae3cdea 100644 --- a/source/core/stream.cpp +++ b/source/core/stream.cpp @@ -153,6 +153,7 @@ namespace Slang break; case Slang::SeekOrigin::End: _origin = SEEK_END; + // JS TODO: This doesn't seem right, the offset can mean it's not at the end endReached = true; break; case Slang::SeekOrigin::Current: @@ -215,4 +216,79 @@ namespace Slang { return endReached; } + + // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! MemoryStream !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + void MemoryStream::Seek(SeekOrigin origin, Int64 offset) + { + Int64 pos = 0; + switch (origin) + { + case Slang::SeekOrigin::Start: + pos = offset; + break; + case Slang::SeekOrigin::End: + pos = Int64(m_contents.Count()) + offset; + break; + case Slang::SeekOrigin::Current: + pos = Int64(m_position) + offset; + break; + default: + throw NotSupportedException("Unsupported seek origin."); + break; + } + + m_atEnd = false; + + // Clamp to the valid range + pos = (pos < 0) ? 0 : pos; + pos = (pos > Int64(m_contents.Count())) ? Int64(m_contents.Count()) : pos; + + m_position = UInt(pos); + } + + Int64 MemoryStream::Read(void * buffer, Int64 length) + { + if (!CanRead()) + { + throw IOException("Cannot read this stream."); + } + + const Int64 maxRead = Int64(m_contents.Count() - m_position); + + if (maxRead == 0 && length > 0) + { + m_atEnd = true; + throw EndOfStreamException("End of file is reached."); + } + + length = length > maxRead ? maxRead : length; + + ::memcpy(buffer, m_contents.begin() + m_position, size_t(length)); + m_position += UInt(length); + return maxRead; + } + + Int64 MemoryStream::Write(const void * buffer, Int64 length) + { + if (!CanWrite()) + { + throw IOException("Cannot write this stream."); + } + + if (m_position == m_contents.Count()) + { + m_contents.AddRange((const uint8_t*)buffer, UInt(length)); + } + else + { + m_contents.InsertRange(m_position, (const uint8_t*)buffer, UInt(length)); + } + + m_atEnd = false; + + m_position += UInt(length); + return length; + } + } diff --git a/source/core/stream.h b/source/core/stream.h index 4eea6a909..67a3549e9 100644 --- a/source/core/stream.h +++ b/source/core/stream.h @@ -53,7 +53,7 @@ namespace Slang enum class FileAccess { - Read = 1, Write = 2, ReadWrite = 3 + None = 0, Read = 1, Write = 2, ReadWrite = 3 }; enum class FileShare @@ -61,6 +61,32 @@ namespace Slang None, ReadOnly, WriteOnly, ReadWrite }; + class MemoryStream : public Stream + { + public: + virtual Int64 GetPosition() SLANG_OVERRIDE { return m_position; } + virtual void Seek(SeekOrigin origin, Int64 offset) SLANG_OVERRIDE; + virtual Int64 Read(void * buffer, Int64 length) SLANG_OVERRIDE; + virtual Int64 Write(const void * buffer, Int64 length) SLANG_OVERRIDE; + virtual bool IsEnd() SLANG_OVERRIDE { return m_atEnd; } + virtual bool CanRead() SLANG_OVERRIDE { return (int(m_access) & int(FileAccess::Read)) != 0; } + virtual bool CanWrite() SLANG_OVERRIDE { return (int(m_access) & int(FileAccess::Write)) != 0; } + virtual void Close() SLANG_OVERRIDE { m_access = FileAccess::None; } + + MemoryStream(FileAccess access) : + m_access(access), + m_position(0), + m_atEnd(false) + {} + + UInt m_position; + + bool m_atEnd; ///< Happens when a read is done and nothing can be returned because already at end + + FileAccess m_access; + List<uint8_t> m_contents; + }; + class FileStream : public Stream { private: diff --git a/source/slang/compiler.cpp b/source/slang/compiler.cpp index 172bc33b9..a9711310b 100644 --- a/source/slang/compiler.cpp +++ b/source/slang/compiler.cpp @@ -260,7 +260,7 @@ namespace Slang { codeBuilder << "#line 1 \""; - const String& path = sourceFile->pathInfo.foundPath; + const String& path = sourceFile->getPathInfo().foundPath; for(auto c : path) { @@ -277,7 +277,7 @@ namespace Slang } codeBuilder << "\"\n"; - codeBuilder << sourceFile->content << "\n"; + codeBuilder << sourceFile->getContent() << "\n"; } return codeBuilder.ProduceString(); @@ -322,7 +322,7 @@ namespace Slang { codeBuilder << "#line 1 " << translationUnitIndex << "\n"; } - codeBuilder << sourceFile->content << "\n"; + codeBuilder << sourceFile->getContent() << "\n"; } return codeBuilder.ProduceString(); diff --git a/source/slang/compiler.h b/source/slang/compiler.h index 2a9b60d9b..99ac9d68f 100644 --- a/source/slang/compiler.h +++ b/source/slang/compiler.h @@ -361,6 +361,9 @@ namespace Slang // serialization a bottleneck or firewall between the front end and the backend bool useSerialIRBottleneck = false; + // If true will serialize and de-serialize with debug information + bool verifyDebugSerialization = false; + // How should `#line` directives be emitted (if at all)? LineDirectiveMode lineDirectiveMode = LineDirectiveMode::Default; diff --git a/source/slang/diagnostic-defs.h b/source/slang/diagnostic-defs.h index a967fca98..25e17f0c4 100644 --- a/source/slang/diagnostic-defs.h +++ b/source/slang/diagnostic-defs.h @@ -446,5 +446,6 @@ DIAGNOSTIC(99999, Internal, internalCompilerError, "Slang internal compiler erro DIAGNOSTIC(99999, Error, compilationAborted, "Slang compilation aborted due to internal error"); DIAGNOSTIC(99999, Error, compilationAbortedDueToException, "Slang compilation aborted due to an exception of $0: $1"); DIAGNOSTIC(99999, Note, noteLocationOfInternalError, "the Slang compiler threw an exception while working on code near this location"); +DIAGNOSTIC(99999, Internal, serialDebugVerificationFailed, "Verification of serial debug information failed."); #undef DIAGNOSTIC diff --git a/source/slang/ir-serialize.cpp b/source/slang/ir-serialize.cpp index f3ccbd6c1..9c09f53e7 100644 --- a/source/slang/ir-serialize.cpp +++ b/source/slang/ir-serialize.cpp @@ -229,7 +229,7 @@ char* StringRepresentationCache::getCStr(Handle handle) } } -/* static */void SerialStringTableUtil::decodeStringTable(const List<char>& stringTable, List<UnownedStringSlice>& slices) +/* static */void SerialStringTableUtil::appendDecodedStringTable(const List<char>& stringTable, List<UnownedStringSlice>& slicesOut) { const char* start = stringTable.begin(); const char* cur = start; @@ -239,11 +239,40 @@ char* StringRepresentationCache::getCStr(Handle handle) { CharReader reader(cur); const int len = GetUnicodePointFromUTF8(reader); - slices.Add(UnownedStringSlice(reader.m_pos, len)); + slicesOut.Add(UnownedStringSlice(reader.m_pos, len)); cur = reader.m_pos + len; } } +/* static */void SerialStringTableUtil::decodeStringTable(const List<char>& stringTable, List<UnownedStringSlice>& slicesOut) +{ + slicesOut.SetSize(2); + slicesOut[0] = UnownedStringSlice(nullptr, size_t(0)); + slicesOut[1] = UnownedStringSlice("", size_t(0)); + + appendDecodedStringTable(stringTable, slicesOut); +} + +/* static */void SerialStringTableUtil::calcStringSlicePoolMap(const List<UnownedStringSlice>& slices, StringSlicePool& pool, List<StringSlicePool::Handle>& indexMapOut) +{ + SLANG_ASSERT(slices.Count() >= StringSlicePool::kNumDefaultHandles); + SLANG_ASSERT(slices[int(StringSlicePool::kNullHandle)] == "" && slices[int(StringSlicePool::kNullHandle)].begin() == nullptr); + SLANG_ASSERT(slices[int(StringSlicePool::kEmptyHandle)] == ""); + + indexMapOut.SetSize(slices.Count()); + // Set up all of the defaults + for (int i = 0; i < StringSlicePool::kNumDefaultHandles; ++i) + { + indexMapOut[i] = StringSlicePool::Handle(i); + } + + const int numSlices = int(slices.Count()); + for (int i = StringSlicePool::kNumDefaultHandles; i < numSlices ; ++i) + { + indexMapOut[i] = pool.add(slices[i]); + } +} + // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! IRSerialData !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! template<typename T> @@ -254,19 +283,24 @@ static size_t _calcArraySize(const List<T>& list) size_t IRSerialData::calcSizeInBytes() const { - return - _calcArraySize(m_insts) + - _calcArraySize(m_childRuns) + - _calcArraySize(m_externalOperands) + - _calcArraySize(m_stringTable) + + return + _calcArraySize(m_insts) + + _calcArraySize(m_childRuns) + + _calcArraySize(m_externalOperands) + + _calcArraySize(m_stringTable) + /* Raw source locs */ _calcArraySize(m_rawSourceLocs) + /* Debug */ - _calcArraySize(m_debugSourceFiles) + - _calcArraySize(m_debugLineOffsets) + - _calcArraySize(m_debugViewEntries) + - _calcArraySize(m_debugLocRuns) + - _calcArraySize(m_debugStrings); + _calcArraySize(m_debugStringTable) + + _calcArraySize(m_debugLineInfos) + + _calcArraySize(m_debugSourceInfos) + + _calcArraySize(m_debugAdjustedLineInfos) + + _calcArraySize(m_debugSourceLocRuns); +} + +IRSerialData::IRSerialData() +{ + clear(); } void IRSerialData::clear() @@ -279,16 +313,14 @@ void IRSerialData::clear() m_externalOperands.Clear(); m_rawSourceLocs.Clear(); + m_stringTable.Clear(); + // Debug data - m_debugSourceFiles.Clear(); - m_debugLineOffsets.Clear(); - m_debugViewEntries.Clear(); - m_debugLocRuns.Clear(); - m_debugStrings.Clear(); - - m_stringTable.SetSize(2); - m_stringTable[int(kNullStringIndex)] = 0; - m_stringTable[int(kEmptyStringIndex)] = 0; + m_debugLineInfos.Clear(); + m_debugAdjustedLineInfos.Clear(); + m_debugSourceInfos.Clear(); + m_debugSourceLocRuns.Clear(); + m_debugStringTable.Clear(); } template <typename T> @@ -320,15 +352,20 @@ static bool _isEqual(const List<T>& aIn, const List<T>& bIn) return true; } - bool IRSerialData::operator==(const ThisType& rhs) const { - return (this == &rhs) || + return (this == &rhs) || (_isEqual(m_insts, rhs.m_insts) && _isEqual(m_childRuns, rhs.m_childRuns) && _isEqual(m_externalOperands, rhs.m_externalOperands) && _isEqual(m_rawSourceLocs, rhs.m_rawSourceLocs) && - _isEqual(m_stringTable, rhs.m_stringTable)); + _isEqual(m_stringTable, rhs.m_stringTable) && + /* Debug */ + _isEqual(m_debugStringTable, rhs.m_debugStringTable) && + _isEqual(m_debugLineInfos, rhs.m_debugLineInfos) && + _isEqual(m_debugAdjustedLineInfos, rhs.m_debugAdjustedLineInfos) && + _isEqual(m_debugSourceInfos, rhs.m_debugSourceInfos) && + _isEqual(m_debugSourceLocRuns, rhs.m_debugSourceLocRuns)); } // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! IRSerialWriter !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -343,8 +380,9 @@ void IRSerialWriter::_addInstruction(IRInst* inst) m_insts.Add(inst); } +#if 0 // Find a view index that matches the view by file (and perhaps other characteristics in the future) -int _findSourceViewIndex(const List<SourceView*>& viewsIn, SourceView* view) +static int _findSourceViewIndex(const List<SourceView*>& viewsIn, SourceView* view) { const int numViews = int(viewsIn.Count()); SourceView*const* views = viewsIn.begin(); @@ -363,14 +401,186 @@ int _findSourceViewIndex(const List<SourceView*>& viewsIn, SourceView* view) } return -1; } +#endif + +void IRSerialWriter::_addDebugSourceLocRun(SourceLoc sourceLoc, uint32_t startInstIndex, uint32_t numInsts) +{ + SourceView* sourceView = m_sourceManager->findSourceView(sourceLoc); + if (!sourceView) + { + return; + } + + SourceFile* sourceFile = sourceView->getSourceFile(); + DebugSourceFile* debugSourceFile; + { + RefPtr<DebugSourceFile>* ptrDebugSourceFile = m_debugSourceFileMap.TryGetValue(sourceFile); + if (ptrDebugSourceFile == nullptr) + { + const SourceLoc::RawValue baseSourceLoc = m_debugFreeSourceLoc; + m_debugFreeSourceLoc += SourceLoc::RawValue(sourceView->getRange().getSize() + 1); + + debugSourceFile = new DebugSourceFile(sourceFile, baseSourceLoc); + m_debugSourceFileMap.Add(sourceFile, debugSourceFile); + } + else + { + debugSourceFile = *ptrDebugSourceFile; + } + } + + // We need to work out the line index + + int offset = sourceView->getRange().getOffset(sourceLoc); + int lineIndex = sourceFile->calcLineIndexFromOffset(offset); + + IRSerialData::DebugLineInfo lineInfo; + lineInfo.m_lineStartOffset = sourceFile->getLineBreakOffsets()[lineIndex]; + lineInfo.m_lineIndex = lineIndex; + + if (!debugSourceFile->hasLineIndex(lineIndex)) + { + // Add the information about the line + int entryIndex = sourceView->findEntryIndex(sourceLoc); + if (entryIndex < 0) + { + debugSourceFile->m_lineInfos.Add(lineInfo); + } + else + { + const auto& entry = sourceView->getEntries()[entryIndex]; + + IRSerialData::DebugAdjustedLineInfo adjustedLineInfo; + adjustedLineInfo.m_lineInfo = lineInfo; + adjustedLineInfo.m_pathStringIndex = Ser::kNullStringIndex; + + if (StringSlicePool::hasContents(entry.m_pathHandle)) + { + UnownedStringSlice slice = sourceView->getSourceManager()->getStringSlicePool().getSlice(entry.m_pathHandle); + SLANG_ASSERT(slice.size() > 0); + adjustedLineInfo.m_pathStringIndex = Ser::StringIndex(m_debugStringSlicePool.add(slice)); + } + + adjustedLineInfo.m_adjustedLineIndex = lineIndex + entry.m_lineAdjust; + + debugSourceFile->m_adjustedLineInfos.Add(adjustedLineInfo); + } + + debugSourceFile->setHasLineIndex(lineIndex); + } + + // Add the run + IRSerialData::SourceLocRun sourceLocRun; + sourceLocRun.m_numInst = numInsts; + sourceLocRun.m_startInstIndex = IRSerialData::InstIndex(startInstIndex); + sourceLocRun.m_sourceLoc = uint32_t(debugSourceFile->m_baseSourceLoc + offset); + + m_serialData->m_debugSourceLocRuns.Add(sourceLocRun); +} + +Result IRSerialWriter::_calcDebugInfo() +{ + // We need to find the unique source Locs + // We are not going to store SourceLocs directly, because there may be multiple views mapping down to + // the same underlying source file + + // First find all the unique locs + struct InstLoc + { + typedef InstLoc ThisType; + + SLANG_FORCE_INLINE bool operator<(const ThisType& rhs) const { return sourceLoc < rhs.sourceLoc || (sourceLoc == rhs.sourceLoc && instIndex < rhs.instIndex); } + + uint32_t instIndex; + uint32_t sourceLoc; + }; + + // Find all of the source locations and their associated instructions + List<InstLoc> instLocs; + const int numInsts = int(m_insts.Count()); + for (int i = 1; i < numInsts; i++) + { + IRInst* srcInst = m_insts[i]; + if (!srcInst->sourceLoc.isValid()) + { + continue; + } + InstLoc instLoc; + instLoc.instIndex = uint32_t(i); + instLoc.sourceLoc = uint32_t(srcInst->sourceLoc.getRaw()); + instLocs.Add(instLoc); + } + + // Sort them + instLocs.Sort(); + m_debugFreeSourceLoc = 1; + + // Look for runs + const InstLoc* startInstLoc = instLocs.begin(); + const InstLoc* endInstLoc = instLocs.end(); + + while (startInstLoc < endInstLoc) + { + const uint32_t startSourceLoc = startInstLoc->sourceLoc; + + // Find the run with the same source loc + + const InstLoc* curInstLoc = startInstLoc + 1; + uint32_t curInstIndex = startInstLoc->instIndex + 1; + + // Find the run size with same source loc and run of instruction indices + for (; curInstLoc < endInstLoc && curInstLoc->sourceLoc == startSourceLoc && curInstLoc->instIndex == curInstIndex; ++curInstLoc, ++curInstIndex) + { + } + + // Try adding the run + _addDebugSourceLocRun(SourceLoc::fromRaw(startSourceLoc), startInstLoc->instIndex, curInstIndex - startInstLoc->instIndex); + + // Next + startInstLoc = curInstLoc; + } + + // Okay we can now calculate the final source information + + for (auto& pair : m_debugSourceFileMap) + { + DebugSourceFile* debugSourceFile = pair.Value; + SourceFile* sourceFile = debugSourceFile->m_sourceFile; + + IRSerialData::DebugSourceInfo sourceInfo; + + sourceInfo.m_numLines = uint32_t(debugSourceFile->m_sourceFile->getLineBreakOffsets().Count()); + + sourceInfo.m_startSourceLoc = uint32_t(debugSourceFile->m_baseSourceLoc); + sourceInfo.m_endSourceLoc = uint32_t(debugSourceFile->m_baseSourceLoc + sourceFile->getContentSize()); + + sourceInfo.m_pathIndex = Ser::StringIndex(m_debugStringSlicePool.add(sourceFile->getPathInfo().foundPath)); + + sourceInfo.m_lineInfosStartIndex = uint32_t(m_serialData->m_debugLineInfos.Count()); + sourceInfo.m_adjustedLineInfosStartIndex = uint32_t(m_serialData->m_debugAdjustedLineInfos.Count()); + + sourceInfo.m_numLineInfos = uint32_t(debugSourceFile->m_lineInfos.Count()); + sourceInfo.m_numAdjustedLineInfos = uint32_t(debugSourceFile->m_adjustedLineInfos.Count()); + // Add the line infos + m_serialData->m_debugLineInfos.AddRange(debugSourceFile->m_lineInfos.begin(), debugSourceFile->m_lineInfos.Count()); + m_serialData->m_debugAdjustedLineInfos.AddRange(debugSourceFile->m_adjustedLineInfos.begin(), debugSourceFile->m_adjustedLineInfos.Count()); + + // Add the source info + m_serialData->m_debugSourceInfos.Add(sourceInfo); + } + + // Convert the string pool + SerialStringTableUtil::encodeStringTable(m_debugStringSlicePool, m_serialData->m_debugStringTable); + + return SLANG_OK; +} Result IRSerialWriter::write(IRModule* module, SourceManager* sourceManager, OptionFlags options, IRSerialData* serialData) { typedef Ser::Inst::PayloadType PayloadType; - SLANG_UNUSED(sourceManager); - + m_sourceManager = sourceManager; m_serialData = serialData; serialData->clear(); @@ -427,6 +637,18 @@ Result IRSerialWriter::write(IRModule* module, SourceManager* sourceManager, Opt } } +#if 0 + { + List<IRInst*> workInsts; + calcInstructionList(module, workInsts); + SLANG_ASSERT(workInsts.Count() == m_insts.Count()); + for (UInt i = 0; i < workInsts.Count(); ++i) + { + SLANG_ASSERT(workInsts[i] == m_insts[i]); + } + } +#endif + // Set to the right size m_serialData->m_insts.SetSize(m_insts.Count()); // Clear all instructions @@ -545,7 +767,6 @@ Result IRSerialWriter::write(IRModule* module, SourceManager* sourceManager, Opt SerialStringTableUtil::encodeStringTable(m_stringSlicePool, serialData->m_stringTable); } - // If the option to use RawSourceLocations is enabled, serialize out as is if (options & OptionFlag::RawSourceLocation) { @@ -561,7 +782,12 @@ Result IRSerialWriter::write(IRModule* module, SourceManager* sourceManager, Opt dstLocs[i] = Ser::RawSourceLoc(srcInst->sourceLoc.getRaw()); } } - + + if (options & OptionFlag::DebugInfo) + { + _calcDebugInfo(); + } + m_serialData = nullptr; return SLANG_OK; } @@ -823,7 +1049,6 @@ static size_t _calcInstChunkSize(IRSerialBinary::CompressionType compressionType size_t numInsts = size_t(instsIn.Count()); size += numInsts * 2; // op and payload - IRSerialData::Inst* insts = instsIn.begin(); for (size_t i = 0; i < numInsts; ++i) @@ -882,13 +1107,22 @@ static size_t _calcInstChunkSize(IRSerialBinary::CompressionType compressionType { size_t totalSize = 0; - totalSize += sizeof(Bin::SlangHeader) + + totalSize += sizeof(Bin::SlangHeader) + _calcInstChunkSize(compressionType, data.m_insts) + _calcChunkSize(compressionType, data.m_childRuns) + _calcChunkSize(compressionType, data.m_externalOperands) + - _calcChunkSize(Bin::CompressionType::None, data.m_stringTable) + + _calcChunkSize(Bin::CompressionType::None, data.m_stringTable) + _calcChunkSize(Bin::CompressionType::None, data.m_rawSourceLocs); + if (data.m_debugSourceInfos.Count()) + { + totalSize += _calcChunkSize(Bin::CompressionType::None, data.m_debugStringTable) + + _calcChunkSize(Bin::CompressionType::None, data.m_debugLineInfos) + + _calcChunkSize(Bin::CompressionType::None, data.m_debugAdjustedLineInfos) + + _calcChunkSize(Bin::CompressionType::None, data.m_debugSourceInfos) + + _calcChunkSize(compressionType, data.m_debugSourceLocRuns); + } + { Bin::Chunk riffHeader; riffHeader.m_type = Bin::kRiffFourCc; @@ -911,7 +1145,16 @@ static size_t _calcInstChunkSize(IRSerialBinary::CompressionType compressionType SLANG_RETURN_ON_FAIL(_writeArrayChunk(Bin::CompressionType::None, Bin::kStringFourCc, data.m_stringTable, stream)); SLANG_RETURN_ON_FAIL(_writeArrayChunk(Bin::CompressionType::None, Bin::kUInt32SourceLocFourCc, data.m_rawSourceLocs, stream)); - + + if (data.m_debugSourceInfos.Count()) + { + _writeArrayChunk(Bin::CompressionType::None, Bin::kDebugStringFourCc, data.m_debugStringTable, stream); + _writeArrayChunk(Bin::CompressionType::None, Bin::kDebugLineInfoFourCc, data.m_debugLineInfos, stream); + _writeArrayChunk(Bin::CompressionType::None, Bin::kDebugAdjustedLineInfoFourCc, data.m_debugAdjustedLineInfos, stream); + _writeArrayChunk(Bin::CompressionType::None, Bin::kDebugSourceInfoFourCc, data.m_debugSourceInfos, stream); + _writeArrayChunk(compressionType, Bin::kDebugSourceLocRunFourCc, data.m_debugSourceLocRuns, stream); + } + return SLANG_OK; } @@ -1261,11 +1504,42 @@ int64_t _calcChunkTotalSize(const IRSerialBinary::Chunk& chunk) } case Bin::kUInt32SourceLocFourCc: { - SLANG_RETURN_ON_FAIL(_readArrayUncompressedChunk(slangHeader, chunk, stream, &bytesRead, dataOut->m_rawSourceLocs)); remainingBytes -= _calcChunkTotalSize(chunk); break; } + case Bin::kDebugStringFourCc: + { + SLANG_RETURN_ON_FAIL(_readArrayUncompressedChunk(slangHeader, chunk, stream, &bytesRead, dataOut->m_debugStringTable)); + remainingBytes -= _calcChunkTotalSize(chunk); + break; + } + case Bin::kDebugLineInfoFourCc: + { + SLANG_RETURN_ON_FAIL(_readArrayUncompressedChunk(slangHeader, chunk, stream, &bytesRead, dataOut->m_debugLineInfos)); + remainingBytes -= _calcChunkTotalSize(chunk); + break; + } + case Bin::kDebugAdjustedLineInfoFourCc: + { + SLANG_RETURN_ON_FAIL(_readArrayUncompressedChunk(slangHeader, chunk, stream, &bytesRead, dataOut->m_debugAdjustedLineInfos)); + remainingBytes -= _calcChunkTotalSize(chunk); + break; + } + case Bin::kDebugSourceInfoFourCc: + { + SLANG_RETURN_ON_FAIL(_readArrayChunk(slangHeader, chunk, stream, &bytesRead, dataOut->m_debugSourceInfos)); + remainingBytes -= _calcChunkTotalSize(chunk); + break; + } + case SLANG_MAKE_COMPRESSED_FOUR_CC(Bin::kDebugSourceLocRunFourCc): + case Bin::kDebugSourceLocRunFourCc: + { + SLANG_RETURN_ON_FAIL(_readArrayChunk(slangHeader, chunk, stream, &bytesRead, dataOut->m_debugSourceLocRuns)); + remainingBytes -= _calcChunkTotalSize(chunk); + break; + } + default: { SLANG_RETURN_ON_FAIL(_skip(chunk, stream, &remainingBytes)); @@ -1277,7 +1551,35 @@ int64_t _calcChunkTotalSize(const IRSerialBinary::Chunk& chunk) return SLANG_OK; } -/* static */Result IRSerialReader::read(const IRSerialData& data, Session* session, RefPtr<IRModule>& moduleOut) +static SourceRange _toSourceRange(const IRSerialData::DebugSourceInfo& info) +{ + SourceRange range; + range.begin = SourceLoc::fromRaw(info.m_startSourceLoc); + range.end = SourceLoc::fromRaw(info.m_endSourceLoc); + return range; +} + +static int _findIndex(const List<IRSerialData::DebugSourceInfo>& infos, SourceLoc sourceLoc) +{ + const int numInfos = int(infos.Count()); + for (int i = 0; i < numInfos; ++i) + { + if (_toSourceRange(infos[i]).contains(sourceLoc)) + { + return i; + } + } + + return -1; +} + +static int _calcFixSourceLoc(const IRSerialData::DebugSourceInfo& info, SourceView* sourceView, SourceRange& rangeOut) +{ + rangeOut = _toSourceRange(info); + return int(sourceView->getRange().begin.getRaw()) - int(info.m_startSourceLoc); +} + +/* static */Result IRSerialReader::read(const IRSerialData& data, Session* session, SourceManager* sourceManager, RefPtr<IRModule>& moduleOut) { typedef Ser::Inst::PayloadType PayloadType; @@ -1472,6 +1774,321 @@ int64_t _calcChunkTotalSize(const IRSerialBinary::Chunk& chunk) } } + if (sourceManager && m_serialData->m_debugSourceInfos.Count()) + { + List<UnownedStringSlice> debugStringSlices; + SerialStringTableUtil::decodeStringTable(m_serialData->m_debugStringTable, debugStringSlices); + + // All of the strings are placed in the manager (and its StringSlicePool) where the SourceView and SourceFile are constructed from + List<StringSlicePool::Handle> stringMap; + SerialStringTableUtil::calcStringSlicePoolMap(debugStringSlices, sourceManager->getStringSlicePool(), stringMap); + + const List<IRSerialData::DebugSourceInfo>& sourceInfos = m_serialData->m_debugSourceInfos; + + // Construct the source files + int numSourceFiles = int(sourceInfos.Count()); + + // These hold the views (and SourceFile as there is only one SourceFile per view) in the same order as the sourceInfos + List<SourceView*> sourceViews; + sourceViews.SetSize(numSourceFiles); + + for (int i = 0; i < numSourceFiles; ++i) + { + const IRSerialData::DebugSourceInfo& srcSourceInfo = sourceInfos[i]; + + PathInfo pathInfo; + pathInfo.type = PathInfo::Type::FoundPath; + pathInfo.foundPath = debugStringSlices[UInt(srcSourceInfo.m_pathIndex)]; + + RefPtr<SourceFile> sourceFile = sourceManager->createSourceFileWithSize(pathInfo, srcSourceInfo.m_endSourceLoc - srcSourceInfo.m_startSourceLoc); + SourceView* sourceView = sourceManager->createSourceView(sourceFile); + + // We need to accumulate all line numbers, for this source file, both adjusted and unadjusted + List<IRSerialData::DebugLineInfo> lineInfos; + // Add the adjusted lines + { + lineInfos.SetSize(srcSourceInfo.m_numAdjustedLineInfos); + IRSerialData::DebugAdjustedLineInfo* srcAdjustedLineInfos = m_serialData->m_debugAdjustedLineInfos.Buffer() + srcSourceInfo.m_adjustedLineInfosStartIndex; + const int numAdjustedLines = int(srcSourceInfo.m_numAdjustedLineInfos); + for (int j = 0; j < numAdjustedLines; ++j) + { + lineInfos[j] = srcAdjustedLineInfos[j].m_lineInfo; + } + } + // Add regular lines + lineInfos.AddRange(m_serialData->m_debugLineInfos.Buffer() + srcSourceInfo.m_lineInfosStartIndex, srcSourceInfo.m_numLineInfos); + // Put in sourceloc order + lineInfos.Sort(); + + List<uint32_t> lineBreakOffsets; + + // We can now set up the line breaks array + const int numLines = int(srcSourceInfo.m_numLines); + lineBreakOffsets.SetSize(numLines); + + { + const int numLineInfos = int(lineInfos.Count()); + int lineIndex = 0; + + // Every line up and including should hold the same offset + for (int lineInfoIndex = 0; lineInfoIndex < numLineInfos; ++lineInfoIndex) + { + const auto& lineInfo = lineInfos[lineInfoIndex]; + + const uint32_t offset = lineInfo.m_lineStartOffset; + SLANG_ASSERT(offset > 0); + const int finishIndex = int(lineInfo.m_lineIndex); + + SLANG_ASSERT(finishIndex < numLines); + + for (; lineIndex < finishIndex; ++lineIndex) + { + lineBreakOffsets[lineIndex] = offset - 1; + } + lineBreakOffsets[lineIndex] = offset; + lineIndex++; + } + + // Do the remaining lines + const uint32_t offset = uint32_t(srcSourceInfo.m_endSourceLoc - srcSourceInfo.m_startSourceLoc); + for (; lineIndex < numLines; ++lineIndex) + { + lineBreakOffsets[lineIndex] = offset; + } + } + + sourceFile->setLineBreakOffsets(lineBreakOffsets.Buffer(), lineBreakOffsets.Count()); + + if (srcSourceInfo.m_numAdjustedLineInfos) + { + List<IRSerialData::DebugAdjustedLineInfo> adjustedLineInfos; + + int numEntries = int(srcSourceInfo.m_numAdjustedLineInfos); + + adjustedLineInfos.AddRange(m_serialData->m_debugAdjustedLineInfos.Buffer() + srcSourceInfo.m_adjustedLineInfosStartIndex, numEntries); + adjustedLineInfos.Sort(); + + // Work out the views adjustments, and place in dstEntries + List<SourceView::Entry> dstEntries; + dstEntries.SetSize(numEntries); + + const uint32_t sourceLocOffset = uint32_t(sourceView->getRange().begin.getRaw()); + + for (int j = 0; j < numEntries; ++j) + { + const auto& srcEntry = adjustedLineInfos[j]; + auto& dstEntry = dstEntries[j]; + + dstEntry.m_pathHandle = stringMap[int(srcEntry.m_pathStringIndex)]; + dstEntry.m_startLoc = SourceLoc::fromRaw(srcEntry.m_lineInfo.m_lineStartOffset + sourceLocOffset); + dstEntry.m_lineAdjust = int32_t(srcEntry.m_adjustedLineIndex) - int32_t(srcEntry.m_lineInfo.m_lineIndex); + } + + // Set the adjustments on the view + sourceView->setEntries(dstEntries.Buffer(), dstEntries.Count()); + } + + sourceViews[i] = sourceView; + } + + // We now need to apply the runs + { + List<IRSerialData::SourceLocRun> sourceRuns(m_serialData->m_debugSourceLocRuns); + // They are now in source location order + sourceRuns.Sort(); + + // Just guess initially 0 for the source file that contains the initial run + SourceRange range; + int fixSourceLoc = _calcFixSourceLoc(sourceInfos[0], sourceViews[0], range); + + const int numRuns = int(sourceRuns.Count()); + for (int i = 0; i < numRuns; ++i) + { + const auto& run = sourceRuns[i]; + const SourceLoc srcSourceLoc = SourceLoc::fromRaw(run.m_sourceLoc); + + if (!range.contains(srcSourceLoc)) + { + int index = _findIndex(sourceInfos, srcSourceLoc); + if (index < 0) + { + // Didn't find the match + continue; + } + fixSourceLoc = _calcFixSourceLoc(sourceInfos[index], sourceViews[index], range); + SLANG_ASSERT(range.contains(srcSourceLoc)); + } + + // Work out the fixed source location + SourceLoc sourceLoc = SourceLoc::fromRaw(int(run.m_sourceLoc) + fixSourceLoc); + + SLANG_ASSERT(uint32_t(run.m_startInstIndex) + run.m_numInst <= insts.Count()); + IRInst** dstInsts = insts.Buffer() + int(run.m_startInstIndex); + + const int runSize = int(run.m_numInst); + for (int j = 0; j < runSize; ++j) + { + dstInsts[j]->sourceLoc = sourceLoc; + } + } + } + } + + return SLANG_OK; +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!! IRSerialUtil !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +/* static */void IRSerialUtil::calcInstructionList(IRModule* module, List<IRInst*>& instsOut) +{ + // We reserve 0 for null + instsOut.SetSize(1); + instsOut[0] = nullptr; + + // Stack for parentInst + List<IRInst*> parentInstStack; + + IRModuleInst* moduleInst = module->getModuleInst(); + parentInstStack.Add(moduleInst); + + // Add to list + instsOut.Add(moduleInst); + + // Traverse all of the instructions + while (parentInstStack.Count()) + { + // If it's in the stack it is assumed it is already in the inst map + IRInst* parentInst = parentInstStack.Last(); + parentInstStack.RemoveLast(); + + IRInstListBase childrenList = parentInst->getDecorationsAndChildren(); + for (IRInst* child : childrenList) + { + instsOut.Add(child); + parentInstStack.Add(child); + } + } +} + +/* static */SlangResult IRSerialUtil::verifySerialize(IRModule* module, Session* session, SourceManager* sourceManager, IRSerialBinary::CompressionType compressionType, IRSerialWriter::OptionFlags optionFlags) +{ + // Verify if we can stream out with debug information + + List<IRInst*> originalInsts; + calcInstructionList(module, originalInsts); + + IRSerialData serialData; + { + // Write IR out to serialData - copying over SourceLoc information directly + IRSerialWriter writer; + SLANG_RETURN_ON_FAIL(writer.write(module, sourceManager, optionFlags, &serialData)); + } + + // Write the data out to stream + MemoryStream memoryStream(FileAccess::ReadWrite); + SLANG_RETURN_ON_FAIL(IRSerialWriter::writeStream(serialData, compressionType, &memoryStream)); + + // Reset stream + memoryStream.Seek(SeekOrigin::Start, 0); + + IRSerialData readData; + + SLANG_RETURN_ON_FAIL(IRSerialReader::readStream(&memoryStream, &readData)); + + // Check the stream read data is the same + if (readData != serialData) + { + SLANG_ASSERT(!"Streamed in data doesn't match"); + return SLANG_FAIL; + } + + RefPtr<IRModule> irReadModule; + + SourceManager workSourceManager; + workSourceManager.initialize(sourceManager); + + { + IRSerialReader reader; + SLANG_RETURN_ON_FAIL(reader.read(serialData, session, &workSourceManager, irReadModule)); + } + + List<IRInst*> readInsts; + calcInstructionList(irReadModule, readInsts); + + if (readInsts.Count() != originalInsts.Count()) + { + SLANG_ASSERT(!"Instruction counts don't match"); + return SLANG_FAIL; + } + + if (optionFlags & IRSerialWriter::OptionFlag::RawSourceLocation) + { + SLANG_ASSERT(readInsts[0] == originalInsts[0]); + // All the source locs should be identical + for (UInt i = 1; i < readInsts.Count(); ++i) + { + IRInst* origInst = originalInsts[i]; + IRInst* readInst = readInsts[i]; + + if (origInst->sourceLoc.getRaw() != readInst->sourceLoc.getRaw()) + { + SLANG_ASSERT(!"Source locs don't match"); + return SLANG_FAIL; + } + } + } + else if (optionFlags & IRSerialWriter::OptionFlag::DebugInfo) + { + // They should be on the same line nos + for (UInt i = 1; i < readInsts.Count(); ++i) + { + IRInst* origInst = originalInsts[i]; + IRInst* readInst = readInsts[i]; + + if (origInst->sourceLoc.getRaw() == readInst->sourceLoc.getRaw()) + { + continue; + } + + // Work out the + SourceView* origSourceView = sourceManager->findSourceView(origInst->sourceLoc); + SourceView* readSourceView = workSourceManager.findSourceView(readInst->sourceLoc); + + // if both are null we are done + if (origSourceView == nullptr && origSourceView == readSourceView) + { + continue; + } + SLANG_ASSERT(origSourceView && readSourceView); + + { + auto origInfo = origSourceView->getHumaneLoc(origInst->sourceLoc, SourceLocType::Actual); + auto readInfo = readSourceView->getHumaneLoc(readInst->sourceLoc, SourceLocType::Actual); + + if (!(origInfo.line == readInfo.line && origInfo.column == readInfo.column && origInfo.pathInfo.foundPath == readInfo.pathInfo.foundPath)) + { + SLANG_ASSERT(!"Debug data didn't match"); + return SLANG_FAIL; + } + } + + // We may have adjusted line numbers -> but they may not match, because we only reconstruct one view + // So for now disable this test + + if (false) + { + auto origInfo = origSourceView->getHumaneLoc(origInst->sourceLoc, SourceLocType::Nominal); + auto readInfo = readSourceView->getHumaneLoc(readInst->sourceLoc, SourceLocType::Nominal); + + if (!(origInfo.line == readInfo.line && origInfo.column == readInfo.column && origInfo.pathInfo.foundPath == readInfo.pathInfo.foundPath)) + { + SLANG_ASSERT(!"Debug data didn't match"); + return SLANG_FAIL; + } + } + } + } + return SLANG_OK; } diff --git a/source/slang/ir-serialize.h b/source/slang/ir-serialize.h index c205e7a35..667ea8743 100644 --- a/source/slang/ir-serialize.h +++ b/source/slang/ir-serialize.h @@ -52,11 +52,16 @@ class StringRepresentationCache struct SerialStringTableUtil { - /// Convert a pool into a string table + /// Convert a pool into a string table static void encodeStringTable(const StringSlicePool& pool, List<char>& stringTable); static void encodeStringTable(const UnownedStringSlice* slices, size_t numSlices, List<char>& stringTable); - /// Converts a pool into a string table, appending the strings to the slices + /// Appends the decoded strings into slicesOut + static void appendDecodedStringTable(const List<char>& stringTable, List<UnownedStringSlice>& slicesOut); + /// Decodes a string table (and does so such that the indices are compatible with StringSlicePool) static void decodeStringTable(const List<char>& stringTable, List<UnownedStringSlice>& slicesOut); + + /// Produces an index map, from slices to indices in pool + static void calcStringSlicePoolMap(const List<UnownedStringSlice>& slices, StringSlicePool& pool, List<StringSlicePool::Handle>& indexMap); }; // Pre-declare @@ -95,12 +100,87 @@ struct IRSerialData SizeType m_numChildren; ///< The number of children }; + struct SourceLocRun + { + typedef SourceLocRun ThisType; + + bool operator==(const ThisType& rhs) const { return m_sourceLoc == rhs.m_sourceLoc && m_startInstIndex == rhs.m_startInstIndex && m_numInst == rhs.m_numInst; } + bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } + bool operator<(const ThisType& rhs) const { return m_sourceLoc < rhs.m_sourceLoc; } + + uint32_t m_sourceLoc; ///< The source location + InstIndex m_startInstIndex; ///< The index to the first instruction + SizeType m_numInst; ///< The number of children + }; + struct PayloadInfo { uint8_t m_numOperands; uint8_t m_numStrings; }; + struct DebugSourceInfo + { + typedef DebugSourceInfo ThisType; + + bool operator==(const ThisType& rhs) const + { + return m_pathIndex == rhs.m_pathIndex && + m_startSourceLoc == rhs.m_startSourceLoc && + m_endSourceLoc == rhs.m_endSourceLoc && + m_numLineInfos == rhs.m_numLineInfos && + m_lineInfosStartIndex == rhs.m_lineInfosStartIndex && + m_numLineInfos == rhs.m_numLineInfos; + } + bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } + + bool isSourceLocInRange(uint32_t sourceLoc) const { return sourceLoc >= m_startSourceLoc && sourceLoc <= m_endSourceLoc; } + + StringIndex m_pathIndex; ///< Index to the string table + uint32_t m_startSourceLoc; ///< The offset to the source + uint32_t m_endSourceLoc; ///< The number of bytes in the source + + uint32_t m_numLines; ///< Total number of lines in source file + + uint32_t m_lineInfosStartIndex; ///< Index into m_debugLineInfos + uint32_t m_numLineInfos; ///< The number of line infos + + uint32_t m_adjustedLineInfosStartIndex; ///< Adjusted start index + uint32_t m_numAdjustedLineInfos; ///< The number of line infos + }; + + struct DebugLineInfo + { + typedef DebugLineInfo ThisType; + bool operator<(const ThisType& rhs) const { return m_lineStartOffset < rhs.m_lineStartOffset; } + bool operator==(const ThisType& rhs) const + { + return m_lineStartOffset == rhs.m_lineStartOffset && + m_lineIndex == rhs.m_lineIndex; + } + bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } + + uint32_t m_lineStartOffset; ///< The offset into the source file + uint32_t m_lineIndex; ///< Original line index + }; + + struct DebugAdjustedLineInfo + { + typedef DebugAdjustedLineInfo ThisType; + bool operator==(const ThisType& rhs) const + { + return m_lineInfo == rhs.m_lineInfo && + m_adjustedLineIndex == rhs.m_adjustedLineIndex && + m_pathStringIndex == rhs.m_pathStringIndex; + } + bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } + bool operator<(const ThisType& rhs) const { return m_lineInfo < rhs.m_lineInfo; } + + DebugLineInfo m_lineInfo; + uint32_t m_adjustedLineIndex; ///< The line index with the adjustment (if there is any). Is 0 if m_pathStringIndex is 0. + StringIndex m_pathStringIndex; ///< The path as an index + }; + // Instruction... // We can store SourceLoc values separately. Just store per index information. // Parent information is stored in m_childRuns @@ -173,32 +253,6 @@ struct IRSerialData Payload m_payload; }; - struct DebugSourceFile - { - uint32_t m_startLoc; ///< Start of the location range - uint32_t m_endLoc; ///< The end of the location range - - uint32_t m_pathIndex; ///< Path associated - - uint32_t m_numLocRuns; ///< The number of location runs associated with this source file - uint32_t m_numLineOffsets; ///< The number of offsets associated with the file - uint32_t m_numDebugViewEntries; ///< The number of debug view entries - }; - - struct DebugViewEntry - { - uint32_t m_startLoc; ///< Where does this entry begin? - uint32_t m_pathIndex; ///< What is the presumed path for this entry. If 0 it means there is no path. - int32_t m_lineAdjust; ///< The line adjustment - }; - - struct DebugLocRun - { - uint32_t m_sourceLoc; ///< The location - uint32_t startInstIndex; ///< The start instruction index - uint32_t numInst; ///< The amount of instructions - }; - /// Clear to initial state void clear(); /// Get the operands of an instruction @@ -212,24 +266,25 @@ struct IRSerialData size_t calcSizeInBytes() const; /// Ctor - IRSerialData() - {} - + IRSerialData(); + List<Inst> m_insts; ///< The instructions List<InstRun> m_childRuns; ///< Holds the information about children that belong to an instruction List<InstIndex> m_externalOperands; ///< Holds external operands (for instructions with more than kNumOperands) - List<char> m_stringTable; ///< All strings. Indexed into by StringIndex + List<char> m_stringTable; ///< All strings. Indexed into by StringIndex List<RawSourceLoc> m_rawSourceLocs; ///< A source location per instruction (saved without modification from IRInst)s - List<DebugSourceFile> m_debugSourceFiles; ///< The files associated - List<uint32_t> m_debugLineOffsets; ///< All of the debug line offsets - List<uint32_t> m_debugViewEntries; ///< The debug view entries - that modify line meanings - List<DebugLocRun> m_debugLocRuns; ///< Maps source locations to instructions - List<char> m_debugStrings; ///< All of the debug strings + // Data only set if we have debug information + + List<char> m_debugStringTable; ///< String table for debug use only + List<DebugLineInfo> m_debugLineInfos; ///< Debug line information + List<DebugAdjustedLineInfo> m_debugAdjustedLineInfos; ///< Adjusted line infos + List<DebugSourceInfo> m_debugSourceInfos; ///< Debug source information + List<SourceLocRun> m_debugSourceLocRuns; ///< Runs of instructions that use a source loc static const PayloadInfo s_payloadInfos[int(Inst::PayloadType::CountOf)]; }; @@ -275,6 +330,7 @@ SLANG_FORCE_INLINE bool IRSerialData::Inst::operator==(const ThisType& rhs) cons default: break; } } + return false; } // -------------------------------------------------------------------------- @@ -327,9 +383,15 @@ struct IRSerialBinary static const uint32_t kCompressedExternalOperandsFourCc = SLANG_MAKE_COMPRESSED_FOUR_CC(kExternalOperandsFourCc); static const uint32_t kStringFourCc = SLANG_FOUR_CC('S', 'L', 's', 't'); - /// 4 bytes per entry + static const uint32_t kUInt32SourceLocFourCc = SLANG_FOUR_CC('S', 'r', 's', '4'); + static const uint32_t kDebugStringFourCc = SLANG_FOUR_CC('S', 'd', 's', 't'); + static const uint32_t kDebugLineInfoFourCc = SLANG_FOUR_CC('S', 'd', 'l', 'n'); + static const uint32_t kDebugAdjustedLineInfoFourCc = SLANG_FOUR_CC('S', 'd', 'a', 'l'); + static const uint32_t kDebugSourceInfoFourCc = SLANG_FOUR_CC('S', 'd', 's', 'o'); + static const uint32_t kDebugSourceLocRunFourCc = SLANG_FOUR_CC('S', 'd', 's', 'r'); + struct SlangHeader { Chunk m_chunk; @@ -360,15 +422,15 @@ struct IRSerialWriter enum Enum: Type { RawSourceLocation = 0x01, + DebugInfo = 0x02, }; }; typedef OptionFlag::Type OptionFlags; Result write(IRModule* module, SourceManager* sourceManager, OptionFlags options, IRSerialData* serialData); - + static Result writeStream(const IRSerialData& data, Bin::CompressionType compressionType, Stream* stream); - /// Get an instruction index from an instruction Ser::InstIndex getInstIndex(IRInst* inst) const { return inst ? Ser::InstIndex(m_instMap[inst]) : Ser::InstIndex(0); } @@ -381,13 +443,49 @@ struct IRSerialWriter Ser::StringIndex getStringIndex(const char* chars) { return Ser::StringIndex(m_stringSlicePool.add(chars)); } Ser::StringIndex getStringIndex(const String& string) { return Ser::StringIndex(m_stringSlicePool.add(string.getUnownedSlice())); } + StringSlicePool& getStringPool() { return m_stringSlicePool; } + StringSlicePool& getDebugStringPool() { return m_debugStringSlicePool; } + IRSerialWriter() : m_serialData(nullptr) {} protected: + class DebugSourceFile : public RefObject + { + public: + DebugSourceFile(SourceFile* sourceFile, SourceLoc::RawValue baseSourceLoc): + m_sourceFile(sourceFile), + m_baseSourceLoc(baseSourceLoc) + { + // Need to know how many lines there are + const List<uint32_t>& lineOffsets = sourceFile->getLineBreakOffsets(); + + const auto numLineIndices = lineOffsets.Count(); + + // Set none as being used initially + m_lineIndexUsed.SetSize(numLineIndices); + ::memset(m_lineIndexUsed.begin(), 0, numLineIndices * sizeof(uint8_t)); + } + /// True if we have information on that line index + bool hasLineIndex(int lineIndex) const { return m_lineIndexUsed[lineIndex] != 0; } + void setHasLineIndex(int lineIndex) { m_lineIndexUsed[lineIndex] = 1; } + + SourceLoc::RawValue m_baseSourceLoc; ///< The base source location + + RefPtr<SourceFile> m_sourceFile; ///< The source file + List<uint8_t> m_lineIndexUsed; ///< Has 1 if the line is used + List<uint32_t> m_usedLineIndices; ///< Holds the lines that have been hit + + List<IRSerialData::DebugLineInfo> m_lineInfos; ///< The line infos + List<IRSerialData::DebugAdjustedLineInfo> m_adjustedLineInfos; ///< The adjusted line infos + }; + void _addInstruction(IRInst* inst); - + Result _calcDebugInfo(); + /// Returns the remapped sourceLoc, or 0 if sourceLoc couldn't be added + void _addDebugSourceLocRun(SourceLoc sourceLoc, uint32_t startInstIndex, uint32_t numInst); + List<IRInst*> m_insts; ///< Instructions in same order as stored in the List<IRDecoration*> m_decorations; ///< Holds all decorations in order of the instructions as found @@ -399,6 +497,11 @@ protected: IRSerialData* m_serialData; ///< Where the data is stored StringSlicePool m_debugStringSlicePool; ///< Slices held just for debug usage + + SourceLoc::RawValue m_debugFreeSourceLoc; /// Locations greater than this are free + Dictionary<SourceFile*, RefPtr<DebugSourceFile> > m_debugSourceFileMap; + + SourceManager* m_sourceManager; ///< The source manager }; struct IRSerialReader @@ -410,7 +513,7 @@ struct IRSerialReader static Result readStream(Stream* stream, IRSerialData* dataOut); /// Read a module from serial data - Result read(const IRSerialData& data, Session* session, RefPtr<IRModule>& moduleOut); + Result read(const IRSerialData& data, Session* session, SourceManager* sourceManager, RefPtr<IRModule>& moduleOut); /// Get the representation cache StringRepresentationCache& getStringRepresentationCache() { return m_stringRepresentationCache; } @@ -423,7 +526,6 @@ struct IRSerialReader protected: - IRDecoration* _createDecoration(const Ser::Inst& srcIns); static Result _skip(const IRSerialBinary::Chunk& chunk, Stream* stream, int64_t* remainingBytesInOut); StringRepresentationCache m_stringRepresentationCache; @@ -432,6 +534,16 @@ struct IRSerialReader IRModule* m_module; }; +struct IRSerialUtil +{ + /// Produces an instruction list which is in same order as written through IRSerialWriter + static void calcInstructionList(IRModule* module, List<IRInst*>& instsOut); + + /// Verify serialization + static SlangResult verifySerialize(IRModule* module, Session* session, SourceManager* sourceManager, IRSerialBinary::CompressionType compressionType, IRSerialWriter::OptionFlags optionFlags); +}; + + } // namespace Slang #endif diff --git a/source/slang/options.cpp b/source/slang/options.cpp index f1a5798ce..f22d8a48d 100644 --- a/source/slang/options.cpp +++ b/source/slang/options.cpp @@ -456,6 +456,10 @@ struct OptionsParser { requestImpl->useSerialIRBottleneck = true; } + else if (argStr == "-verify-debug-serial-ir") + { + requestImpl->verifyDebugSerialization = true; + } else if(argStr == "-validate-ir" ) { requestImpl->shouldValidateIR = true; diff --git a/source/slang/preprocessor.cpp b/source/slang/preprocessor.cpp index 21942c8e5..d91836c40 100644 --- a/source/slang/preprocessor.cpp +++ b/source/slang/preprocessor.cpp @@ -842,7 +842,7 @@ top: // We create a dummy file to represent the token-paste operation PathInfo pathInfo = PathInfo::makeTokenPaste(); - SourceFile* sourceFile = sourceManager->createSourceFile(pathInfo, sb.ProduceString()); + RefPtr<SourceFile> sourceFile = sourceManager->createSourceFileWithString(pathInfo, sb.ProduceString()); SourceView* sourceView = sourceManager->createSourceView(sourceFile); @@ -1634,7 +1634,7 @@ static void HandleIncludeDirective(PreprocessorDirectiveContext* context) auto sourceManager = context->preprocessor->getCompileRequest()->getSourceManager(); // See if this an already loaded source file - SourceFile* sourceFile = sourceManager->findSourceFileRecursively(filePathInfo.canonicalPath); + RefPtr<SourceFile> sourceFile = sourceManager->findSourceFileRecursively(filePathInfo.canonicalPath); // If not create a new one, and add to the list of known source files if (!sourceFile) { @@ -1645,7 +1645,7 @@ static void HandleIncludeDirective(PreprocessorDirectiveContext* context) return; } - sourceFile = sourceManager->createSourceFile(filePathInfo, foundSourceBlob); + sourceFile = sourceManager->createSourceFileWithBlob(filePathInfo, foundSourceBlob); sourceManager->addSourceFile(filePathInfo.canonicalPath, sourceFile); } @@ -2268,8 +2268,8 @@ static void DefineMacro( auto sourceManager = preprocessor->translationUnit->compileRequest->getSourceManager(); - SourceFile* keyFile = sourceManager->createSourceFile(pathInfo, key); - SourceFile* valueFile = sourceManager->createSourceFile(pathInfo, value); + RefPtr<SourceFile> keyFile = sourceManager->createSourceFileWithString(pathInfo, key); + RefPtr<SourceFile> valueFile = sourceManager->createSourceFileWithString(pathInfo, value); SourceView* keyView = sourceManager->createSourceView(keyFile); SourceView* valueView = sourceManager->createSourceView(valueFile); diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp index b0adb2025..89b56e409 100644 --- a/source/slang/slang.cpp +++ b/source/slang/slang.cpp @@ -15,6 +15,8 @@ #include "slang-file-system.h" #include "../core/slang-writer.h" +#include "source-loc.h" + #include "ir-serialize.h" // Used to print exception type names in internal-compiler-error messages @@ -418,7 +420,7 @@ RefPtr<Expr> CompileRequest::parseTypeString(TranslationUnitRequest * translatio SourceManager localSourceManager; localSourceManager.initialize(sourceManager); - Slang::RefPtr<Slang::SourceFile> srcFile(localSourceManager.createSourceFile(PathInfo::makeTypeParse(), typeStr)); + Slang::RefPtr<Slang::SourceFile> srcFile(localSourceManager.createSourceFileWithString(PathInfo::makeTypeParse(), typeStr)); // We'll use a temporary diagnostic sink DiagnosticSink sink; @@ -559,22 +561,45 @@ void CompileRequest::generateIR() // in isolation. for( auto& translationUnit : translationUnits ) { + // TODO JS: + // This is a bit of HACK. Apparently if we call generateIRForTranslationUnit(translationUnit) twice + // we get a different result (!). + // So here, we only create once even if we run verification. + RefPtr<IRModule> irModule; + + if (verifyDebugSerialization) + { + /// Generate IR for translation unit + irModule = generateIRForTranslationUnit(translationUnit); + + // Verify debug information + if (SLANG_FAILED(IRSerialUtil::verifySerialize(irModule, mSession, sourceManager, IRSerialBinary::CompressionType::None, IRSerialWriter::OptionFlag::DebugInfo))) + { + mSink.diagnose(irModule->moduleInst->sourceLoc, Diagnostics::serialDebugVerificationFailed); + } + } + if (useSerialIRBottleneck) { IRSerialData serialData; { /// Generate IR for translation unit - RefPtr<IRModule> irModule(generateIRForTranslationUnit(translationUnit)); + if (!irModule) + { + irModule = generateIRForTranslationUnit(translationUnit); + } // Write IR out to serialData - copying over SourceLoc information directly IRSerialWriter writer; writer.write(irModule, sourceManager, IRSerialWriter::OptionFlag::RawSourceLocation, &serialData); + + irModule = nullptr; } RefPtr<IRModule> irReadModule; { // Read IR back from serialData IRSerialReader reader; - reader.read(serialData, mSession, irReadModule); + reader.read(serialData, mSession, nullptr, irReadModule); } // Use the serialized irModule @@ -582,7 +607,12 @@ void CompileRequest::generateIR() } else { - translationUnit->irModule = generateIRForTranslationUnit(translationUnit); + if (!irModule) + { + irModule = generateIRForTranslationUnit(translationUnit); + } + + translationUnit->irModule = irModule; } } } @@ -749,8 +779,8 @@ void CompileRequest::addTranslationUnitSourceBlob( ISlangBlob* sourceBlob) { PathInfo pathInfo = PathInfo::makePath(path); - RefPtr<SourceFile> sourceFile = getSourceManager()->createSourceFile(pathInfo, sourceBlob); - + RefPtr<SourceFile> sourceFile = getSourceManager()->createSourceFileWithBlob(pathInfo, sourceBlob); + addTranslationUnitSourceFile(translationUnitIndex, sourceFile); } @@ -760,8 +790,8 @@ void CompileRequest::addTranslationUnitSourceString( String const& source) { PathInfo pathInfo = PathInfo::makePath(path); - RefPtr<SourceFile> sourceFile = getSourceManager()->createSourceFile(pathInfo, source); - + RefPtr<SourceFile> sourceFile = getSourceManager()->createSourceFileWithString(pathInfo, source); + addTranslationUnitSourceFile(translationUnitIndex, sourceFile); } @@ -885,8 +915,8 @@ RefPtr<ModuleDecl> CompileRequest::loadModule( translationUnit->compileFlags = 0; // Create with the 'friendly' name - RefPtr<SourceFile> sourceFile = getSourceManager()->createSourceFile(filePathInfo, sourceBlob); - + RefPtr<SourceFile> sourceFile = getSourceManager()->createSourceFileWithBlob(filePathInfo, sourceBlob); + translationUnit->sourceFiles.Add(sourceFile); int errorCountBefore = mSink.GetErrorCount(); diff --git a/source/slang/source-loc.cpp b/source/slang/source-loc.cpp index 6d6d63ae1..c567dbf22 100644 --- a/source/slang/source-loc.cpp +++ b/source/slang/source-loc.cpp @@ -159,7 +159,7 @@ PathInfo SourceView::_getPathInfo(StringSlicePool::Handle pathHandle) const // If there is no override path, then just the source files path if (pathHandle == StringSlicePool::Handle(0)) { - return m_sourceFile->pathInfo; + return m_sourceFile->getPathInfo(); } else { @@ -172,7 +172,7 @@ PathInfo SourceView::getPathInfo(SourceLoc loc, SourceLocType type) { if (type == SourceLocType::Actual) { - return m_sourceFile->pathInfo; + return m_sourceFile->getPathInfo(); } const int entryIndex = findEntryIndex(loc); @@ -181,6 +181,12 @@ PathInfo SourceView::getPathInfo(SourceLoc loc, SourceLocType type) /* !!!!!!!!!!!!!!!!!!!!!!! SourceFile !!!!!!!!!!!!!!!!!!!!!!!!!!!! */ +void SourceFile::setLineBreakOffsets(const uint32_t* offsets, UInt numOffsets) +{ + m_lineBreakOffsets.Clear(); + m_lineBreakOffsets.AddRange(offsets, numOffsets); +} + const List<uint32_t>& SourceFile::getLineBreakOffsets() { // We now have a raw input file that we can search for line breaks. @@ -188,6 +194,8 @@ const List<uint32_t>& SourceFile::getLineBreakOffsets() // cache an array of line break locations in the file. if (m_lineBreakOffsets.Count() == 0) { + UnownedStringSlice content = getContent(); + char const* begin = content.begin(); char const* end = content.end(); @@ -232,7 +240,7 @@ const List<uint32_t>& SourceFile::getLineBreakOffsets() int SourceFile::calcLineIndexFromOffset(int offset) { - SLANG_ASSERT(UInt(offset) <= content.size()); + SLANG_ASSERT(UInt(offset) <= getContentSize()); // Make sure we have the line break offsets const auto& lineBreakOffsets = getLineBreakOffsets(); @@ -266,6 +274,38 @@ int SourceFile::calcColumnIndex(int lineIndex, int offset) return offset - lineBreakOffsets[lineIndex]; } +/* !!!!!!!!!!!!!!!!!!!!!!!!! SourceFile !!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +void SourceFile::setContents(ISlangBlob* blob) +{ + const UInt contentSize = blob->getBufferSize(); + + SLANG_ASSERT(contentSize == m_contentSize); + + char const* contentBegin = (char const*)blob->getBufferPointer(); + char const* contentEnd = contentBegin + contentSize; + + m_contentBlob = blob; + m_content = UnownedStringSlice(contentBegin, contentEnd); +} + +void SourceFile::setContents(const String& content) +{ + ComPtr<ISlangBlob> contentBlob = StringUtil::createStringBlob(content); + setContents(contentBlob); +} + +SourceFile::SourceFile(const PathInfo& pathInfo, size_t contentSize) : + m_pathInfo(pathInfo), + m_contentSize(contentSize) +{ +} + +SourceFile::~SourceFile() +{ + +} + /* !!!!!!!!!!!!!!!!!!!!!!!!! SourceManager !!!!!!!!!!!!!!!!!!!!!!!!!!!! */ void SourceManager::initialize( @@ -319,29 +359,29 @@ SourceRange SourceManager::allocateSourceRange(UInt size) return SourceRange(beginLoc, endLoc); } -SourceFile* SourceManager::createSourceFile(const PathInfo& pathInfo, ISlangBlob* contentBlob) +RefPtr<SourceFile> SourceManager::createSourceFileWithSize(const PathInfo& pathInfo, size_t contentSize) { - char const* contentBegin = (char const*) contentBlob->getBufferPointer(); - UInt contentSize = contentBlob->getBufferSize(); - char const* contentEnd = contentBegin + contentSize; + SourceFile* sourceFile = new SourceFile(pathInfo, contentSize); + return sourceFile; +} - SourceFile* sourceFile = new SourceFile(); - sourceFile->pathInfo = pathInfo; - sourceFile->contentBlob = contentBlob; - sourceFile->content = UnownedStringSlice(contentBegin, contentEnd); - +RefPtr<SourceFile> SourceManager::createSourceFileWithString(const PathInfo& pathInfo, const String& contents) +{ + SourceFile* sourceFile = new SourceFile(pathInfo, contents.Length()); + sourceFile->setContents(contents); return sourceFile; } - -SourceFile* SourceManager::createSourceFile(const PathInfo& pathInfo, const String& content) + +RefPtr<SourceFile> SourceManager::createSourceFileWithBlob(const PathInfo& pathInfo, ISlangBlob* blob) { - ComPtr<ISlangBlob> contentBlob = StringUtil::createStringBlob(content); - return createSourceFile(pathInfo, contentBlob); + RefPtr<SourceFile> sourceFile(new SourceFile(pathInfo, blob->getBufferSize())); + sourceFile->setContents(blob); + return sourceFile; } SourceView* SourceManager::createSourceView(SourceFile* sourceFile) { - SourceRange range = allocateSourceRange(sourceFile->content.size()); + SourceRange range = allocateSourceRange(sourceFile->getContentSize()); SourceView* sourceView = new SourceView(this, sourceFile, range); m_sourceViews.Add(sourceView); diff --git a/source/slang/source-loc.h b/source/slang/source-loc.h index 6949d659a..b6b353181 100644 --- a/source/slang/source-loc.h +++ b/source/slang/source-loc.h @@ -149,17 +149,47 @@ public: /// Note that this is lazily evaluated - the line breaks are only calculated on the first request const List<uint32_t>& getLineBreakOffsets(); + /// Set the line break offsets + void setLineBreakOffsets(const uint32_t* offsets, UInt numOffsets); + /// Calculate the line based on the offset int calcLineIndexFromOffset(int offset); /// Calculate the offset for a line int calcColumnIndex(int line, int offset); - PathInfo pathInfo; ///< The path The logical file path to report for locations inside this span. - ComPtr<ISlangBlob> contentBlob; ///< A blob that owns the storage for the file contents - UnownedStringSlice content; ///< The actual contents of the file. + /// Get the content holding blob + ISlangBlob* getContentBlob() const { return m_contentBlob; } + + /// True if has full set content + bool hasContent() const { return m_contentBlob != nullptr; } + + /// Get the content size + size_t getContentSize() const { return m_contentSize; } + + /// Get the content + const UnownedStringSlice& getContent() const { return m_content; } + + /// Get path info + const PathInfo& getPathInfo() const { return m_pathInfo; } + + /// Set the content as a blob + void setContents(ISlangBlob* blob); + /// Set the content as a string + void setContents(const String& content); + + /// Ctor + SourceFile(const PathInfo& pathInfo, size_t contentSize); + /// Dtor + ~SourceFile(); protected: + + PathInfo m_pathInfo; ///< The path The logical file path to report for locations inside this span. + ComPtr<ISlangBlob> m_contentBlob; ///< A blob that owns the storage for the file contents. If nullptr, there is no contents + UnownedStringSlice m_content; ///< The actual contents of the file. + size_t m_contentSize; ///< The size of the actual contents + // In order to speed up lookup of line number information, // we will cache the starting offset of each line break in // the input file: @@ -221,13 +251,19 @@ class SourceView: public RefObject const SourceRange& getRange() const { return m_range; } /// Get the entries const List<Entry>& getEntries() const { return m_entries; } + /// Set the entries list + void setEntries(const Entry* entries, UInt numEntries) { m_entries.Clear(); m_entries.AddRange(entries, numEntries); } + /// Get the source file holds the contents this view SourceFile* getSourceFile() const { return m_sourceFile; } /// Get the source manager SourceManager* getSourceManager() const { return m_sourceManager; } /// Get the associated 'content' (the source text) - const UnownedStringSlice& getContent() const { return m_sourceFile->content; } + const UnownedStringSlice& getContent() const { return m_sourceFile->getContent(); } + + /// Get the size of the content + size_t getContentSize() const { return m_sourceFile->getContentSize(); } /// Get the humane location /// Type determines if the location wanted is the original, or the 'normal' (which modifys behavior based on #line directives) @@ -262,9 +298,9 @@ struct SourceManager SourceRange allocateSourceRange(UInt size); /// Create a SourceFile defined with the specified path, and content held within a blob - SourceFile* createSourceFile(const PathInfo& pathInfo, ISlangBlob* content); - /// Create a SourceFile with specified path. Create a Blob that contains the content. - SourceFile* createSourceFile(const PathInfo& pathInfo, String const& content); + RefPtr<SourceFile> createSourceFileWithSize(const PathInfo& pathInfo, size_t contentSize); + RefPtr<SourceFile> createSourceFileWithString(const PathInfo& pathInfo, const String& contents); + RefPtr<SourceFile> createSourceFileWithBlob(const PathInfo& pathInfo, ISlangBlob* blob); /// Get the humane source location HumaneSourceLoc getHumaneLoc(SourceLoc loc, SourceLocType type = SourceLocType::Nominal); diff --git a/tests/cross-compile/non-uniform-indexing.slang b/tests/cross-compile/non-uniform-indexing.slang index 647742df8..747ca2eef 100644 --- a/tests/cross-compile/non-uniform-indexing.slang +++ b/tests/cross-compile/non-uniform-indexing.slang @@ -1,4 +1,5 @@ //TEST:CROSS_COMPILE:-target spirv-assembly -entry main -stage fragment +//TEST:CROSS_COMPILE:-target spirv-assembly -entry main -stage fragment -verify-debug-serial-ir // Confirm that `NonUniformResourceIndex` translates to SPIR-V as expeted diff --git a/tests/hlsl/simple/allow-uav-conditional.hlsl b/tests/hlsl/simple/allow-uav-conditional.hlsl index ec6fa8590..b9780e97d 100644 --- a/tests/hlsl/simple/allow-uav-conditional.hlsl +++ b/tests/hlsl/simple/allow-uav-conditional.hlsl @@ -1,4 +1,5 @@ //TEST:COMPARE_HLSL:-no-mangle -profile cs_5_0 +//TEST:COMPARE_HLSL:-no-mangle -profile cs_5_0 -verify-debug-serial-ir // Check output for `[allow_uav_conditional]` |
