diff options
| -rw-r--r-- | source/compiler-core/slang-artifact-desc-util.cpp | 11 | ||||
| -rw-r--r-- | source/compiler-core/slang-artifact.h | 2 | ||||
| -rw-r--r-- | source/compiler-core/slang-source-map.cpp | 80 | ||||
| -rw-r--r-- | source/compiler-core/slang-source-map.h | 25 | ||||
| -rwxr-xr-x | source/slang/slang-compiler.h | 1 | ||||
| -rw-r--r-- | source/slang/slang-emit-c-like.h | 1 | ||||
| -rw-r--r-- | source/slang/slang-emit-cpp.cpp | 2 | ||||
| -rw-r--r-- | source/slang/slang-emit-source-writer.cpp | 101 | ||||
| -rw-r--r-- | source/slang/slang-emit-source-writer.h | 21 | ||||
| -rw-r--r-- | source/slang/slang-emit.cpp | 38 | ||||
| -rw-r--r-- | source/slang/slang-options.cpp | 5 | ||||
| -rw-r--r-- | source/slang/slang-serialize-ast.cpp | 4 | ||||
| -rw-r--r-- | source/slang/slang.cpp | 2 |
13 files changed, 282 insertions, 11 deletions
diff --git a/source/compiler-core/slang-artifact-desc-util.cpp b/source/compiler-core/slang-artifact-desc-util.cpp index fff68738c..ee014daf0 100644 --- a/source/compiler-core/slang-artifact-desc-util.cpp +++ b/source/compiler-core/slang-artifact-desc-util.cpp @@ -223,7 +223,8 @@ SLANG_HIERARCHICAL_ENUM(ArtifactKind, SLANG_ARTIFACT_KIND, SLANG_ARTIFACT_KIND_E x(Diagnostics, Metadata) \ x(Miscellaneous, Base) \ x(Log, Miscellaneous) \ - x(Lock, Miscellaneous) + x(Lock, Miscellaneous) \ + x(SourceMap, Base) #define SLANG_ARTIFACT_PAYLOAD_ENTRY(TYPE, PARENT) { Index(ArtifactPayload::TYPE), Index(ArtifactPayload::PARENT), #TYPE }, @@ -550,6 +551,13 @@ static const KindExtension g_cpuKindExts[] = return ArtifactDesc::make(ArtifactKind::Assembly, ArtifactPayload::HostCPU); } + // TODO(JS): Unfortunately map extension is also used from output for linkage from + // Visual Studio. It's used here for source map. + if (slice == toSlice("map")) + { + return ArtifactDesc::make(ArtifactKind::Text, ArtifactPayload::SourceMap); + } + if (slice == toSlice("pdb")) { // Program database @@ -621,6 +629,7 @@ static UnownedStringSlice _getPayloadExtension(ArtifactPayload payload) case Payload::MetalAIR: return toSlice("air"); case Payload::PdbDebugInfo: return toSlice("pdb"); + case Payload::SourceMap: return toSlice("map"); default: break; } diff --git a/source/compiler-core/slang-artifact.h b/source/compiler-core/slang-artifact.h index df0bb21d5..c51ee729a 100644 --- a/source/compiler-core/slang-artifact.h +++ b/source/compiler-core/slang-artifact.h @@ -180,6 +180,8 @@ enum class ArtifactPayload : uint8_t PdbDebugInfo, ///< PDB debug info + SourceMap, ///< SourceMap + CountOf, }; diff --git a/source/compiler-core/slang-source-map.cpp b/source/compiler-core/slang-source-map.cpp index 413c32108..830314008 100644 --- a/source/compiler-core/slang-source-map.cpp +++ b/source/compiler-core/slang-source-map.cpp @@ -199,6 +199,86 @@ void SourceMap::clear() m_slicePool.clear(); } +void SourceMap::advanceToLine(Index nextLineIndex) +{ + const Count currentLineIndex = getGeneratedLineCount(); + + SLANG_ASSERT(nextLineIndex >= currentLineIndex); + + if (nextLineIndex <= currentLineIndex) + { + return; + } + + const auto lastEntryIndex = m_lineEntries.getCount(); + + // For all the new entries they will need to point to the end + m_lineStarts.setCount(nextLineIndex + 1); + + Index* starts = m_lineStarts.getBuffer() + currentLineIndex; + const Count startsCount = nextLineIndex + 1 - currentLineIndex; + + for (Index i = 0; i < startsCount; ++i) + { + starts[i] = lastEntryIndex; + } +} + +void SourceMap::addEntry(const Entry& entry) +{ + m_lineEntries.add(entry); + ++m_lineStarts.getLast(); + + // Check things seem normal... + SLANG_ASSERT(m_lineStarts.getLast() == m_lineEntries.getCount()); +} + +Index SourceMap::getNameIndex(const UnownedStringSlice& slice) +{ + StringSlicePool::Handle handle; + + if (!m_slicePool.findOrAdd(slice, handle)) + { + // We know it can't possibly be used, so must be new (!) + + m_names.add(handle); + return m_names.getCount() - 1; + } + + // Okay, could already be in the list + const auto index = m_names.indexOf(handle); + if (index >= 0) + { + return index; + } + + m_names.add(handle); + return m_names.getCount() - 1; +} + +Index SourceMap::getSourceFileIndex(const UnownedStringSlice& slice) +{ + StringSlicePool::Handle handle; + + if (!m_slicePool.findOrAdd(slice, handle)) + { + // We know it can't possibly be used, so must be new (!) + + m_sources.add(handle); + return m_sources.getCount() - 1; + } + + // Okay, could already be in the list + const auto index = m_sources.indexOf(handle); + if (index >= 0) + { + return index; + } + + m_sources.add(handle); + return m_sources.getCount() - 1; +} + SlangResult SourceMap::decode(JSONContainer* container, JSONValue root, DiagnosticSink* sink) { clear(); diff --git a/source/compiler-core/slang-source-map.h b/source/compiler-core/slang-source-map.h index 7fd64510a..739cf1992 100644 --- a/source/compiler-core/slang-source-map.h +++ b/source/compiler-core/slang-source-map.h @@ -15,10 +15,19 @@ namespace Slang { -struct SourceMap +struct SourceMap : public RefObject { struct Entry { + void init() + { + generatedColumn = 0; + sourceFileIndex = 0; + sourceLine = 0; + sourceColumn = 0; + nameIndex = 0; + } + // Note! All column/line are zero indexed Index generatedColumn; ///< The generated column Index sourceFileIndex; ///< The index into the source name/contents @@ -38,6 +47,20 @@ struct SourceMap /// Get the entries on the line SLANG_FORCE_INLINE ConstArrayView<Entry> getEntriesForLine(Index generatedLine) const; + /// Advance to the specified line index. + /// It is an error to specify a line *before* the current line. It should either be the current + /// output line or a later output line. Interveining lines will be set as empty + void advanceToLine(Index lineIndex); + + /// Add an entry to the current line + void addEntry(const Entry& entry); + + /// Given the slice returns the index + Index getSourceFileIndex(const UnownedStringSlice& slice); + + /// Get the name index + Index getNameIndex(const UnownedStringSlice& slice); + /// Clear the contents of the source map void clear(); diff --git a/source/slang/slang-compiler.h b/source/slang/slang-compiler.h index 784026761..fcf62d826 100755 --- a/source/slang/slang-compiler.h +++ b/source/slang/slang-compiler.h @@ -1767,6 +1767,7 @@ namespace Slang SourceManager* m_sourceManager = nullptr; bool m_obfuscateCode = false; + bool m_generateSourceMap = false; /// Holds any args that are destined for downstream compilers/tools etc DownstreamArgs m_downstreamArgs; diff --git a/source/slang/slang-emit-c-like.h b/source/slang/slang-emit-c-like.h index 1cd2045c7..9426531a8 100644 --- a/source/slang/slang-emit-c-like.h +++ b/source/slang/slang-emit-c-like.h @@ -73,6 +73,7 @@ public: /// combining information from the target and entry point. Profile effectiveProfile = Profile::RawEnum::Unknown; + /// The source writer to use SourceWriter* sourceWriter = nullptr; }; diff --git a/source/slang/slang-emit-cpp.cpp b/source/slang/slang-emit-cpp.cpp index 795ec74b0..2a2ae06c6 100644 --- a/source/slang/slang-emit-cpp.cpp +++ b/source/slang/slang-emit-cpp.cpp @@ -338,7 +338,7 @@ SlangResult CPPSourceEmitter::calcTypeName(IRType* type, CodeGenTarget target, S // TODO(JS): This is a bit of a hack. We don't want to emit the result here, // so we replace the writer, write out the type, grab the contents, and restore the writer - SourceWriter writer(sourceManager, LineDirectiveMode::None); + SourceWriter writer(sourceManager, LineDirectiveMode::None, nullptr); m_writer = &writer; m_writer->emit(prefix); diff --git a/source/slang/slang-emit-source-writer.cpp b/source/slang/slang-emit-source-writer.cpp index 7692bd0ec..b27e0f8b4 100644 --- a/source/slang/slang-emit-source-writer.cpp +++ b/source/slang/slang-emit-source-writer.cpp @@ -16,10 +16,11 @@ namespace Slang { -SourceWriter::SourceWriter(SourceManager* sourceManager, LineDirectiveMode lineDirectiveMode) +SourceWriter::SourceWriter(SourceManager* sourceManager, LineDirectiveMode lineDirectiveMode, SourceMap* sourceMap) { + m_sourceMap = sourceMap; m_lineDirectiveMode = lineDirectiveMode; - this->m_sourceManager = sourceManager; + m_sourceManager = sourceManager; } String SourceWriter::getContentAndClear() @@ -274,7 +275,10 @@ void SourceWriter::advanceToSourceLocationIfValid(const SourceLoc& sourceLocatio void SourceWriter::advanceToSourceLocation(const SourceLoc& sourceLocation) { - if (getLineDirectiveMode() == LineDirectiveMode::None) + // If we don't have any line directives *and* we don't want to output + // source map, we can just ignore + if (getLineDirectiveMode() == LineDirectiveMode::None && + m_sourceMap == nullptr) { // Ignore if we aren't outputting directives return; @@ -328,7 +332,14 @@ void SourceWriter::_flushSourceLocationChange() // advances the location, and outputting text is what // triggers this flush operation. m_needToUpdateSourceLocation = false; + _emitLineDirectiveIfNeeded(m_nextHumaneSourceLocation); + + // If we have a source map update state + if (m_sourceMap) + { + _updateSourceMap(m_nextHumaneSourceLocation); + } } void SourceWriter::_emitLineDirectiveAndUpdateSourceLocation(const HumaneSourceLoc& sourceLocation) @@ -341,6 +352,31 @@ void SourceWriter::_emitLineDirectiveAndUpdateSourceLocation(const HumaneSourceL m_loc = newLoc; } +void SourceWriter::_updateSourceMap(const HumaneSourceLoc& sourceLocation) +{ + // Ignore invalid source locations + if (sourceLocation.line <= 0) + return; + + // We need to work out the current column in the generated (ie being written) output + Index generatedLineIndex, generatedColumnIndex; + _calcLocation(generatedLineIndex, generatedColumnIndex); + + // Advance to the current output line + m_sourceMap->advanceToLine(generatedLineIndex); + + // Add the entry into the map, mapping back to the original source + SourceMap::Entry entry; + entry.init(); + + entry.sourceFileIndex = m_sourceMap->getSourceFileIndex(sourceLocation.pathInfo.getName().getUnownedSlice()); + entry.sourceLine = sourceLocation.line - 1; + entry.sourceColumn = sourceLocation.column - 1; + entry.generatedColumn = generatedColumnIndex; + + m_sourceMap->addEntry(entry); +} + void SourceWriter::_emitLineDirectiveIfNeeded(const HumaneSourceLoc& sourceLocation) { if (m_supressLineDirective) @@ -478,4 +514,63 @@ void SourceWriter::_emitLineDirective(const HumaneSourceLoc& sourceLocation) emitRawText("\n"); } +void SourceWriter::_calcLocation(Index& outLineIndex, Index& outColumnIndex) +{ + // If we are at the end, then we are done. + if (m_currentOutputOffset == m_builder.getLength()) + { + outLineIndex = m_currentLineIndex; + outColumnIndex = m_currentColumnIndex; + return; + } + + const char* cur = m_builder.getBuffer() + m_currentOutputOffset; + const char* end = m_builder.end(); + + const char* start = cur; + + while (cur < end) + { + // Reset start + start = cur; + + // Look for the end of the line + while (*cur != '\n' && *cur != '\r' && cur < end) + { + cur++; + } + + // If we are not at the total end then we must have hit a \n or \r + if (cur < end) + { + const auto c = *cur++; + + ++m_currentLineIndex; + // Reset the column + m_currentColumnIndex = 0; + + // Check the next char to see if it's part of a CR/LF combination + if (cur < end) + { + const auto d = *cur; + // If it is combination skip the next byte + cur += ((c ^ d) == ('\r' ^ '\n')); + } + } + } + + // Fix up the current index. + // TODO(JS): + // NOTE! This isn't strictly correct because it assumes one byte is a *column* which isn't actually the case with utf8 + // encoding... + m_currentColumnIndex += Index(cur - start); + + // Set the current offset is the end + m_currentOutputOffset = m_builder.getLength(); + + // Output the values + outLineIndex = m_currentLineIndex; + outColumnIndex = m_currentColumnIndex; +} + } // namespace Slang diff --git a/source/slang/slang-emit-source-writer.h b/source/slang/slang-emit-source-writer.h index 294cfec18..c47b5f5fa 100644 --- a/source/slang/slang-emit-source-writer.h +++ b/source/slang/slang-emit-source-writer.h @@ -5,6 +5,8 @@ #include "../core/slang-basic.h" #include "slang-compiler.h" +#include "../compiler-core/slang-source-map.h" + namespace Slang { @@ -72,8 +74,11 @@ public: /// Get the source manager user SourceManager* getSourceManager() const { return m_sourceManager; } + /// Get the associated source map. If source map tracking is not required, can return nullptr. + SourceMap* getSourceMap() const { return m_sourceMap; } + /// Ctor - SourceWriter(SourceManager* sourceManager, LineDirectiveMode lineDirectiveMode); + SourceWriter(SourceManager* sourceManager, LineDirectiveMode lineDirectiveMode, SourceMap* sourceMap); protected: void _emitTextSpan(char const* textBegin, char const* textEnd); @@ -86,10 +91,15 @@ protected: void _emitLineDirectiveIfNeeded(const HumaneSourceLoc& sourceLocation); + void _updateSourceMap(const HumaneSourceLoc& sourceLocation); + // Emit a `#line` directive to the output. // Doesn't update state of source-location tracking. void _emitLineDirective(const HumaneSourceLoc& sourceLocation); + /// Calculate the current location in the ouput + void _calcLocation(Index& outLineIndex, Index& outColumnIndex); + // The string of code we've built so far. // TODO(JS): We could store the text in chunks, and then only sew together into one buffer // when we are done. Doing so would not require copies/reallocs until the full buffer has been @@ -103,6 +113,13 @@ protected: SourceLoc m_nextSourceLoc; HumaneSourceLoc m_nextHumaneSourceLocation; + // Used to determine the current location in the output for outputting the source map + // This is separate from m_loc, because m_loc doesn't appear to track the line/column directly + // in the output stream - for example when #line emits a "raw" emit takes place. + Count m_currentOutputOffset = 0; + Index m_currentLineIndex = 0; + Index m_currentColumnIndex = 0; + bool m_needToUpdateSourceLocation = false; bool m_supressLineDirective = false; @@ -123,6 +140,8 @@ protected: Dictionary<String, int> m_mapGLSLSourcePathToID; int m_glslSourceIDCount = 0; + RefPtr<SourceMap> m_sourceMap; + LineDirectiveMode m_lineDirectiveMode; }; diff --git a/source/slang/slang-emit.cpp b/source/slang/slang-emit.cpp index a25fae5ae..335e95c9e 100644 --- a/source/slang/slang-emit.cpp +++ b/source/slang/slang-emit.cpp @@ -935,7 +935,15 @@ SlangResult CodeGenContext::emitEntryPointsSourceFromIR(ComPtr<IArtifact>& outAr lineDirectiveMode = LineDirectiveMode::GLSL; } - SourceWriter sourceWriter(sourceManager, lineDirectiveMode ); + RefPtr<SourceMap> sourceMap; + + // If SourceMap is enabled, we create one and associate it with the sourceWriter + if (targetRequest->getLinkage()->m_generateSourceMap) + { + sourceMap = new SourceMap; + } + + SourceWriter sourceWriter(sourceManager, lineDirectiveMode, sourceMap ); CLikeSourceEmitter::Desc desc; @@ -1096,6 +1104,34 @@ SlangResult CodeGenContext::emitEntryPointsSourceFromIR(ComPtr<IArtifact>& outAr artifact->addAssociated(metadata); } + if (sourceMap) + { + SourceManager sourceMapSourceManager; + sourceMapSourceManager.initialize(nullptr, nullptr); + + // Create a sink + DiagnosticSink sourceMapSink(&sourceMapSourceManager, nullptr); + + // Turn into JSON + RefPtr<JSONContainer> jsonContainer(new JSONContainer(&sourceMapSourceManager)); + + JSONValue jsonValue; + SLANG_RETURN_ON_FAIL(sourceMap->encode(jsonContainer, &sourceMapSink, jsonValue)); + + // Okay now convert this into a text file and then a blob + + // Convert into a string + JSONWriter writer(JSONWriter::IndentationStyle::KNR); + jsonContainer->traverseRecursively(jsonValue, &writer); + + auto sourceMapBlob = StringBlob::moveCreate(writer.getBuilder()); + + auto sourceMapArtifact = ArtifactUtil::createArtifact(ArtifactDesc::make(ArtifactKind::Text, ArtifactPayload::SourceMap, ArtifactStyle::None)); + sourceMapArtifact->addRepresentationUnknown(sourceMapBlob); + + artifact->addAssociated(sourceMapArtifact); + } + outArtifact.swap(artifact); return SLANG_OK; } diff --git a/source/slang/slang-options.cpp b/source/slang/slang-options.cpp index 944162acc..f6932b9e0 100644 --- a/source/slang/slang-options.cpp +++ b/source/slang/slang-options.cpp @@ -669,6 +669,7 @@ struct OptionsParser " -save-stdlib-bin-source <filename>: Same as -save-stdlib but output\n" " the data as a C array.\n" " -track-liveness: Enable liveness tracking. Places SLANG_LIVE_START, and SLANG_LIVE_END in output source to indicate value liveness.\n" + " -source-map: Enables outputting of a source map. Note this is *distinct* from line-directive-mode.\n" "\n" "Deprecated options (allowed but ignored; may be removed in future):\n" "\n" @@ -1176,6 +1177,10 @@ struct OptionsParser { getCurrentTarget()->targetFlags |= SLANG_TARGET_FLAG_PARAMETER_BLOCKS_USE_REGISTER_SPACES; } + else if(argValue == "-source-map") + { + requestImpl->getLinkage()->m_generateSourceMap = true; + } else if (argValue == "-ir-compression") { CommandLineArg name; diff --git a/source/slang/slang-serialize-ast.cpp b/source/slang/slang-serialize-ast.cpp index 04b5f71b1..b51775710 100644 --- a/source/slang/slang-serialize-ast.cpp +++ b/source/slang/slang-serialize-ast.cpp @@ -129,14 +129,14 @@ struct ASTFieldAccess String readDump; { - SourceWriter sourceWriter(sourceManager, LineDirectiveMode::None); + SourceWriter sourceWriter(sourceManager, LineDirectiveMode::None, nullptr); ASTDumpUtil::dump(reader.getPointer(SerialIndex(1)).dynamicCast<NodeBase>(), ASTDumpUtil::Style::Hierachical, dumpFlags, &sourceWriter); readDump = sourceWriter.getContentAndClear(); } String origDump; { - SourceWriter sourceWriter(sourceManager, LineDirectiveMode::None); + SourceWriter sourceWriter(sourceManager, LineDirectiveMode::None, nullptr); ASTDumpUtil::dump(node, ASTDumpUtil::Style::Hierachical, dumpFlags, &sourceWriter); origDump = sourceWriter.getContentAndClear(); } diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp index da8fcdcd6..14b59d156 100644 --- a/source/slang/slang.cpp +++ b/source/slang/slang.cpp @@ -2277,7 +2277,7 @@ void FrontEndCompileRequest::parseTranslationUnit( if (shouldDumpAST) { StringBuilder buf; - SourceWriter writer(linkage->getSourceManager(), LineDirectiveMode::None); + SourceWriter writer(linkage->getSourceManager(), LineDirectiveMode::None, nullptr); ASTDumpUtil::dump(translationUnit->getModuleDecl(), ASTDumpUtil::Style::Flat, 0, &writer); |