diff options
| -rw-r--r-- | source/compiler-core/slang-source-map.cpp | 270 | ||||
| -rw-r--r-- | source/compiler-core/slang-source-map.h | 69 | ||||
| -rw-r--r-- | tools/slang-unit-test/unit-test-source-map.cpp | 43 |
3 files changed, 288 insertions, 94 deletions
diff --git a/source/compiler-core/slang-source-map.cpp b/source/compiler-core/slang-source-map.cpp index 6a3083ce0..413c32108 100644 --- a/source/compiler-core/slang-source-map.cpp +++ b/source/compiler-core/slang-source-map.cpp @@ -8,6 +8,54 @@ namespace Slang { +/* +Support for source maps. Source maps provide a standardized mechanism to associate a location in one output file +with another. + +* [Source Map Proposal](https://docs.google.com/document/d/1U1RGAehQwRypUTovF1KRlpiOFze0b-_2gc6fAH0KY0k/edit?hl=en_US&pli=1&pli=1) +* [Chrome Source Map post](https://developer.chrome.com/blog/sourcemaps/) +* [Base64 VLQs in Source Maps](https://www.lucidchart.com/techblog/2019/08/22/decode-encoding-base64-vlqs-source-maps/) + +Example... + +{ +"version" : 3, +"file": "out.js", +"sourceRoot": "", +"sources": ["foo.js", "bar.js"], +"sourcesContent": [null, null], +"names": ["src", "maps", "are", "fun"], +"mappings": "A,AAAB;;ABCDE;" +} +*/ + +namespace { // anonymous + +struct JSONSourceMap +{ + /// File version (always the first entry in the object) and must be a positive integer. + int32_t version = 3; + /// An optional name of the generated code that this source map is associated with. + String file; + /// An optional source root, useful for relocating source files on a server or removing repeated values in + /// the “sources” entry. This value is prepended to the individual entries in the “source” field. + String sourceRoot; + /// A list of original sources used by the “mappings” entry. + List<UnownedStringSlice> sources; + /// An optional list of source content, useful when the “source” can’t be hosted. The contents are listed in the same order as the sources in line 5. + /// “null” may be used if some original sources should be retrieved by name. + /// Because could be a string or nullptr, we use JSONValue to hold value. + List<JSONValue> sourcesContent; + /// A list of symbol names used by the “mappings” entry. + List<UnownedStringSlice> names; + /// A string with the encoded mapping data. + UnownedStringSlice mappings; + + static const StructRttiInfo g_rttiInfo; +}; + +} // anonymous + static const StructRttiInfo _makeJSONSourceMap_Rtti() { JSONSourceMap obj; @@ -16,10 +64,10 @@ static const StructRttiInfo _makeJSONSourceMap_Rtti() builder.addField("version", &obj.version); builder.addField("file", &obj.file); - builder.addField("sourceRoot", &obj.sourceRoot); + builder.addField("sourceRoot", &obj.sourceRoot, StructRttiInfo::Flag::Optional); builder.addField("sources", &obj.sources); - builder.addField("sourcesContent", &obj.sourcesContent); - builder.addField("names", &obj.names); + builder.addField("sourcesContent", &obj.sourcesContent, StructRttiInfo::Flag::Optional); + builder.addField("names", &obj.names, StructRttiInfo::Flag::Optional); builder.addField("mappings", &obj.mappings); return builder.make(); @@ -40,7 +88,7 @@ struct VlqDecodeTable } } /// Returns a *negative* value if invalid - int8_t operator[](char c) const { return (c & ~char(0x7f)) ? -1 : map[c]; } + SLANG_FORCE_INLINE int8_t operator[](char c) const { return (c & ~char(0x7f)) ? -1 : map[c]; } int8_t map[128]; }; @@ -90,25 +138,89 @@ static SlangResult _decode(UnownedStringSlice& ioEncoded, Index& out) // Save out the remaining part ioEncoded = UnownedStringSlice(cur, end); - // Double to make setting lower bit simpler + // Handle negating + out = (v & 1) ? -(v >> 1) : (v >> 1); + return SLANG_OK; +} + +void _encode(Index v, StringBuilder& out) +{ + // Double to free up low bit to hold the sign v += v; - // If it's negative we make positive and set the bottom bit - // otherwise we just return with the LSB not set. - out = (v < 0) ? (1 - v) : v; - return SLANG_OK; + // We want to make v always positive to encode + // we use the last bit to indicate negativity + v = (v < 0) ? (1 - v) : v; + + // We'll use a simple buffer, so as to not have to constantly update he StringBuffer + char dst[8]; + char* cur = dst; + + do + { + // Encode it + const auto nextV = v >> 5; + + // Encode 5 bits + char c = g_vlqEncodeTable[(v & 0x1f)]; + + // See what bits are remaining + v = (v >> 5); + + // Set the continuation bit's if there is more to encode + c |= v ? 0x20 : 0; + + // Save the char + *cur++ = c; + } + while (v); + + out.append(dst, cur); } -SlangResult SourceMap::decode(JSONContainer* container, const JSONSourceMap& src) +void SourceMap::clear() { + String empty; + + m_file = empty; + m_sourceRoot = empty; + + m_sources.clear(); + + m_names.clear(); + + m_sourcesContent.clear(); + + m_lineStarts.setCount(1); + m_lineStarts[0] = 0; + + m_lineEntries.clear(); + + m_slicePool.clear(); +} + +SlangResult SourceMap::decode(JSONContainer* container, JSONValue root, DiagnosticSink* sink) +{ + clear(); + + // Let's try and decode the JSON into native types to make this easier... + + RttiTypeFuncsMap typeMap = JSONNativeUtil::getTypeFuncsMap(); + + // Convert to native + JSONSourceMap src; + { + JSONToNativeConverter converter(container, &typeMap, sink); + + // Convert to the native type + SLANG_RETURN_ON_FAIL(converter.convert(root, GetRttiInfo<JSONSourceMap>::get(), &src)); + } + m_slicePool.clear(); m_file = src.file; m_sourceRoot = src.sourceRoot; - m_lineStarts.clear(); - m_lineEntries.clear(); - const Count sourcesCount = src.sources.getCount(); // These should all be unique, but for simplicity, we build a table @@ -141,7 +253,6 @@ SlangResult SourceMap::decode(JSONContainer* container, const JSONSourceMap& src } } - List<UnownedStringSlice> lines; StringUtil::split(src.mappings, ';', lines); @@ -227,6 +338,7 @@ SlangResult SourceMap::decode(JSONContainer* container, const JSONSourceMap& src entry.sourceColumn = sourceColumn; entry.sourceLine = sourceLine; entry.sourceFileIndex = sourceFileIndex; + entry.nameIndex = nameIndex; m_lineEntries.add(entry); } @@ -238,5 +350,135 @@ SlangResult SourceMap::decode(JSONContainer* container, const JSONSourceMap& src return SLANG_OK; } +SlangResult SourceMap::encode(JSONContainer* container, DiagnosticSink* sink, JSONValue& outValue) +{ + // Convert to native + JSONSourceMap native; + + native.file = m_file; + native.sourceRoot = m_sourceRoot; + + // Copy over the sources + { + const auto count = m_sources.getCount(); + native.sources.setCount(count); + for (Index i = 0; i < count; ++i) + { + native.sources[i] = m_slicePool.getSlice(m_sources[i]); + } + } + + // Copy out the sourcesContent, care is needed around handling null + { + const auto count = m_sourcesContent.getCount(); + native.sourcesContent.setCount(count); + for (Index i = 0; i < count; ++i) + { + const auto srcValue = m_sourcesContent[i]; + + const JSONValue dstValue = (srcValue == StringSlicePool::kNullHandle) ? + native.sourcesContent[i] = JSONValue::makeNull() : + container->createString(m_slicePool.getSlice(srcValue)); + + native.sourcesContent[i] = dstValue; + } + } + + // Copy out the names + { + const auto count = m_names.getCount(); + native.names.setCount(count); + for (Index i = 0; i < count; ++i) + { + native.names[i] = m_slicePool.getSlice(m_names[i]); + } + } + + StringBuilder mappings; + + // Do the encoding! + { + const Count linesCount = getGeneratedLineCount(); + + Index sourceFileIndex = 0; + + Index sourceLine = 0; + Index sourceColumn = 0; + Index nameIndex = 0; + + for (Index i = 0; i < linesCount; ++i) + { + // Add the semicolon to start the line + if (i > 0) + { + mappings.appendChar(';'); + } + + const auto entries = getEntriesForLine(i); + const auto entriesCount = entries.getCount(); + + if (entriesCount == 0) + { + continue; + } + + // We reset the generated column index at the start of each new generated line + Index generatedColumn = 0; + + for (Index j = 0; j < entriesCount; ++j) + { + auto entry = entries[j]; + + if (j > 0) + { + mappings.appendChar(','); + } + + Index generatedDelta = entry.generatedColumn - generatedColumn; + generatedColumn = entry.generatedColumn; + + _encode(generatedDelta, mappings); + + // See if there any other deltas we need to handle + const Index sourceFileDelta = entry.sourceFileIndex - sourceFileIndex; + const Index sourceLineDelta = entry.sourceLine - sourceLine; + const Index sourceColumnDelta = entry.sourceColumn - sourceColumn; + const Index nameIndexDelta = entry.nameIndex - nameIndex; + + if (sourceFileDelta || sourceLineDelta || sourceColumnDelta || nameIndex) + { + // Okay we have to encode all these deltae + _encode(sourceFileDelta, mappings); + _encode(sourceLineDelta, mappings); + _encode(sourceColumnDelta, mappings); + + // Update these values + sourceFileIndex = entry.sourceFileIndex; + sourceLine = entry.sourceLine; + sourceColumn = entry.sourceColumn; + + if (nameIndexDelta) + { + _encode(nameIndexDelta, mappings); + nameIndex = entry.nameIndex; + } + } + } + } + } + + // Set the mappings + native.mappings = mappings.getUnownedSlice(); + + // Write it out + { + RttiTypeFuncsMap typeMap = JSONNativeUtil::getTypeFuncsMap(); + + NativeToJSONConverter converter(container, &typeMap, sink); + SLANG_RETURN_ON_FAIL(converter.convert(GetRttiInfo<JSONSourceMap>::get(), &native, outValue)); + } + + return SLANG_OK; +} } // namespace Slang diff --git a/source/compiler-core/slang-source-map.h b/source/compiler-core/slang-source-map.h index 0f84878fe..7fd64510a 100644 --- a/source/compiler-core/slang-source-map.h +++ b/source/compiler-core/slang-source-map.h @@ -15,73 +15,37 @@ namespace Slang { -/* -Support for source maps. Source maps provide a standardized mechanism to associate a location in one output file -with another. - -* [Source Map Proposal](https://docs.google.com/document/d/1U1RGAehQwRypUTovF1KRlpiOFze0b-_2gc6fAH0KY0k/edit?hl=en_US&pli=1&pli=1) -* [Chrome Source Map post](https://developer.chrome.com/blog/sourcemaps/) -* [Base64 VLQs in Source Maps](https://www.lucidchart.com/techblog/2019/08/22/decode-encoding-base64-vlqs-source-maps/) - -Example... - -{ -"version" : 3, -"file": "out.js", -"sourceRoot": "", -"sources": ["foo.js", "bar.js"], -"sourcesContent": [null, null], -"names": ["src", "maps", "are", "fun"], -"mappings": "A,AAAB;;ABCDE;" -} -*/ - -struct JSONSourceMap -{ - /// File version (always the first entry in the object) and must be a positive integer. - int32_t version = 3; - /// An optional name of the generated code that this source map is associated with. - String file; - /// An optional source root, useful for relocating source files on a server or removing repeated values in - /// the sources entry. This value is prepended to the individual entries in the source field. - String sourceRoot; - /// A list of original sources used by the mappings entry. - List<UnownedStringSlice> sources; - /// An optional list of source content, useful when the source cant be hosted. The contents are listed in the same order as the sources in line 5. - /// null may be used if some original sources should be retrieved by name. - /// Because could be a string or nullptr, we use JSONValue to hold value. - List<JSONValue> sourcesContent; - /// A list of symbol names used by the mappings entry. - List<UnownedStringSlice> names; - /// A string with the encoded mapping data. - UnownedStringSlice mappings; - - static const StructRttiInfo g_rttiInfo; -}; - struct SourceMap { struct Entry { // Note! All column/line are zero indexed - - Index generatedColumn; ///< The generated column - Index sourceFileIndex; ///< The index into the source name/contents - Index sourceLine; ///< The line number in the originating source - Index sourceColumn; ///< The column number in the originating source + Index generatedColumn; ///< The generated column + Index sourceFileIndex; ///< The index into the source name/contents + Index sourceLine; ///< The line number in the originating source + Index sourceColumn; ///< The column number in the originating source + Index nameIndex; ///< Name index }; - SlangResult decode(JSONContainer* container, const JSONSourceMap& src); + /// Decode from root into the source map + SlangResult decode(JSONContainer* container, JSONValue root, DiagnosticSink* sink); + + /// Converts the source map contents into JSON + SlangResult encode(JSONContainer* container, DiagnosticSink* sink, JSONValue& outValue); /// Get the total number of generated lines Count getGeneratedLineCount() const { return m_lineStarts.getCount() - 1; } /// Get the entries on the line SLANG_FORCE_INLINE ConstArrayView<Entry> getEntriesForLine(Index generatedLine) const; + /// Clear the contents of the source map + void clear(); + /// Ctor SourceMap(): m_slicePool(StringSlicePool::Style::Default) { + clear(); } String m_file; @@ -92,6 +56,9 @@ struct SourceMap /// Storage for the contents. Can be unset null to indicate not set. List<StringSlicePool::Handle> m_sourcesContent; + /// The names + List<StringSlicePool::Handle> m_names; + List<Index> m_lineStarts; List<Entry> m_lineEntries; @@ -107,7 +74,7 @@ SLANG_FORCE_INLINE ConstArrayView<SourceMap::Entry> SourceMap::getEntriesForLine const auto entries = m_lineEntries.begin(); SLANG_ASSERT(start >= 0 && start < m_lineEntries.getCount()); - SLANG_ASSERT(end >= start && end >= 0 && end < m_lineEntries.getCount()); + SLANG_ASSERT(end >= start && end >= 0 && end <= m_lineEntries.getCount()); return ConstArrayView<SourceMap::Entry>(entries + start, end - start); } diff --git a/tools/slang-unit-test/unit-test-source-map.cpp b/tools/slang-unit-test/unit-test-source-map.cpp index 1b48980f8..b973a9c62 100644 --- a/tools/slang-unit-test/unit-test-source-map.cpp +++ b/tools/slang-unit-test/unit-test-source-map.cpp @@ -20,7 +20,7 @@ static SlangResult _check() sourceManager.initialize(nullptr, nullptr); DiagnosticSink sink(&sourceManager, nullptr); - const char json[] = R"( + const char jsonSource[] = R"( { "version" : 3, "file" : "out.js", @@ -28,18 +28,16 @@ static SlangResult _check() "sources" : ["foo.js", "bar.js"], "sourcesContent" : [null, null], "names" : ["src", "maps", "are", "fun"], - "mappings" : "A,AAAB;;ABCEG;" + "mappings" : "A,AAAB;;ABCEG;" } )"; RefPtr<JSONContainer> container = new JSONContainer(&sourceManager); - RttiTypeFuncsMap typeMap = JSONNativeUtil::getTypeFuncsMap(); - - JSONValue readValue; + JSONValue rootValue; { // Now need to parse as JSON - String contents(json); + String contents(jsonSource); SourceFile* sourceFile = sourceManager.createSourceFileWithString(PathInfo::makeUnknown(), contents); SourceView* sourceView = sourceManager.createSourceView(sourceFile, nullptr, SourceLoc()); @@ -51,40 +49,27 @@ static SlangResult _check() JSONParser parser; SLANG_RETURN_ON_FAIL(parser.parse(&lexer, sourceView, &builder, &sink)); - readValue = builder.getRootValue(); - } - - // Convert to native - JSONSourceMap readS; - { - JSONToNativeConverter converter(container, &typeMap, &sink); - - // Read it back - SLANG_RETURN_ON_FAIL(converter.convert(readValue, GetRttiInfo<JSONSourceMap>::get(), &readS)); + rootValue = builder.getRootValue(); } + SourceMap sourceMap; + + SLANG_RETURN_ON_FAIL(sourceMap.decode(container, rootValue, &sink)); + // Write it out + String json; { - String json; - - NativeToJSONConverter converter(container, &typeMap, &sink); - - JSONValue value; - SLANG_RETURN_ON_FAIL(converter.convert(GetRttiInfo<JSONSourceMap>::get(), &readS, value)); + JSONValue jsonValue; + SLANG_RETURN_ON_FAIL(sourceMap.encode(container, &sink, jsonValue)); + // Convert into a string JSONWriter writer(JSONWriter::IndentationStyle::Allman); - container->traverseRecursively(value, &writer); + container->traverseRecursively(jsonValue, &writer); json = writer.getBuilder(); } - { - SourceMap sourceMap; - - SLANG_RETURN_ON_FAIL(sourceMap.decode(container, readS)); - } - return SLANG_OK; } |