diff options
| -rw-r--r-- | build/visual-studio/compiler-core/compiler-core.vcxproj | 2 | ||||
| -rw-r--r-- | build/visual-studio/compiler-core/compiler-core.vcxproj.filters | 6 | ||||
| -rw-r--r-- | source/compiler-core/slang-json-source-map-util.cpp | 464 | ||||
| -rw-r--r-- | source/compiler-core/slang-json-source-map-util.h | 21 | ||||
| -rw-r--r-- | source/compiler-core/slang-source-loc.cpp | 23 | ||||
| -rw-r--r-- | source/compiler-core/slang-source-loc.h | 12 | ||||
| -rw-r--r-- | source/compiler-core/slang-source-map.cpp | 463 | ||||
| -rw-r--r-- | source/compiler-core/slang-source-map.h | 24 | ||||
| -rw-r--r-- | source/slang/slang-compiler.cpp | 4 | ||||
| -rw-r--r-- | source/slang/slang-emit.cpp | 4 | ||||
| -rw-r--r-- | source/slang/slang-ir-obfuscate-loc.cpp | 17 | ||||
| -rw-r--r-- | tests/serialization/obfuscated-check-loc.slang | 36 | ||||
| -rw-r--r-- | tests/serialization/obfuscated-check-loc.slang.expected | 6 | ||||
| -rw-r--r-- | tools/slang-unit-test/unit-test-source-map.cpp | 7 |
14 files changed, 627 insertions, 462 deletions
diff --git a/build/visual-studio/compiler-core/compiler-core.vcxproj b/build/visual-studio/compiler-core/compiler-core.vcxproj index 4dfbdd085..5aad57b06 100644 --- a/build/visual-studio/compiler-core/compiler-core.vcxproj +++ b/build/visual-studio/compiler-core/compiler-core.vcxproj @@ -295,6 +295,7 @@ <ClInclude Include="..\..\..\source\compiler-core\slang-json-parser.h" />
<ClInclude Include="..\..\..\source\compiler-core\slang-json-rpc-connection.h" />
<ClInclude Include="..\..\..\source\compiler-core\slang-json-rpc.h" />
+ <ClInclude Include="..\..\..\source\compiler-core\slang-json-source-map-util.h" />
<ClInclude Include="..\..\..\source\compiler-core\slang-json-value.h" />
<ClInclude Include="..\..\..\source\compiler-core\slang-language-server-protocol.h" />
<ClInclude Include="..\..\..\source\compiler-core\slang-lexer-diagnostic-defs.h" />
@@ -340,6 +341,7 @@ <ClCompile Include="..\..\..\source\compiler-core\slang-json-parser.cpp" />
<ClCompile Include="..\..\..\source\compiler-core\slang-json-rpc-connection.cpp" />
<ClCompile Include="..\..\..\source\compiler-core\slang-json-rpc.cpp" />
+ <ClCompile Include="..\..\..\source\compiler-core\slang-json-source-map-util.cpp" />
<ClCompile Include="..\..\..\source\compiler-core\slang-json-value.cpp" />
<ClCompile Include="..\..\..\source\compiler-core\slang-language-server-protocol.cpp" />
<ClCompile Include="..\..\..\source\compiler-core\slang-lexer.cpp" />
diff --git a/build/visual-studio/compiler-core/compiler-core.vcxproj.filters b/build/visual-studio/compiler-core/compiler-core.vcxproj.filters index 3b51335c8..c5e2fcfce 100644 --- a/build/visual-studio/compiler-core/compiler-core.vcxproj.filters +++ b/build/visual-studio/compiler-core/compiler-core.vcxproj.filters @@ -99,6 +99,9 @@ <ClInclude Include="..\..\..\source\compiler-core\slang-json-rpc.h">
<Filter>Header Files</Filter>
</ClInclude>
+ <ClInclude Include="..\..\..\source\compiler-core\slang-json-source-map-util.h">
+ <Filter>Header Files</Filter>
+ </ClInclude>
<ClInclude Include="..\..\..\source\compiler-core\slang-json-value.h">
<Filter>Header Files</Filter>
</ClInclude>
@@ -230,6 +233,9 @@ <ClCompile Include="..\..\..\source\compiler-core\slang-json-rpc.cpp">
<Filter>Source Files</Filter>
</ClCompile>
+ <ClCompile Include="..\..\..\source\compiler-core\slang-json-source-map-util.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
<ClCompile Include="..\..\..\source\compiler-core\slang-json-value.cpp">
<Filter>Source Files</Filter>
</ClCompile>
diff --git a/source/compiler-core/slang-json-source-map-util.cpp b/source/compiler-core/slang-json-source-map-util.cpp new file mode 100644 index 000000000..f7c1711f3 --- /dev/null +++ b/source/compiler-core/slang-json-source-map-util.cpp @@ -0,0 +1,464 @@ +#include "slang-json-source-map-util.h" + +#include "../../slang-com-helper.h" + +#include "../core/slang-string-util.h" + +#include "slang-json-native.h" + +namespace Slang { + +/* +Support for source maps. Source maps provide a standardized mechanism to associate a location in one output file +with another. + +* [Source Map Proposal](https://docs.google.com/document/d/1U1RGAehQwRypUTovF1KRlpiOFze0b-_2gc6fAH0KY0k/edit?hl=en_US&pli=1&pli=1) +* [Chrome Source Map post](https://developer.chrome.com/blog/sourcemaps/) +* [Base64 VLQs in Source Maps](https://www.lucidchart.com/techblog/2019/08/22/decode-encoding-base64-vlqs-source-maps/) + +Example... + +{ +"version" : 3, +"file": "out.js", +"sourceRoot": "", +"sources": ["foo.js", "bar.js"], +"sourcesContent": [null, null], +"names": ["src", "maps", "are", "fun"], +"mappings": "A,AAAB;;ABCDE;" +} +*/ + +namespace { // anonymous + +struct JSONSourceMap +{ + /// File version (always the first entry in the object) and must be a positive integer. + int32_t version = 3; + /// An optional name of the generated code that this source map is associated with. + String file; + /// An optional source root, useful for relocating source files on a server or removing repeated values in + /// the “sources” entry. This value is prepended to the individual entries in the “source” field. + String sourceRoot; + /// A list of original sources used by the “mappings” entry. + List<UnownedStringSlice> sources; + /// An optional list of source content, useful when the “source” can’t be hosted. The contents are listed in the same order as the sources in line 5. + /// “null” may be used if some original sources should be retrieved by name. + /// Because could be a string or nullptr, we use JSONValue to hold value. + List<JSONValue> sourcesContent; + /// A list of symbol names used by the “mappings” entry. + List<UnownedStringSlice> names; + /// A string with the encoded mapping data. + UnownedStringSlice mappings; + + static const StructRttiInfo g_rttiInfo; +}; + +} // anonymous + +static const StructRttiInfo _makeJSONSourceMap_Rtti() +{ + JSONSourceMap obj; + + StructRttiBuilder builder(&obj, "SourceMap", nullptr); + + builder.addField("version", &obj.version); + builder.addField("file", &obj.file); + builder.addField("sourceRoot", &obj.sourceRoot, StructRttiInfo::Flag::Optional); + builder.addField("sources", &obj.sources); + builder.addField("sourcesContent", &obj.sourcesContent, StructRttiInfo::Flag::Optional); + builder.addField("names", &obj.names, StructRttiInfo::Flag::Optional); + builder.addField("mappings", &obj.mappings); + + return builder.make(); +} +/* static */const StructRttiInfo JSONSourceMap::g_rttiInfo = _makeJSONSourceMap_Rtti(); + +// Encode a 6 bit value to VLQ encoding +static const char g_vlqEncodeTable[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +struct VlqDecodeTable +{ + VlqDecodeTable() + { + ::memset(map, -1, sizeof(map)); + for (Index i = 0; i < SLANG_COUNT_OF(g_vlqEncodeTable); ++i) + { + map[g_vlqEncodeTable[i]] = int8_t(i); + } + } + /// Returns a *negative* value if invalid + SLANG_FORCE_INLINE int8_t operator[](char c) const { return (c & ~char(0x7f)) ? -1 : map[c]; } + + int8_t map[128]; +}; + +static const VlqDecodeTable g_vlqDecodeTable; + +/* +https://docs.google.com/document/d/1U1RGAehQwRypUTovF1KRlpiOFze0b-_2gc6fAH0KY0k/edit?hl=en_US&pli=1&pli=1# +The VLQ is a Base64 value, where the most significant bit (the 6th bit) is used as the continuation +bit, and the “digits” are encoded into the string least significant first, and where the least significant +bit of the first digit is used as the sign bit. */ + +static SlangResult _decode(UnownedStringSlice& ioEncoded, Index& out) +{ + Index v = 0; + + Index shift = 0; + const char* cur = ioEncoded.begin(); + const char* end = ioEncoded.end(); + + // Must have some chars + if (cur >= end) + { + return SLANG_FAIL; + } + + for (; cur < end; ++cur) + { + const Index value = g_vlqDecodeTable[*cur]; + if (value < 0) + { + return SLANG_FAIL; + } + + v += (value & 0x1f) << shift; + + // If the continuation bit is not set we are done + if (( value & 0x20) == 0) + { + ++cur; + break; + } + + shift += 5; + } + + // Save out the remaining part + ioEncoded = UnownedStringSlice(cur, end); + + // Handle negating + out = (v & 1) ? -(v >> 1) : (v >> 1); + return SLANG_OK; +} + +void _encode(Index v, StringBuilder& out) +{ + // Double to free up low bit to hold the sign + v += v; + + // We want to make v always positive to encode + // we use the last bit to indicate negativity + v = (v < 0) ? (1 - v) : v; + + // We'll use a simple buffer, so as to not have to constantly update he StringBuffer + char dst[8]; + char* cur = dst; + + do + { + // Encode it + const auto nextV = v >> 5; + + // Encode 5 bits + char c = g_vlqEncodeTable[(v & 0x1f)]; + + // See what bits are remaining + v = (v >> 5); + + // Set the continuation bit's if there is more to encode + c |= v ? 0x20 : 0; + + // Save the char + *cur++ = c; + } + while (v); + + out.append(dst, cur); +} + +/* static */SlangResult JSONSourceMapUtil::decode(JSONContainer* container, JSONValue root, DiagnosticSink* sink, RefPtr<SourceMap>& out) +{ + RefPtr<SourceMap> sourceMap; + + // Let's try and decode the JSON into native types to make this easier... + RttiTypeFuncsMap typeMap = JSONNativeUtil::getTypeFuncsMap(); + + // Convert to native + JSONSourceMap native; + { + JSONToNativeConverter converter(container, &typeMap, sink); + + // Convert to the native type + SLANG_RETURN_ON_FAIL(converter.convert(root, GetRttiInfo<JSONSourceMap>::get(), &native)); + } + + sourceMap->m_file = native.file; + sourceMap->m_sourceRoot = native.sourceRoot; + + const Count sourcesCount = native.sources.getCount(); + + // These should all be unique, but for simplicity, we build a table + sourceMap->m_sources.setCount(sourcesCount); + for (Index i = 0; i < sourcesCount; ++i) + { + sourceMap->m_sources[i] = sourceMap->m_slicePool.add(native.sources[i]); + } + + Count sourcesContentCount = native.sourcesContent.getCount(); + sourcesContentCount = std::min(sourcesContentCount, sourcesCount); + + sourceMap->m_sourcesContent.setCount(sourcesContentCount); + for (auto& cur : sourceMap->m_sourcesContent) + { + cur = StringSlicePool::kNullHandle; + } + + for (Index i = 0; i < sourcesContentCount; ++i) + { + auto value = native.sourcesContent[i]; + + if (value.type != JSONValue::Type::Null) + { + if (value.getKind() == JSONValue::Kind::String) + { + auto stringValue = container->getString(value); + sourceMap->m_sourcesContent[i] = sourceMap->m_slicePool.add(stringValue); + } + } + } + + List<UnownedStringSlice> lines; + StringUtil::split(native.mappings, ';', lines); + + List<UnownedStringSlice> segments; + + // Index into sources + Index sourceFileIndex = 0; + + Index sourceLine = 0; + Index sourceColumn = 0; + Index nameIndex = 0; + + const Count linesCount = lines.getCount(); + + sourceMap->m_lineStarts.setCount(linesCount + 1); + + for (Index generatedLine = 0; generatedLine < linesCount; ++generatedLine) + { + const auto line = lines[generatedLine]; + + sourceMap->m_lineStarts[generatedLine] = sourceMap->m_lineEntries.getCount(); + + // If it's empty move to next line + if (line.getLength() == 0) + { + continue; + } + + // Split the line into segments + segments.clear(); + StringUtil::split(line, ',', segments); + + Index generatedColumn = 0; + + for (auto segment : segments) + { + Index colDelta; + SLANG_RETURN_ON_FAIL(_decode(segment, colDelta)); + + generatedColumn += colDelta; + SLANG_ASSERT(generatedColumn >= 0); + + // It can be 4 or 5 parts + if (segment.getLength()) + { + /* If present, an zero-based index into the "sources" list. This field is a base 64 VLQ relative to the previous occurrence of this field, unless this is the first occurrence of this field, in which case the whole value is represented. + If present, the zero-based starting line in the original source represented. This field is a base 64 VLQ relative to the previous occurrence of this field, unless this is the first occurrence of this field, in which case the whole value is represented. Always present if there is a source field. + If present, the zero-based starting column of the line in the source represented. This field is a base 64 VLQ relative to the previous occurrence of this field, unless this is the first occurrence of this field, in which case the whole value is represented. Always present if there is a source field. + */ + + Index sourceFileDelta; + Index sourceLineDelta; + Index sourceColumnDelta; + + SLANG_RETURN_ON_FAIL(_decode(segment, sourceFileDelta)); + SLANG_RETURN_ON_FAIL(_decode(segment, sourceLineDelta)); + SLANG_RETURN_ON_FAIL(_decode(segment, sourceColumnDelta)); + + sourceFileIndex += sourceFileDelta; + sourceLine += sourceLineDelta; + sourceColumn += sourceColumnDelta; + + SLANG_ASSERT(sourceFileIndex >= 0); + SLANG_ASSERT(sourceLine >= 0); + SLANG_ASSERT(sourceColumn >= 0); + + // 5 parts + if (segment.getLength() > 0) + { + /* If present, the zero - based index into the "names" list associated with this segment. + This field is a base 64 VLQ relative to the previous occurrence of this field, unless this is the first occurrence + of this field, in which case the whole value is represented. + */ + + Index nameDelta; + SLANG_RETURN_ON_FAIL(_decode(segment, nameDelta)); + + nameIndex += nameDelta; + SLANG_ASSERT(nameIndex >= 0); + } + } + + SourceMap::Entry entry; + entry.generatedColumn = generatedColumn; + entry.sourceColumn = sourceColumn; + entry.sourceLine = sourceLine; + entry.sourceFileIndex = sourceFileIndex; + entry.nameIndex = nameIndex; + + sourceMap->m_lineEntries.add(entry); + } + } + + // Mark the end + sourceMap->m_lineStarts[linesCount] = sourceMap->m_lineEntries.getCount(); + + out.swapWith(sourceMap); + + return SLANG_OK; +} + +SlangResult JSONSourceMapUtil::encode(SourceMap* sourceMap, JSONContainer* container, DiagnosticSink* sink, JSONValue& outValue) +{ + // Convert to native + JSONSourceMap native; + + native.file = sourceMap->m_file; + native.sourceRoot = sourceMap->m_sourceRoot; + + // Copy over the sources + { + const auto count = sourceMap->m_sources.getCount(); + native.sources.setCount(count); + for (Index i = 0; i < count; ++i) + { + native.sources[i] = sourceMap->m_slicePool.getSlice(sourceMap->m_sources[i]); + } + } + + // Copy out the sourcesContent, care is needed around handling null + { + const auto count = sourceMap->m_sourcesContent.getCount(); + native.sourcesContent.setCount(count); + for (Index i = 0; i < count; ++i) + { + const auto srcValue = sourceMap->m_sourcesContent[i]; + + const JSONValue dstValue = (srcValue == StringSlicePool::kNullHandle) ? + native.sourcesContent[i] = JSONValue::makeNull() : + container->createString(sourceMap->m_slicePool.getSlice(srcValue)); + + native.sourcesContent[i] = dstValue; + } + } + + // Copy out the names + { + const auto count = sourceMap->m_names.getCount(); + native.names.setCount(count); + for (Index i = 0; i < count; ++i) + { + native.names[i] = sourceMap->m_slicePool.getSlice(sourceMap->m_names[i]); + } + } + + StringBuilder mappings; + + // Do the encoding! + { + const Count linesCount = sourceMap->getGeneratedLineCount(); + + Index sourceFileIndex = 0; + + Index sourceLine = 0; + Index sourceColumn = 0; + Index nameIndex = 0; + + for (Index i = 0; i < linesCount; ++i) + { + // Add the semicolon to start the line + if (i > 0) + { + mappings.appendChar(';'); + } + + const auto entries = sourceMap->getEntriesForLine(i); + const auto entriesCount = entries.getCount(); + + if (entriesCount == 0) + { + continue; + } + + // We reset the generated column index at the start of each new generated line + Index generatedColumn = 0; + + for (Index j = 0; j < entriesCount; ++j) + { + auto entry = entries[j]; + + if (j > 0) + { + mappings.appendChar(','); + } + + Index generatedDelta = entry.generatedColumn - generatedColumn; + generatedColumn = entry.generatedColumn; + + _encode(generatedDelta, mappings); + + // See if there any other deltas we need to handle + const Index sourceFileDelta = entry.sourceFileIndex - sourceFileIndex; + const Index sourceLineDelta = entry.sourceLine - sourceLine; + const Index sourceColumnDelta = entry.sourceColumn - sourceColumn; + const Index nameIndexDelta = entry.nameIndex - nameIndex; + + if (sourceFileDelta || sourceLineDelta || sourceColumnDelta || nameIndex) + { + // Okay we have to encode all these deltae + _encode(sourceFileDelta, mappings); + _encode(sourceLineDelta, mappings); + _encode(sourceColumnDelta, mappings); + + // Update these values + sourceFileIndex = entry.sourceFileIndex; + sourceLine = entry.sourceLine; + sourceColumn = entry.sourceColumn; + + if (nameIndexDelta) + { + _encode(nameIndexDelta, mappings); + nameIndex = entry.nameIndex; + } + } + } + } + } + + // Set the mappings + native.mappings = mappings.getUnownedSlice(); + + // Write it out + { + RttiTypeFuncsMap typeMap = JSONNativeUtil::getTypeFuncsMap(); + + NativeToJSONConverter converter(container, &typeMap, sink); + SLANG_RETURN_ON_FAIL(converter.convert(GetRttiInfo<JSONSourceMap>::get(), &native, outValue)); + } + + return SLANG_OK; +} + +} // namespace Slang diff --git a/source/compiler-core/slang-json-source-map-util.h b/source/compiler-core/slang-json-source-map-util.h new file mode 100644 index 000000000..51b11b6cd --- /dev/null +++ b/source/compiler-core/slang-json-source-map-util.h @@ -0,0 +1,21 @@ +#ifndef SLANG_COMPILER_CORE_JSON_SOURCE_MAP_UTIL_H +#define SLANG_COMPILER_CORE_JSON_SOURCE_MAP_UTIL_H + +#include "slang-source-map.h" + +#include "slang-json-value.h" + +namespace Slang { + +struct JSONSourceMapUtil +{ + /// Decode from root into the source map + static SlangResult decode(JSONContainer* container, JSONValue root, DiagnosticSink* sink, RefPtr<SourceMap>& out); + + /// Converts the source map contents into JSON + static SlangResult encode(SourceMap* sourceMap, JSONContainer* container, DiagnosticSink* sink, JSONValue& outValue); +}; + +} // namespace Slang + +#endif // SLANG_COMPILER_CORE_JSON_SOURCE_MAP_UTIL_H diff --git a/source/compiler-core/slang-source-loc.cpp b/source/compiler-core/slang-source-loc.cpp index 1314d8066..951f7feec 100644 --- a/source/compiler-core/slang-source-loc.cpp +++ b/source/compiler-core/slang-source-loc.cpp @@ -191,6 +191,29 @@ void SourceView::addDefaultLineDirective(SourceLoc directiveLoc) HandleSourceLoc SourceView::getHandleLoc(SourceLoc loc, SourceLocType type) { + auto obfuscatedSourceMap = getSourceFile()->getObfuscatedSourceMap(); + if (obfuscatedSourceMap) + { + const Index col = getRange().getOffset(loc); + + const Index entryIndex = obfuscatedSourceMap->findEntry(0, col); + if (entryIndex >= 0) + { + const auto& entry = obfuscatedSourceMap->getEntryByIndex(entryIndex); + + // Generate the HandleSourceLoc + + HandleSourceLoc handleLoc; + handleLoc.line = entry.sourceLine + 1; + handleLoc.column = entry.sourceColumn + 1; + + auto& managerPool = getSourceManager()->getStringSlicePool(); + + handleLoc.pathHandle = managerPool.add(obfuscatedSourceMap->getSourceFileName(entry.sourceFileIndex)); + return handleLoc; + } + } + const int offset = m_range.getOffset(loc); // We need the line index from the original source file diff --git a/source/compiler-core/slang-source-loc.h b/source/compiler-core/slang-source-loc.h index 15c7c0a53..4103c9d6d 100644 --- a/source/compiler-core/slang-source-loc.h +++ b/source/compiler-core/slang-source-loc.h @@ -6,6 +6,8 @@ #include "../core/slang-memory-arena.h" #include "../core/slang-string-slice-pool.h" +#include "slang-source-map.h" + #include "../../slang-com-ptr.h" #include "../../slang.h" @@ -245,6 +247,12 @@ public: /// Get the source manager this was created on SourceManager* getSourceManager() const { return m_sourceManager; } + /// If set this "file" only exists as a way to obfuscate locations + /// The mapping between the two is specified in the specified source map + SourceMap* getObfuscatedSourceMap() const { return m_obfuscatedSourceMap; } + /// Set the obfuscated source map + void setObfuscatedSourceMap(SourceMap* sourceMap) { m_obfuscatedSourceMap = sourceMap; } + /// Ctor SourceFile(SourceManager* sourceManager, const PathInfo& pathInfo, size_t contentSize); /// Dtor @@ -263,6 +271,10 @@ public: // we will cache the starting offset of each line break in // the input file: List<uint32_t> m_lineBreakOffsets; + + // If set then this file isn't a regular source file, but provides obfuscation. + // The mapping of that obfuscation can be found via the obfuscated source map + RefPtr<SourceMap> m_obfuscatedSourceMap; }; enum class SourceLocType diff --git a/source/compiler-core/slang-source-map.cpp b/source/compiler-core/slang-source-map.cpp index 1dd0fe8a5..4ea50eb3f 100644 --- a/source/compiler-core/slang-source-map.cpp +++ b/source/compiler-core/slang-source-map.cpp @@ -1,183 +1,7 @@ #include "slang-source-map.h" -#include "../../slang-com-helper.h" - -#include "../core/slang-string-util.h" - -#include "slang-json-native.h" - namespace Slang { -/* -Support for source maps. Source maps provide a standardized mechanism to associate a location in one output file -with another. - -* [Source Map Proposal](https://docs.google.com/document/d/1U1RGAehQwRypUTovF1KRlpiOFze0b-_2gc6fAH0KY0k/edit?hl=en_US&pli=1&pli=1) -* [Chrome Source Map post](https://developer.chrome.com/blog/sourcemaps/) -* [Base64 VLQs in Source Maps](https://www.lucidchart.com/techblog/2019/08/22/decode-encoding-base64-vlqs-source-maps/) - -Example... - -{ -"version" : 3, -"file": "out.js", -"sourceRoot": "", -"sources": ["foo.js", "bar.js"], -"sourcesContent": [null, null], -"names": ["src", "maps", "are", "fun"], -"mappings": "A,AAAB;;ABCDE;" -} -*/ - -namespace { // anonymous - -struct JSONSourceMap -{ - /// File version (always the first entry in the object) and must be a positive integer. - int32_t version = 3; - /// An optional name of the generated code that this source map is associated with. - String file; - /// An optional source root, useful for relocating source files on a server or removing repeated values in - /// the “sources” entry. This value is prepended to the individual entries in the “source” field. - String sourceRoot; - /// A list of original sources used by the “mappings” entry. - List<UnownedStringSlice> sources; - /// An optional list of source content, useful when the “source” can’t be hosted. The contents are listed in the same order as the sources in line 5. - /// “null” may be used if some original sources should be retrieved by name. - /// Because could be a string or nullptr, we use JSONValue to hold value. - List<JSONValue> sourcesContent; - /// A list of symbol names used by the “mappings” entry. - List<UnownedStringSlice> names; - /// A string with the encoded mapping data. - UnownedStringSlice mappings; - - static const StructRttiInfo g_rttiInfo; -}; - -} // anonymous - -static const StructRttiInfo _makeJSONSourceMap_Rtti() -{ - JSONSourceMap obj; - - StructRttiBuilder builder(&obj, "SourceMap", nullptr); - - builder.addField("version", &obj.version); - builder.addField("file", &obj.file); - builder.addField("sourceRoot", &obj.sourceRoot, StructRttiInfo::Flag::Optional); - builder.addField("sources", &obj.sources); - builder.addField("sourcesContent", &obj.sourcesContent, StructRttiInfo::Flag::Optional); - builder.addField("names", &obj.names, StructRttiInfo::Flag::Optional); - builder.addField("mappings", &obj.mappings); - - return builder.make(); -} -/* static */const StructRttiInfo JSONSourceMap::g_rttiInfo = _makeJSONSourceMap_Rtti(); - -// Encode a 6 bit value to VLQ encoding -static const char g_vlqEncodeTable[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - -struct VlqDecodeTable -{ - VlqDecodeTable() - { - ::memset(map, -1, sizeof(map)); - for (Index i = 0; i < SLANG_COUNT_OF(g_vlqEncodeTable); ++i) - { - map[g_vlqEncodeTable[i]] = int8_t(i); - } - } - /// Returns a *negative* value if invalid - SLANG_FORCE_INLINE int8_t operator[](char c) const { return (c & ~char(0x7f)) ? -1 : map[c]; } - - int8_t map[128]; -}; - -static const VlqDecodeTable g_vlqDecodeTable; - -/* -https://docs.google.com/document/d/1U1RGAehQwRypUTovF1KRlpiOFze0b-_2gc6fAH0KY0k/edit?hl=en_US&pli=1&pli=1# -The VLQ is a Base64 value, where the most significant bit (the 6th bit) is used as the continuation -bit, and the “digits” are encoded into the string least significant first, and where the least significant -bit of the first digit is used as the sign bit. */ - -static SlangResult _decode(UnownedStringSlice& ioEncoded, Index& out) -{ - Index v = 0; - - Index shift = 0; - const char* cur = ioEncoded.begin(); - const char* end = ioEncoded.end(); - - // Must have some chars - if (cur >= end) - { - return SLANG_FAIL; - } - - for (; cur < end; ++cur) - { - const Index value = g_vlqDecodeTable[*cur]; - if (value < 0) - { - return SLANG_FAIL; - } - - v += (value & 0x1f) << shift; - - // If the continuation bit is not set we are done - if (( value & 0x20) == 0) - { - ++cur; - break; - } - - shift += 5; - } - - // Save out the remaining part - ioEncoded = UnownedStringSlice(cur, end); - - // Handle negating - out = (v & 1) ? -(v >> 1) : (v >> 1); - return SLANG_OK; -} - -void _encode(Index v, StringBuilder& out) -{ - // Double to free up low bit to hold the sign - v += v; - - // We want to make v always positive to encode - // we use the last bit to indicate negativity - v = (v < 0) ? (1 - v) : v; - - // We'll use a simple buffer, so as to not have to constantly update he StringBuffer - char dst[8]; - char* cur = dst; - - do - { - // Encode it - const auto nextV = v >> 5; - - // Encode 5 bits - char c = g_vlqEncodeTable[(v & 0x1f)]; - - // See what bits are remaining - v = (v >> 5); - - // Set the continuation bit's if there is more to encode - c |= v ? 0x20 : 0; - - // Save the char - *cur++ = c; - } - while (v); - - out.append(dst, cur); -} - void SourceMap::clear() { String empty; @@ -245,6 +69,11 @@ Index SourceMap::getNameIndex(const UnownedStringSlice& slice) return m_names.getCount() - 1; } +UnownedStringSlice SourceMap::getSourceFileName(Index sourceFileIndex) const +{ + return m_slicePool.getSlice(m_sources[sourceFileIndex]); +} + Index SourceMap::getSourceFileIndex(const UnownedStringSlice& slice) { StringSlicePool::Handle handle; @@ -268,285 +97,41 @@ Index SourceMap::getSourceFileIndex(const UnownedStringSlice& slice) return m_sources.getCount() - 1; } -SlangResult SourceMap::decode(JSONContainer* container, JSONValue root, DiagnosticSink* sink) +Index SourceMap::findEntry(Index lineIndex, Index colIndex) const { - clear(); + auto entries = getEntriesForLine(lineIndex); - // Let's try and decode the JSON into native types to make this easier... - RttiTypeFuncsMap typeMap = JSONNativeUtil::getTypeFuncsMap(); + Index closestDist = 0x7fffffff; + Index bestIndex = -1; - // Convert to native - JSONSourceMap native; + const Count count = entries.getCount(); + for (Index i = 0; i < count; ++i) { - JSONToNativeConverter converter(container, &typeMap, sink); + const Entry& entry = entries[i]; - // Convert to the native type - SLANG_RETURN_ON_FAIL(converter.convert(root, GetRttiInfo<JSONSourceMap>::get(), &native)); - } - - m_file = native.file; - m_sourceRoot = native.sourceRoot; - - const Count sourcesCount = native.sources.getCount(); - - // These should all be unique, but for simplicity, we build a table - m_sources.setCount(sourcesCount); - for (Index i = 0; i < sourcesCount; ++i) - { - m_sources[i] = m_slicePool.add(native.sources[i]); - } - - Count sourcesContentCount = native.sourcesContent.getCount(); - sourcesContentCount = std::min(sourcesContentCount, sourcesCount); - - m_sourcesContent.setCount(sourcesContentCount); - for (auto& cur : m_sourcesContent) - { - cur = StringSlicePool::kNullHandle; - } - - for (Index i = 0; i < sourcesContentCount; ++i) - { - auto value = native.sourcesContent[i]; - - if (value.type != JSONValue::Type::Null) - { - if (value.getKind() == JSONValue::Kind::String) - { - auto stringValue = container->getString(value); - m_sourcesContent[i] = m_slicePool.add(stringValue); - } - } - } - - List<UnownedStringSlice> lines; - StringUtil::split(native.mappings, ';', lines); - - List<UnownedStringSlice> segments; - - // Index into sources - Index sourceFileIndex = 0; - - Index sourceLine = 0; - Index sourceColumn = 0; - Index nameIndex = 0; - - const Count linesCount = lines.getCount(); - - m_lineStarts.setCount(linesCount + 1); - - for (Index generatedLine = 0; generatedLine < linesCount; ++generatedLine) - { - const auto line = lines[generatedLine]; - - m_lineStarts[generatedLine] = m_lineEntries.getCount(); - - // If it's empty move to next line - if (line.getLength() == 0) - { - continue; - } - - // Split the line into segments - segments.clear(); - StringUtil::split(line, ',', segments); - - Index generatedColumn = 0; - - for (auto segment : segments) + // We found an exact match + if (entry.generatedColumn == colIndex) { - Index colDelta; - SLANG_RETURN_ON_FAIL(_decode(segment, colDelta)); - - generatedColumn += colDelta; - SLANG_ASSERT(generatedColumn >= 0); - - // It can be 4 or 5 parts - if (segment.getLength()) - { - /* If present, an zero-based index into the "sources" list. This field is a base 64 VLQ relative to the previous occurrence of this field, unless this is the first occurrence of this field, in which case the whole value is represented. - If present, the zero-based starting line in the original source represented. This field is a base 64 VLQ relative to the previous occurrence of this field, unless this is the first occurrence of this field, in which case the whole value is represented. Always present if there is a source field. - If present, the zero-based starting column of the line in the source represented. This field is a base 64 VLQ relative to the previous occurrence of this field, unless this is the first occurrence of this field, in which case the whole value is represented. Always present if there is a source field. - */ - - Index sourceFileDelta; - Index sourceLineDelta; - Index sourceColumnDelta; - - SLANG_RETURN_ON_FAIL(_decode(segment, sourceFileDelta)); - SLANG_RETURN_ON_FAIL(_decode(segment, sourceLineDelta)); - SLANG_RETURN_ON_FAIL(_decode(segment, sourceColumnDelta)); - - sourceFileIndex += sourceFileDelta; - sourceLine += sourceLineDelta; - sourceColumn += sourceColumnDelta; - - SLANG_ASSERT(sourceFileIndex >= 0); - SLANG_ASSERT(sourceLine >= 0); - SLANG_ASSERT(sourceColumn >= 0); - - // 5 parts - if (segment.getLength() > 0) - { - /* If present, the zero - based index into the "names" list associated with this segment. - This field is a base 64 VLQ relative to the previous occurrence of this field, unless this is the first occurrence - of this field, in which case the whole value is represented. - */ - - Index nameDelta; - SLANG_RETURN_ON_FAIL(_decode(segment, nameDelta)); - - nameIndex += nameDelta; - SLANG_ASSERT(nameIndex >= 0); - } - } - - Entry entry; - entry.generatedColumn = generatedColumn; - entry.sourceColumn = sourceColumn; - entry.sourceLine = sourceLine; - entry.sourceFileIndex = sourceFileIndex; - entry.nameIndex = nameIndex; - - m_lineEntries.add(entry); - } - } - - // Mark the end - m_lineStarts[linesCount] = m_lineEntries.getCount(); - - return SLANG_OK; -} - -SlangResult SourceMap::encode(JSONContainer* container, DiagnosticSink* sink, JSONValue& outValue) -{ - // Convert to native - JSONSourceMap native; - - native.file = m_file; - native.sourceRoot = m_sourceRoot; - - // Copy over the sources - { - const auto count = m_sources.getCount(); - native.sources.setCount(count); - for (Index i = 0; i < count; ++i) - { - native.sources[i] = m_slicePool.getSlice(m_sources[i]); - } - } - - // Copy out the sourcesContent, care is needed around handling null - { - const auto count = m_sourcesContent.getCount(); - native.sourcesContent.setCount(count); - for (Index i = 0; i < count; ++i) - { - const auto srcValue = m_sourcesContent[i]; - - const JSONValue dstValue = (srcValue == StringSlicePool::kNullHandle) ? - native.sourcesContent[i] = JSONValue::makeNull() : - container->createString(m_slicePool.getSlice(srcValue)); - - native.sourcesContent[i] = dstValue; + bestIndex = i; + break; } - } - // Copy out the names - { - const auto count = m_names.getCount(); - native.names.setCount(count); - for (Index i = 0; i < count; ++i) - { - native.names[i] = m_slicePool.getSlice(m_names[i]); - } - } + Index dist = entry.generatedColumn - colIndex; + dist = (dist < 0) ? -dist : dist; - StringBuilder mappings; - - // Do the encoding! - { - const Count linesCount = getGeneratedLineCount(); - - Index sourceFileIndex = 0; - - Index sourceLine = 0; - Index sourceColumn = 0; - Index nameIndex = 0; - - for (Index i = 0; i < linesCount; ++i) + if (dist < closestDist) { - // Add the semicolon to start the line - if (i > 0) - { - mappings.appendChar(';'); - } - - const auto entries = getEntriesForLine(i); - const auto entriesCount = entries.getCount(); - - if (entriesCount == 0) - { - continue; - } - - // We reset the generated column index at the start of each new generated line - Index generatedColumn = 0; - - for (Index j = 0; j < entriesCount; ++j) - { - auto entry = entries[j]; - - if (j > 0) - { - mappings.appendChar(','); - } - - Index generatedDelta = entry.generatedColumn - generatedColumn; - generatedColumn = entry.generatedColumn; - - _encode(generatedDelta, mappings); - - // See if there any other deltas we need to handle - const Index sourceFileDelta = entry.sourceFileIndex - sourceFileIndex; - const Index sourceLineDelta = entry.sourceLine - sourceLine; - const Index sourceColumnDelta = entry.sourceColumn - sourceColumn; - const Index nameIndexDelta = entry.nameIndex - nameIndex; - - if (sourceFileDelta || sourceLineDelta || sourceColumnDelta || nameIndex) - { - // Okay we have to encode all these deltae - _encode(sourceFileDelta, mappings); - _encode(sourceLineDelta, mappings); - _encode(sourceColumnDelta, mappings); - - // Update these values - sourceFileIndex = entry.sourceFileIndex; - sourceLine = entry.sourceLine; - sourceColumn = entry.sourceColumn; - - if (nameIndexDelta) - { - _encode(nameIndexDelta, mappings); - nameIndex = entry.nameIndex; - } - } - } + closestDist = dist; + bestIndex = i; } } - // Set the mappings - native.mappings = mappings.getUnownedSlice(); - - // Write it out + if (bestIndex < 0) { - RttiTypeFuncsMap typeMap = JSONNativeUtil::getTypeFuncsMap(); - - NativeToJSONConverter converter(container, &typeMap, sink); - SLANG_RETURN_ON_FAIL(converter.convert(GetRttiInfo<JSONSourceMap>::get(), &native, outValue)); + return bestIndex; } - return SLANG_OK; + return m_lineStarts[lineIndex] + bestIndex; } } // namespace Slang diff --git a/source/compiler-core/slang-source-map.h b/source/compiler-core/slang-source-map.h index accd2473f..1ccc87e90 100644 --- a/source/compiler-core/slang-source-map.h +++ b/source/compiler-core/slang-source-map.h @@ -2,21 +2,17 @@ #define SLANG_COMPILER_CORE_SOURCE_MAP_H #include "../../slang.h" -#include "../../slang-com-helper.h" -#include "../../slang-com-ptr.h" #include "../core/slang-string.h" #include "../core/slang-list.h" -#include "../core/slang-rtti-info.h" #include "../core/slang-string-slice-pool.h" -#include "slang-json-value.h" - namespace Slang { -struct SourceMap : public RefObject +class SourceMap : public RefObject { +public: struct Entry { void init() @@ -36,12 +32,6 @@ struct SourceMap : public RefObject Index nameIndex; ///< Name index }; - /// Decode from root into the source map - SlangResult decode(JSONContainer* container, JSONValue root, DiagnosticSink* sink); - - /// Converts the source map contents into JSON - SlangResult encode(JSONContainer* container, DiagnosticSink* sink, JSONValue& outValue); - /// Get the total number of generated lines Count getGeneratedLineCount() const { return m_lineStarts.getCount(); } /// Get the entries on the line @@ -61,6 +51,16 @@ struct SourceMap : public RefObject /// Get the name index Index getNameIndex(const UnownedStringSlice& slice); + /// Given a row and col index, find the closest entry + /// NOTE! Zero indexed line and column. + Index findEntry(Index lineIndex, Index colIndex) const; + + /// Given an entry index return the entry + const Entry& getEntryByIndex(Index i) const {return m_lineEntries[i]; } + + /// Given the sourceFileIndex return the name + UnownedStringSlice getSourceFileName(Index sourceFileIndex) const; + /// Clear the contents of the source map void clear(); diff --git a/source/slang/slang-compiler.cpp b/source/slang/slang-compiler.cpp index ff38bbbde..839a4e67c 100644 --- a/source/slang/slang-compiler.cpp +++ b/source/slang/slang-compiler.cpp @@ -14,6 +14,8 @@ #include "../compiler-core/slang-lexer.h" +#include "../compiler-core/slang-json-source-map-util.h" + // Artifact #include "../compiler-core/slang-artifact-desc-util.h" #include "../compiler-core/slang-artifact-representation-impl.h" @@ -1907,7 +1909,7 @@ namespace Slang JSONValue jsonValue; - SLANG_RETURN_ON_FAIL(sourceMap->encode(jsonContainer, sink, jsonValue)); + SLANG_RETURN_ON_FAIL(JSONSourceMapUtil::encode(sourceMap, jsonContainer, sink, jsonValue)); // Convert into a string JSONWriter writer(JSONWriter::IndentationStyle::Allman); diff --git a/source/slang/slang-emit.cpp b/source/slang/slang-emit.cpp index fe72efcc7..6a0f46e0a 100644 --- a/source/slang/slang-emit.cpp +++ b/source/slang/slang-emit.cpp @@ -82,6 +82,8 @@ #include "../compiler-core/slang-artifact-impl.h" #include "../compiler-core/slang-artifact-associated-impl.h" +#include "../compiler-core/slang-json-source-map-util.h" + #include <assert.h> Slang::String get_slang_cpp_host_prelude(); @@ -1148,7 +1150,7 @@ SlangResult CodeGenContext::emitEntryPointsSourceFromIR(ComPtr<IArtifact>& outAr RefPtr<JSONContainer> jsonContainer(new JSONContainer(&sourceMapSourceManager)); JSONValue jsonValue; - SLANG_RETURN_ON_FAIL(sourceMap->encode(jsonContainer, &sourceMapSink, jsonValue)); + SLANG_RETURN_ON_FAIL(JSONSourceMapUtil::encode(sourceMap, jsonContainer, &sourceMapSink, jsonValue)); // Okay now convert this into a text file and then a blob diff --git a/source/slang/slang-ir-obfuscate-loc.cpp b/source/slang/slang-ir-obfuscate-loc.cpp index 79e855633..3ff2f3713 100644 --- a/source/slang/slang-ir-obfuscate-loc.cpp +++ b/source/slang/slang-ir-obfuscate-loc.cpp @@ -153,6 +153,8 @@ SlangResult obfuscateModuleLocs(IRModule* module, SourceManager* sourceManager) // Create the view we are going to use from the obfusctated "file". SourceView* obfuscatedView = sourceManager->createSourceView(obfuscatedFile, nullptr, SourceLoc()); + const auto obfuscatedRange = obfuscatedView->getRange(); + // Okay now we want to produce a map from these locs to a new source location { // Create a "bag" and put all of the indices in it. @@ -160,13 +162,11 @@ SlangResult obfuscateModuleLocs(IRModule* module, SourceManager* sourceManager) bag.setCount(uniqueLocCount); - const SourceLoc baseLoc = obfuscatedView->getRange().begin; - { SourceLoc* dst = bag.getBuffer(); for (Index i = 0; i < uniqueLocCount; ++i) { - dst[i] = baseLoc + i; + dst[i] = obfuscatedRange.begin + i; } } @@ -227,7 +227,6 @@ SlangResult obfuscateModuleLocs(IRModule* module, SourceManager* sourceManager) { const auto& pair = locPairs[i]; - // First find the view if (curView == nullptr || !curView->getRange().contains(pair.originalLoc)) @@ -282,9 +281,11 @@ SlangResult obfuscateModuleLocs(IRModule* module, SourceManager* sourceManager) entry.sourceFileIndex = sourceFileIndex; - // i is the generated column - entry.generatedColumn = i; + // Calculate the column offset, from the pair obfuscated loc + entry.generatedColumn = Index(obfuscatedRange.getOffset(pair.obfuscatedLoc)); + // We need to subtract 1, because handleLoc locations are 1 indexed, but SourceMap + // entry is 0 indexed. entry.sourceColumn = handleLoc.column - 1; entry.sourceLine = handleLoc.line - 1; @@ -293,6 +294,10 @@ SlangResult obfuscateModuleLocs(IRModule* module, SourceManager* sourceManager) } } + // Associate the sourceMap with the obfuscated file + obfuscatedFile->setObfuscatedSourceMap(sourceMap); + + // Set the obfuscated map onto the module module->setObfuscatedSourceMap(sourceMap); return SLANG_OK; diff --git a/tests/serialization/obfuscated-check-loc.slang b/tests/serialization/obfuscated-check-loc.slang new file mode 100644 index 000000000..8575a2650 --- /dev/null +++ b/tests/serialization/obfuscated-check-loc.slang @@ -0,0 +1,36 @@ +//TEST:SIMPLE:-target cpp -stage compute -entry computeMain -obfuscate -source-map -line-directive-mode none + +//TEST_INPUT:ubuffer(data=[0 0 0 0], stride=4):out,name=outputBuffer +RWStructuredBuffer<float> outputBuffer; + +#if 0 +[DllExport] +String getString() +{ + return "Hello!"; +} +#endif + +int silly(int a) +{ + int t = 0; + [ForceUnroll(10)] + while ( a > 0) + { + t = t + t + a; + } + return t; +} + +[numthreads(1, 1, 1)] +void computeMain(uint3 dispatchThreadID : SV_DispatchThreadID) +{ + int x = int(dispatchThreadID.x); + + //String v = getString(); + int v = silly(x); + + outputBuffer[x] = v; + +} + diff --git a/tests/serialization/obfuscated-check-loc.slang.expected b/tests/serialization/obfuscated-check-loc.slang.expected new file mode 100644 index 000000000..0743b437a --- /dev/null +++ b/tests/serialization/obfuscated-check-loc.slang.expected @@ -0,0 +1,6 @@ +result code = -1 +standard error = { +tests/serialization/obfuscated-check-loc.slang(18): error 40020: loop does not terminate within the limited number of iterations, unrolling is aborted. +} +standard output = { +} diff --git a/tools/slang-unit-test/unit-test-source-map.cpp b/tools/slang-unit-test/unit-test-source-map.cpp index b973a9c62..bf97096e9 100644 --- a/tools/slang-unit-test/unit-test-source-map.cpp +++ b/tools/slang-unit-test/unit-test-source-map.cpp @@ -7,6 +7,7 @@ #include "../../source/compiler-core/slang-json-native.h" #include "../../source/compiler-core/slang-source-map.h" +#include "../../source/compiler-core/slang-json-source-map-util.h" #include "../../source/core/slang-rtti-info.h" @@ -52,16 +53,16 @@ static SlangResult _check() rootValue = builder.getRootValue(); } - SourceMap sourceMap; + RefPtr<SourceMap> sourceMap; - SLANG_RETURN_ON_FAIL(sourceMap.decode(container, rootValue, &sink)); + SLANG_RETURN_ON_FAIL(JSONSourceMapUtil::decode(container, rootValue, &sink, sourceMap)); // Write it out String json; { JSONValue jsonValue; - SLANG_RETURN_ON_FAIL(sourceMap.encode(container, &sink, jsonValue)); + SLANG_RETURN_ON_FAIL(JSONSourceMapUtil::encode(sourceMap, container, &sink, jsonValue)); // Convert into a string JSONWriter writer(JSONWriter::IndentationStyle::Allman); |