Preliminary SourceMap support (#2701)

* #include an absolute path didn't work - because paths were taken to always be relative.

* WIP source map.

* Split out handling of RttiTypeFuncs to a map type.

* Make RttiTypeFuncsMap hold default impls.

* Slightly more sophisticated RttiTypeFuncsMap

* Source map decoding.

* Fix tabs.

* Fix asserts due to negative values.

* Use less obscure mechanisms in SourceMap.

7 files changed, 403 insertions, 15 deletions

diff --git a/source/compiler-core/slang-json-native.cpp b/source/compiler-core/slang-json-native.cpp
index c7a90df4c..8b0ec1194 100644
--- a/source/compiler-core/slang-json-native.cpp
+++ b/source/compiler-core/slang-json-native.cpp
@@ -8,6 +8,13 @@
 
 namespace Slang {
 
+/* static */RttiTypeFuncsMap JSONNativeUtil::getTypeFuncsMap()
+{
+    RttiTypeFuncsMap typeMap;
+    typeMap.add(GetRttiInfo<JSONValue>::get(), GetRttiTypeFuncsForZeroPod<JSONValue>::getFuncs());
+    return typeMap;
+}
+
 /* !!!!!!!!!!!!!!!!!!!!!!!!!!!! JSONToNativeConverter !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */
 
 /* static */Index JSONToNativeConverter::_getFieldCount(const StructRttiInfo* structRttiInfo)
@@ -193,10 +200,9 @@ SlangResult JSONToNativeConverter::convert(const JSONValue& in, const RttiInfo*
             const ListRttiInfo* listRttiInfo = static_cast<const ListRttiInfo*>(rttiInfo);
             auto elementType = listRttiInfo->m_elementType;
 
-            SLANG_RETURN_ON_FAIL(RttiUtil::setListCount(elementType, out, arr.getCount()));
+            SLANG_RETURN_ON_FAIL(RttiUtil::setListCount(m_typeMap, elementType, out, arr.getCount()));
 
             // Okay, we need to copy over one by one
-
             Byte* dstEles = list.getBuffer();
             for (Index i = 0; i < count; ++i, dstEles += elementType->m_size)
             {
diff --git a/source/compiler-core/slang-json-native.h b/source/compiler-core/slang-json-native.h
index 4235a99a8..72e339621 100644
--- a/source/compiler-core/slang-json-native.h
+++ b/source/compiler-core/slang-json-native.h
@@ -19,8 +19,9 @@ struct JSONToNativeConverter
     SlangResult convertArrayToStruct(const JSONValue& value, T* in) { return convertArrayToStruct(value, GetRttiInfo<T>::get(), (void*)in); }
     SlangResult convertArrayToStruct(const JSONValue& value, const RttiInfo* rttiInfo, void* out);
 
-    JSONToNativeConverter(JSONContainer* container, DiagnosticSink* sink):
+    JSONToNativeConverter(JSONContainer* container, RttiTypeFuncsMap* typeMap, DiagnosticSink* sink):
         m_container(container),
+        m_typeMap(typeMap),
         m_sink(sink)
     {}
 
@@ -31,6 +32,7 @@ protected:
     SlangResult _structToNative(const ConstArrayView<JSONKeyValue>& pairs, const StructRttiInfo* structRttiInfo, void* out, Index& outFieldCount);
 
     DiagnosticSink* m_sink;
+    RttiTypeFuncsMap* m_typeMap;
     JSONContainer* m_container;
 };
 
@@ -45,9 +47,9 @@ struct NativeToJSONConverter
     template <typename T>
     SlangResult convertStructToArray(T* in, JSONValue& out) { return convertStructToArray(GetRttiInfo<T>::get(), (const void*)in, out); }
 
-
-    NativeToJSONConverter(JSONContainer* container, DiagnosticSink* sink) :
+    NativeToJSONConverter(JSONContainer* container, RttiTypeFuncsMap* typeMap, DiagnosticSink* sink) :
         m_container(container),
+        m_typeMap(typeMap),
         m_sink(sink)
     {}
 
@@ -55,9 +57,14 @@ protected:
     SlangResult _structToJSON(const StructRttiInfo* structRttiInfo, const void* src, List<JSONKeyValue>& outPairs);
 
     DiagnosticSink* m_sink;
+    RttiTypeFuncsMap* m_typeMap;
     JSONContainer* m_container;
 };
 
+struct JSONNativeUtil
+{
+    static RttiTypeFuncsMap getTypeFuncsMap();
+};
 
 } // namespace Slang
 
diff --git a/source/compiler-core/slang-json-rpc-connection.cpp b/source/compiler-core/slang-json-rpc-connection.cpp
index ec45f5b82..6d687d8ca 100644
--- a/source/compiler-core/slang-json-rpc-connection.cpp
+++ b/source/compiler-core/slang-json-rpc-connection.cpp
@@ -10,6 +10,13 @@
 
 namespace Slang {
 
+/// Ctor
+JSONRPCConnection::JSONRPCConnection():
+    m_container(nullptr),
+    m_typeMap(JSONNativeUtil::getTypeFuncsMap())
+{
+}
+
 SlangResult JSONRPCConnection::init(HTTPPacketConnection* connection, CallStyle defaultCallStyle, Process* process)
 {
     m_connection = connection;
@@ -98,8 +105,10 @@ void JSONRPCConnection::disconnect()
 
 SlangResult JSONRPCConnection::sendRPC(const RttiInfo* rttiInfo, const void* data)
 {
+    auto typeMap = JSONNativeUtil::getTypeFuncsMap();
+
     // Convert to JSON
-    NativeToJSONConverter converter(&m_container, &m_diagnosticSink);
+    NativeToJSONConverter converter(&m_container, &typeMap, &m_diagnosticSink);
     JSONValue value;
 
     SLANG_RETURN_ON_FAIL(converter.convert(rttiInfo, data, value));
@@ -132,7 +141,7 @@ SlangResult JSONRPCConnection::toNativeArgsOrSendError(const JSONValue& srcArgs,
     if (dstArgsRttiInfo->m_kind == RttiInfo::Kind::Struct &&
         srcArgs.getKind() == JSONValue::Kind::Array)
     {
-        JSONToNativeConverter converter(&m_container, &m_diagnosticSink);
+        JSONToNativeConverter converter(&m_container, &m_typeMap, &m_diagnosticSink);
         if (SLANG_FAILED(converter.convertArrayToStruct(srcArgs, dstArgsRttiInfo, dstArgs)))
         {
             return sendError(JSONRPC::ErrorCode::InvalidRequest, id);
@@ -149,7 +158,7 @@ SlangResult JSONRPCConnection::toNativeOrSendError(const JSONValue& value, const
 {
     m_diagnosticSink.outputBuffer.Clear();
 
-    JSONToNativeConverter converter(&m_container, &m_diagnosticSink);
+    JSONToNativeConverter converter(&m_container, &m_typeMap, &m_diagnosticSink);
     
     if (SLANG_FAILED(converter.convert(value, info, dst)))
     {
@@ -174,7 +183,7 @@ SlangResult JSONRPCConnection::sendResult(const RttiInfo* rttiInfo, const void*
     JSONResultResponse response;
     response.id = id;
 
-    NativeToJSONConverter converter(&m_container, &m_diagnosticSink);
+    NativeToJSONConverter converter(&m_container, &m_typeMap, &m_diagnosticSink);
     SLANG_RETURN_ON_FAIL(converter.convert(rttiInfo, result, response.result));
 
     // Send the RPC
@@ -194,7 +203,7 @@ SlangResult JSONRPCConnection::sendCall(CallStyle callStyle, const UnownedString
     call.method = method;
 
     // Set up the converter to now convert the args.
-    NativeToJSONConverter converter(&m_container, &m_diagnosticSink);
+    NativeToJSONConverter converter(&m_container, &m_typeMap, &m_diagnosticSink);
     
     // If we have a struct *and* call style is 'array', do special handling
     if (argsRttiInfo->m_kind == RttiInfo::Kind::Struct &&
@@ -263,7 +272,7 @@ SlangResult JSONRPCConnection::getMessage(const RttiInfo* rttiInfo, void* out)
     }
 
     m_diagnosticSink.outputBuffer.Clear();
-    JSONToNativeConverter converter(&m_container, &m_diagnosticSink);
+    JSONToNativeConverter converter(&m_container, &m_typeMap, &m_diagnosticSink);
 
     // Get the RPC response
     JSONResultResponse resultResponse;
@@ -297,7 +306,7 @@ SlangResult JSONRPCConnection::getRPC(const RttiInfo* rttiInfo, void* out)
     }
 
     m_diagnosticSink.outputBuffer.Clear();
-    JSONToNativeConverter converter(&m_container, &m_diagnosticSink);
+    JSONToNativeConverter converter(&m_container, &m_typeMap, &m_diagnosticSink);
 
     // Convert the result in the response
     SLANG_RETURN_ON_FAIL(converter.convert(m_jsonRoot, rttiInfo, out));
diff --git a/source/compiler-core/slang-json-rpc-connection.h b/source/compiler-core/slang-json-rpc-connection.h
index 18749229c..7246fb780 100644
--- a/source/compiler-core/slang-json-rpc-connection.h
+++ b/source/compiler-core/slang-json-rpc-connection.h
@@ -154,7 +154,7 @@ public:
     ~JSONRPCConnection() { disconnect(); }
 
         /// Ctor
-    JSONRPCConnection():m_container(nullptr) {}
+    JSONRPCConnection();
 
 protected:
     CallStyle _getCallStyle(CallStyle callStyle) const { return (callStyle == CallStyle::Default) ? m_defaultCallStyle : callStyle; }
@@ -171,6 +171,8 @@ protected:
 
     CallStyle m_defaultCallStyle = CallStyle::Array;    ///< The default calling style
     
+    RttiTypeFuncsMap m_typeMap;
+
     Int m_terminationTimeOutInMs = 1 * 1000;        ///< Time to wait for termination response. Default is 1 second
 };
 
diff --git a/source/compiler-core/slang-json-rpc.cpp b/source/compiler-core/slang-json-rpc.cpp
index f4101be8c..3474718c7 100644
--- a/source/compiler-core/slang-json-rpc.cpp
+++ b/source/compiler-core/slang-json-rpc.cpp
@@ -126,7 +126,9 @@ static const StructRttiInfo _makeJSONResultResponseResponseRtti()
 
 /* static */SlangResult JSONRPCUtil::convertToNative(JSONContainer* container, const JSONValue& value, DiagnosticSink* sink, const RttiInfo* rttiInfo, void* out)
 {
-    JSONToNativeConverter converter(container, sink);
+    auto typeMap = JSONNativeUtil::getTypeFuncsMap();
+
+    JSONToNativeConverter converter(container, &typeMap, sink);
     SLANG_RETURN_ON_FAIL(converter.convert(value, rttiInfo, out));
     return SLANG_OK;
 }
@@ -136,7 +138,9 @@ static const StructRttiInfo _makeJSONResultResponseResponseRtti()
     SourceManager* sourceManager = sink->getSourceManager();
     JSONContainer container(sourceManager);
 
-    NativeToJSONConverter converter(&container, sink);
+    auto typeMap = JSONNativeUtil::getTypeFuncsMap();
+
+    NativeToJSONConverter converter(&container, &typeMap, sink);
 
     JSONValue value;
     SLANG_RETURN_ON_FAIL(converter.convert(rttiInfo, in, value));
diff --git a/source/compiler-core/slang-source-map.cpp b/source/compiler-core/slang-source-map.cpp
new file mode 100644
index 000000000..6a3083ce0
--- /dev/null
+++ b/source/compiler-core/slang-source-map.cpp
@@ -0,0 +1,242 @@
+#include "slang-source-map.h"
+
+#include "../../slang-com-helper.h"
+
+#include "../core/slang-string-util.h"
+
+#include "slang-json-native.h"
+
+namespace Slang {
+
+static const StructRttiInfo _makeJSONSourceMap_Rtti()
+{
+    JSONSourceMap obj;
+
+    StructRttiBuilder builder(&obj, "SourceMap", nullptr);
+
+    builder.addField("version", &obj.version);
+    builder.addField("file", &obj.file);
+    builder.addField("sourceRoot", &obj.sourceRoot);
+    builder.addField("sources", &obj.sources);
+    builder.addField("sourcesContent", &obj.sourcesContent);
+    builder.addField("names", &obj.names);
+    builder.addField("mappings", &obj.mappings);
+
+    return builder.make();
+}
+/* static */const StructRttiInfo JSONSourceMap::g_rttiInfo = _makeJSONSourceMap_Rtti();
+
+// Encode a 6 bit value to VLQ encoding
+static const char g_vlqEncodeTable[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+
+struct VlqDecodeTable
+{
+    VlqDecodeTable()
+    {
+        ::memset(map, -1, sizeof(map));
+        for (Index i = 0; i < SLANG_COUNT_OF(g_vlqEncodeTable); ++i)
+        {
+            map[g_vlqEncodeTable[i]] = int8_t(i);
+        }
+    }
+        /// Returns a *negative* value if invalid
+    int8_t operator[](char c) const { return (c & ~char(0x7f)) ? -1 : map[c]; }
+
+    int8_t map[128];
+};
+
+static const VlqDecodeTable g_vlqDecodeTable;
+
+/* 
+https://docs.google.com/document/d/1U1RGAehQwRypUTovF1KRlpiOFze0b-_2gc6fAH0KY0k/edit?hl=en_US&pli=1&pli=1#
+The VLQ is a Base64 value, where the most significant bit (the 6th bit) is used as the continuation 
+bit, and the “digits” are encoded into the string least significant first, and where the least significant 
+bit of the first digit is used as the sign bit. */
+
+static SlangResult _decode(UnownedStringSlice& ioEncoded, Index& out)
+{
+    Index v = 0;
+
+    Index shift = 0;
+    const char* cur = ioEncoded.begin();
+    const char* end = ioEncoded.end();
+
+    // Must have some chars
+    if (cur >= end)
+    {
+        return SLANG_FAIL;
+    }
+
+    for (; cur < end; ++cur)
+    {
+        const Index value = g_vlqDecodeTable[*cur];
+        if (value < 0)
+        {
+            return SLANG_FAIL;
+        }
+        
+        v += (value & 0x1f) << shift;
+
+        // If the continuation bit is not set we are done
+        if (( value & 0x20) == 0)
+        {
+            ++cur;
+            break;
+        }
+
+        shift += 5;
+    }
+
+    // Save out the remaining part
+    ioEncoded = UnownedStringSlice(cur, end);
+
+    // Double to make setting lower bit simpler
+    v += v;
+
+    // If it's negative we make positive and set the bottom bit
+    // otherwise we just return with the LSB not set.
+    out = (v < 0) ? (1 - v) : v;
+    return SLANG_OK;
+}
+
+SlangResult SourceMap::decode(JSONContainer* container, const JSONSourceMap& src)
+{
+    m_slicePool.clear();
+
+    m_file = src.file;
+    m_sourceRoot = src.sourceRoot;
+
+    m_lineStarts.clear();
+    m_lineEntries.clear();
+
+    const Count sourcesCount = src.sources.getCount();
+    
+    // These should all be unique, but for simplicity, we build a table
+    m_sources.setCount(sourcesCount);
+    for (Index i = 0; i < sourcesCount; ++i)
+    {
+        m_sources[i] = m_slicePool.add(src.sources[i]);
+    }
+
+    Count sourcesContentCount = src.sourcesContent.getCount();
+    sourcesContentCount = std::min(sourcesContentCount, sourcesCount);
+
+    m_sourcesContent.setCount(sourcesContentCount);
+    for (auto& cur : m_sourcesContent)
+    {
+        cur = StringSlicePool::kNullHandle;
+    }
+
+    for (Index i = 0; i < sourcesContentCount; ++i)
+    {
+        auto value = src.sourcesContent[i];
+
+        if (value.type != JSONValue::Type::Null)
+        {
+            if (value.getKind() == JSONValue::Kind::String)
+            {
+                auto stringValue = container->getString(value);
+                m_sourcesContent[i] = m_slicePool.add(stringValue);
+            }
+        }
+    }
+
+    
+    List<UnownedStringSlice> lines;
+    StringUtil::split(src.mappings, ';', lines);
+    
+    List<UnownedStringSlice> segments;
+
+    // Index into sources 
+    Index sourceFileIndex = 0;
+
+    Index sourceLine = 0;
+    Index sourceColumn = 0;
+    Index nameIndex = 0;
+
+    const Count linesCount = lines.getCount();
+
+    m_lineStarts.setCount(linesCount + 1);
+
+    for (Index generatedLine = 0; generatedLine < linesCount; ++generatedLine)
+    {
+        const auto line = lines[generatedLine];
+
+        m_lineStarts[generatedLine] = m_lineEntries.getCount();
+
+        // If it's empty move to next line
+        if (line.getLength() == 0)
+        {
+            continue;
+        }
+
+        // Split the line into segments
+        segments.clear();
+        StringUtil::split(line, ',', segments);
+
+        Index generatedColumn = 0;
+
+        for (auto segment : segments)
+        {
+            Index colDelta;
+            SLANG_RETURN_ON_FAIL(_decode(segment, colDelta));
+
+            generatedColumn += colDelta;
+            SLANG_ASSERT(generatedColumn >= 0);
+
+            // It can be 4 or 5 parts
+            if (segment.getLength())
+            {
+                /* If present, an zero-based index into the “sources” list. This field is a base 64 VLQ relative to the previous occurrence of this field, unless this is the first occurrence of this field, in which case the whole value is represented.
+                    If present, the zero-based starting line in the original source represented. This field is a base 64 VLQ relative to the previous occurrence of this field, unless this is the first occurrence of this field, in which case the whole value is represented. Always present if there is a source field.
+                    If present, the zero-based starting column of the line in the source represented. This field is a base 64 VLQ relative to the previous occurrence of this field, unless this is the first occurrence of this field, in which case the whole value is represented. Always present if there is a source field.
+                     */
+
+                Index sourceFileDelta;
+                Index sourceLineDelta;
+                Index sourceColumnDelta;
+
+                SLANG_RETURN_ON_FAIL(_decode(segment, sourceFileDelta));
+                SLANG_RETURN_ON_FAIL(_decode(segment, sourceLineDelta));
+                SLANG_RETURN_ON_FAIL(_decode(segment, sourceColumnDelta));
+
+                sourceFileIndex += sourceFileDelta;
+                sourceLine += sourceLineDelta;
+                sourceColumn += sourceColumnDelta;
+
+                SLANG_ASSERT(sourceFileIndex >= 0);
+                SLANG_ASSERT(sourceLine >= 0);
+                SLANG_ASSERT(sourceColumn >= 0);
+
+                // 5 parts
+                if (segment.getLength() > 0)
+                {
+                    /* If present, the zero - based index into the “names” list associated with this segment.This field is a base 64 VLQ relative to the previous occurrence of this field, unless this is the first occurrence of this field, in which case the whole value is represented.
+                    */
+
+                    Index nameDelta;
+                    SLANG_RETURN_ON_FAIL(_decode(segment, nameDelta));
+
+                    nameIndex += nameDelta;
+                    SLANG_ASSERT(nameIndex >= 0);
+                }
+            }
+
+            Entry entry;
+            entry.generatedColumn = generatedColumn;
+            entry.sourceColumn = sourceColumn;
+            entry.sourceLine = sourceLine;
+            entry.sourceFileIndex = sourceFileIndex;
+
+            m_lineEntries.add(entry);
+        }
+    }
+
+    // Mark the end
+    m_lineStarts[linesCount] = m_lineEntries.getCount();
+
+    return SLANG_OK;
+}
+
+
+} // namespace Slang
diff --git a/source/compiler-core/slang-source-map.h b/source/compiler-core/slang-source-map.h
new file mode 100644
index 000000000..0f84878fe
--- /dev/null
+++ b/source/compiler-core/slang-source-map.h
@@ -0,0 +1,118 @@
+#ifndef SLANG_COMPILER_CORE_SOURCE_MAP_H
+#define SLANG_COMPILER_CORE_SOURCE_MAP_H
+
+#include "../../slang.h"
+#include "../../slang-com-helper.h"
+#include "../../slang-com-ptr.h"
+
+#include "../core/slang-string.h"
+#include "../core/slang-list.h"
+#include "../core/slang-rtti-info.h"
+
+#include "../core/slang-string-slice-pool.h"
+
+#include "slang-json-value.h"
+
+namespace Slang {
+
+/* 
+Support for source maps. Source maps provide a standardized mechanism to associate a location in one output file 
+with another.
+
+* [Source Map Proposal](https://docs.google.com/document/d/1U1RGAehQwRypUTovF1KRlpiOFze0b-_2gc6fAH0KY0k/edit?hl=en_US&pli=1&pli=1)
+* [Chrome Source Map post](https://developer.chrome.com/blog/sourcemaps/)
+* [Base64 VLQs in Source Maps](https://www.lucidchart.com/techblog/2019/08/22/decode-encoding-base64-vlqs-source-maps/)
+
+Example...
+
+{
+"version" : 3,
+"file": "out.js",
+"sourceRoot": "",
+"sources": ["foo.js", "bar.js"],
+"sourcesContent": [null, null],
+"names": ["src", "maps", "are", "fun"],
+"mappings": "A,AAAB;;ABCDE;"
+}
+*/
+
+struct JSONSourceMap
+{
+        /// File version (always the first entry in the object) and must be a positive integer.
+    int32_t version = 3;                    
+        /// An optional name of the generated code that this source map is associated with.
+    String file;                            
+        /// An optional source root, useful for relocating source files on a server or removing repeated values in 
+        /// the �sources� entry.  This value is prepended to the individual entries in the �source� field.
+    String sourceRoot;                        
+        /// A list of original sources used by the �mappings� entry.
+    List<UnownedStringSlice> sources;            
+        /// An optional list of source content, useful when the �source� can�t be hosted. The contents are listed in the same order as the sources in line 5. 
+        /// �null� may be used if some original sources should be retrieved by name.
+        /// Because could be a string or nullptr, we use JSONValue to hold value.
+    List<JSONValue> sourcesContent;    
+        /// A list of symbol names used by the �mappings� entry.
+    List<UnownedStringSlice> names;
+        /// A string with the encoded mapping data.
+    UnownedStringSlice mappings;
+
+    static const StructRttiInfo g_rttiInfo;
+};
+
+struct SourceMap
+{
+    struct Entry
+    {
+        // Note! All column/line are zero indexed
+
+        Index generatedColumn;        ///< The generated column
+        Index sourceFileIndex;        ///< The index into the source name/contents
+        Index sourceLine;            ///< The line number in the originating source
+        Index sourceColumn;            ///< The column number in the originating source
+    };
+
+    SlangResult decode(JSONContainer* container, const JSONSourceMap& src);
+
+        /// Get the total number of generated lines
+    Count getGeneratedLineCount() const { return m_lineStarts.getCount() - 1; }
+        /// Get the entries on the line
+    SLANG_FORCE_INLINE ConstArrayView<Entry> getEntriesForLine(Index generatedLine) const;
+    
+        /// Ctor
+    SourceMap():
+        m_slicePool(StringSlicePool::Style::Default)
+    {
+    }
+
+    String m_file;
+    String m_sourceRoot;
+
+    List<StringSlicePool::Handle> m_sources;
+
+        /// Storage for the contents. Can be unset null to indicate not set. 
+    List<StringSlicePool::Handle> m_sourcesContent;
+    
+    List<Index> m_lineStarts;
+    List<Entry> m_lineEntries;
+
+    StringSlicePool m_slicePool;
+};
+
+// -------------------------------------------------------------
+SLANG_FORCE_INLINE ConstArrayView<SourceMap::Entry> SourceMap::getEntriesForLine(Index generatedLine) const
+{
+    const Index start = m_lineStarts[generatedLine];
+    const Index end = m_lineStarts[generatedLine + 1];
+    
+    const auto entries = m_lineEntries.begin();
+
+    SLANG_ASSERT(start >= 0 && start < m_lineEntries.getCount());
+    SLANG_ASSERT(end >= start && end >= 0 && end < m_lineEntries.getCount());
+
+    return ConstArrayView<SourceMap::Entry>(entries + start, end - start);
+}
+
+
+} // namespace Slang
+
+#endif // SLANG_COMPILER_CORE_SOURCE_MAP_H