Support for producing SourceMap on emit (#2707)

* #include an absolute path didn't work - because paths were taken to always be relative.

* WIP source map.

* Split out handling of RttiTypeFuncs to a map type.

* Make RttiTypeFuncsMap hold default impls.

* Slightly more sophisticated RttiTypeFuncsMap

* Source map decoding.

* Fix tabs.

* Fix asserts due to negative values.

* Use less obscure mechanisms in SourceMap.

* Source map decoding.
Simplifying SourceMap usage.

* First attempt at ouputting a source map as part of emit.

* Added support for -source-map option. SourceMap is added to the artifact.

13 files changed, 282 insertions, 11 deletions

diff --git a/source/compiler-core/slang-artifact-desc-util.cpp b/source/compiler-core/slang-artifact-desc-util.cpp
index fff68738c..ee014daf0 100644
--- a/source/compiler-core/slang-artifact-desc-util.cpp
+++ b/source/compiler-core/slang-artifact-desc-util.cpp
@@ -223,7 +223,8 @@ SLANG_HIERARCHICAL_ENUM(ArtifactKind, SLANG_ARTIFACT_KIND, SLANG_ARTIFACT_KIND_E
             x(Diagnostics, Metadata) \
         x(Miscellaneous, Base) \
             x(Log, Miscellaneous) \
-            x(Lock, Miscellaneous)
+            x(Lock, Miscellaneous) \
+        x(SourceMap, Base)
 
 #define SLANG_ARTIFACT_PAYLOAD_ENTRY(TYPE, PARENT) { Index(ArtifactPayload::TYPE), Index(ArtifactPayload::PARENT), #TYPE },
 
@@ -550,6 +551,13 @@ static const KindExtension g_cpuKindExts[] =
         return ArtifactDesc::make(ArtifactKind::Assembly, ArtifactPayload::HostCPU);
     }
 
+    // TODO(JS): Unfortunately map extension is also used from output for linkage from 
+    // Visual Studio. It's used here for source map.
+    if (slice == toSlice("map"))
+    {
+        return ArtifactDesc::make(ArtifactKind::Text, ArtifactPayload::SourceMap);
+    }
+
     if (slice == toSlice("pdb"))
     {
         // Program database
@@ -621,6 +629,7 @@ static UnownedStringSlice _getPayloadExtension(ArtifactPayload payload)
         case Payload::MetalAIR:     return toSlice("air");
 
         case Payload::PdbDebugInfo: return toSlice("pdb");
+        case Payload::SourceMap:    return toSlice("map");
 
         default: break;
     }
diff --git a/source/compiler-core/slang-artifact.h b/source/compiler-core/slang-artifact.h
index df0bb21d5..c51ee729a 100644
--- a/source/compiler-core/slang-artifact.h
+++ b/source/compiler-core/slang-artifact.h
@@ -180,6 +180,8 @@ enum class ArtifactPayload : uint8_t
 
     PdbDebugInfo,   ///< PDB debug info
 
+    SourceMap,      ///< SourceMap
+
     CountOf,
 };
 
diff --git a/source/compiler-core/slang-source-map.cpp b/source/compiler-core/slang-source-map.cpp
index 413c32108..830314008 100644
--- a/source/compiler-core/slang-source-map.cpp
+++ b/source/compiler-core/slang-source-map.cpp
@@ -199,6 +199,86 @@ void SourceMap::clear()
     m_slicePool.clear();
 }
 
+void SourceMap::advanceToLine(Index nextLineIndex)
+{
+    const Count currentLineIndex = getGeneratedLineCount();
+
+    SLANG_ASSERT(nextLineIndex >= currentLineIndex);
+    
+    if (nextLineIndex <= currentLineIndex)
+    {
+        return;
+    }
+
+    const auto lastEntryIndex = m_lineEntries.getCount();
+
+    // For all the new entries they will need to point to the end 
+    m_lineStarts.setCount(nextLineIndex + 1);
+
+    Index* starts = m_lineStarts.getBuffer() + currentLineIndex;
+    const Count startsCount = nextLineIndex + 1 - currentLineIndex;
+
+    for (Index i = 0; i < startsCount; ++i)
+    {
+        starts[i] = lastEntryIndex;
+    }
+}
+
+void SourceMap::addEntry(const Entry& entry)
+{
+    m_lineEntries.add(entry);
+    ++m_lineStarts.getLast();
+
+    // Check things seem normal...
+    SLANG_ASSERT(m_lineStarts.getLast() == m_lineEntries.getCount());
+}
+
+Index SourceMap::getNameIndex(const UnownedStringSlice& slice)
+{
+    StringSlicePool::Handle handle;
+
+    if (!m_slicePool.findOrAdd(slice, handle))
+    {
+        // We know it can't possibly be used, so must be new (!)
+
+        m_names.add(handle);
+        return m_names.getCount() - 1;
+    }
+
+    // Okay, could already be in the list
+    const auto index = m_names.indexOf(handle);
+    if (index >= 0)
+    {
+        return index;
+    }
+
+    m_names.add(handle);
+    return m_names.getCount() - 1;
+}
+
+Index SourceMap::getSourceFileIndex(const UnownedStringSlice& slice)
+{
+    StringSlicePool::Handle handle;
+
+    if (!m_slicePool.findOrAdd(slice, handle))
+    {
+        // We know it can't possibly be used, so must be new (!)
+
+        m_sources.add(handle);
+        return m_sources.getCount() - 1;
+    }
+
+    // Okay, could already be in the list
+    const auto index = m_sources.indexOf(handle);
+    if (index >= 0)
+    {
+        return index;
+    }
+
+    m_sources.add(handle);
+    return m_sources.getCount() - 1;
+}
+
 SlangResult SourceMap::decode(JSONContainer* container, JSONValue root, DiagnosticSink* sink)
 {
     clear();
diff --git a/source/compiler-core/slang-source-map.h b/source/compiler-core/slang-source-map.h
index 7fd64510a..739cf1992 100644
--- a/source/compiler-core/slang-source-map.h
+++ b/source/compiler-core/slang-source-map.h
@@ -15,10 +15,19 @@
 
 namespace Slang {
 
-struct SourceMap
+struct SourceMap : public RefObject
 {
     struct Entry
     {
+        void init()
+        {
+            generatedColumn = 0;
+            sourceFileIndex = 0;
+            sourceLine = 0;
+            sourceColumn = 0;
+            nameIndex = 0;
+        }
+
         // Note! All column/line are zero indexed
         Index generatedColumn;          ///< The generated column
         Index sourceFileIndex;          ///< The index into the source name/contents
@@ -38,6 +47,20 @@ struct SourceMap
         /// Get the entries on the line
     SLANG_FORCE_INLINE ConstArrayView<Entry> getEntriesForLine(Index generatedLine) const;
     
+        /// Advance to the specified line index. 
+        /// It is an error to specify a line *before* the current line. It should either be the current 
+        /// output line or a later output line. Interveining lines will be set as empty
+    void advanceToLine(Index lineIndex);
+
+        /// Add an entry to the current line
+    void addEntry(const Entry& entry);
+
+        /// Given the slice returns the index
+    Index getSourceFileIndex(const UnownedStringSlice& slice);
+
+        /// Get the name index
+    Index getNameIndex(const UnownedStringSlice& slice);
+
         /// Clear the contents of the source map
     void clear();
 
diff --git a/source/slang/slang-compiler.h b/source/slang/slang-compiler.h
index 784026761..fcf62d826 100755
--- a/source/slang/slang-compiler.h
+++ b/source/slang/slang-compiler.h
@@ -1767,6 +1767,7 @@ namespace Slang
         SourceManager* m_sourceManager = nullptr;
 
         bool m_obfuscateCode = false;
+        bool m_generateSourceMap = false;
 
         /// Holds any args that are destined for downstream compilers/tools etc
         DownstreamArgs m_downstreamArgs;
diff --git a/source/slang/slang-emit-c-like.h b/source/slang/slang-emit-c-like.h
index 1cd2045c7..9426531a8 100644
--- a/source/slang/slang-emit-c-like.h
+++ b/source/slang/slang-emit-c-like.h
@@ -73,6 +73,7 @@ public:
             /// combining information from the target and entry point.
         Profile effectiveProfile = Profile::RawEnum::Unknown;
 
+            /// The source writer to use
         SourceWriter* sourceWriter = nullptr;
     };
 
diff --git a/source/slang/slang-emit-cpp.cpp b/source/slang/slang-emit-cpp.cpp
index 795ec74b0..2a2ae06c6 100644
--- a/source/slang/slang-emit-cpp.cpp
+++ b/source/slang/slang-emit-cpp.cpp
@@ -338,7 +338,7 @@ SlangResult CPPSourceEmitter::calcTypeName(IRType* type, CodeGenTarget target, S
                     // TODO(JS): This is a bit of a hack. We don't want to emit the result here,
                     // so we replace the writer, write out the type, grab the contents, and restore the writer
 
-                    SourceWriter writer(sourceManager, LineDirectiveMode::None);
+                    SourceWriter writer(sourceManager, LineDirectiveMode::None, nullptr);
                     m_writer = &writer;
 
                     m_writer->emit(prefix);
diff --git a/source/slang/slang-emit-source-writer.cpp b/source/slang/slang-emit-source-writer.cpp
index 7692bd0ec..b27e0f8b4 100644
--- a/source/slang/slang-emit-source-writer.cpp
+++ b/source/slang/slang-emit-source-writer.cpp
@@ -16,10 +16,11 @@
 
 namespace Slang {
 
-SourceWriter::SourceWriter(SourceManager* sourceManager, LineDirectiveMode lineDirectiveMode)
+SourceWriter::SourceWriter(SourceManager* sourceManager, LineDirectiveMode lineDirectiveMode, SourceMap* sourceMap)
 {
+    m_sourceMap = sourceMap;
     m_lineDirectiveMode = lineDirectiveMode;
-    this->m_sourceManager = sourceManager;
+    m_sourceManager = sourceManager;
 }
 
 String SourceWriter::getContentAndClear()
@@ -274,7 +275,10 @@ void SourceWriter::advanceToSourceLocationIfValid(const SourceLoc& sourceLocatio
 
 void SourceWriter::advanceToSourceLocation(const SourceLoc& sourceLocation)
 {
-    if (getLineDirectiveMode() == LineDirectiveMode::None)
+    // If we don't have any line directives *and* we don't want to output 
+    // source map, we can just ignore
+    if (getLineDirectiveMode() == LineDirectiveMode::None &&
+        m_sourceMap == nullptr)
     {
         // Ignore if we aren't outputting directives 
         return;
@@ -328,7 +332,14 @@ void SourceWriter::_flushSourceLocationChange()
     // advances the location, and outputting text is what
     // triggers this flush operation.
     m_needToUpdateSourceLocation = false;
+    
     _emitLineDirectiveIfNeeded(m_nextHumaneSourceLocation);
+
+    // If we have a source map update state
+    if (m_sourceMap)
+    {
+        _updateSourceMap(m_nextHumaneSourceLocation);
+    }
 }
 
 void SourceWriter::_emitLineDirectiveAndUpdateSourceLocation(const HumaneSourceLoc& sourceLocation)
@@ -341,6 +352,31 @@ void SourceWriter::_emitLineDirectiveAndUpdateSourceLocation(const HumaneSourceL
     m_loc = newLoc;
 }
 
+void SourceWriter::_updateSourceMap(const HumaneSourceLoc& sourceLocation)
+{
+    // Ignore invalid source locations
+    if (sourceLocation.line <= 0)
+        return;
+
+    // We need to work out the current column in the generated (ie being written) output
+    Index generatedLineIndex, generatedColumnIndex;
+    _calcLocation(generatedLineIndex, generatedColumnIndex);
+
+    // Advance to the current output line
+    m_sourceMap->advanceToLine(generatedLineIndex);
+
+    // Add the entry into the map, mapping back to the original source
+    SourceMap::Entry entry;
+    entry.init();
+
+    entry.sourceFileIndex = m_sourceMap->getSourceFileIndex(sourceLocation.pathInfo.getName().getUnownedSlice());
+    entry.sourceLine = sourceLocation.line - 1;
+    entry.sourceColumn = sourceLocation.column - 1;
+    entry.generatedColumn = generatedColumnIndex;
+    
+    m_sourceMap->addEntry(entry);
+}
+
 void SourceWriter::_emitLineDirectiveIfNeeded(const HumaneSourceLoc& sourceLocation)
 {
     if (m_supressLineDirective)
@@ -478,4 +514,63 @@ void SourceWriter::_emitLineDirective(const HumaneSourceLoc& sourceLocation)
     emitRawText("\n");
 }
 
+void SourceWriter::_calcLocation(Index& outLineIndex, Index& outColumnIndex)
+{
+    // If we are at the end, then we are done.
+    if (m_currentOutputOffset == m_builder.getLength())
+    {
+        outLineIndex = m_currentLineIndex;
+        outColumnIndex = m_currentColumnIndex;
+        return;
+    }
+
+    const char* cur = m_builder.getBuffer() + m_currentOutputOffset;
+    const char* end = m_builder.end();
+
+    const char* start = cur;
+
+    while (cur < end)
+    {
+        // Reset start
+        start = cur;
+
+        // Look for the end of the line
+        while (*cur != '\n' && *cur != '\r' && cur < end)
+        {
+            cur++;
+        }
+
+        // If we are not at the total end then we must have hit a \n or \r
+        if (cur < end)
+        {
+            const auto c = *cur++;
+
+            ++m_currentLineIndex;
+            // Reset the column 
+            m_currentColumnIndex = 0;
+
+            // Check the next char to see if it's part of a CR/LF combination
+            if (cur < end)
+            {
+                const auto d = *cur;
+                // If it is combination skip the next byte
+                cur += ((c ^ d) == ('\r' ^ '\n'));
+            }
+        }
+    }
+
+    // Fix up the current index.
+    // TODO(JS):
+    // NOTE! This isn't strictly correct because it assumes one byte is a *column* which isn't actually the case with utf8 
+    // encoding... 
+    m_currentColumnIndex += Index(cur - start);
+
+    // Set the current offset is the end
+    m_currentOutputOffset = m_builder.getLength();
+
+    // Output the values
+    outLineIndex = m_currentLineIndex;
+    outColumnIndex = m_currentColumnIndex;
+}
+
 } // namespace Slang
diff --git a/source/slang/slang-emit-source-writer.h b/source/slang/slang-emit-source-writer.h
index 294cfec18..c47b5f5fa 100644
--- a/source/slang/slang-emit-source-writer.h
+++ b/source/slang/slang-emit-source-writer.h
@@ -5,6 +5,8 @@
 #include "../core/slang-basic.h"
 
 #include "slang-compiler.h"
+#include "../compiler-core/slang-source-map.h"
+
 
 namespace Slang
 {
@@ -72,8 +74,11 @@ public:
         /// Get the source manager user
     SourceManager* getSourceManager() const { return m_sourceManager; }
 
+        /// Get the associated source map. If source map tracking is not required, can return nullptr.
+    SourceMap* getSourceMap() const { return m_sourceMap; }
+
         /// Ctor
-    SourceWriter(SourceManager* sourceManager, LineDirectiveMode lineDirectiveMode);
+    SourceWriter(SourceManager* sourceManager, LineDirectiveMode lineDirectiveMode, SourceMap* sourceMap);
 
 protected:
     void _emitTextSpan(char const* textBegin, char const* textEnd);
@@ -86,10 +91,15 @@ protected:
 
     void _emitLineDirectiveIfNeeded(const HumaneSourceLoc& sourceLocation);
 
+    void _updateSourceMap(const HumaneSourceLoc& sourceLocation);
+
         // Emit a `#line` directive to the output.
         // Doesn't update state of source-location tracking.
     void _emitLineDirective(const HumaneSourceLoc& sourceLocation);
 
+        /// Calculate the current location in the ouput
+    void _calcLocation(Index& outLineIndex, Index& outColumnIndex);
+
     // The string of code we've built so far.
     // TODO(JS): We could store the text in chunks, and then only sew together into one buffer
     // when we are done. Doing so would not require copies/reallocs until the full buffer has been
@@ -103,6 +113,13 @@ protected:
     SourceLoc m_nextSourceLoc;
     HumaneSourceLoc m_nextHumaneSourceLocation;
     
+    // Used to determine the current location in the output for outputting the source map
+    // This is separate from m_loc, because m_loc doesn't appear to track the line/column directly 
+    // in the output stream - for example when #line emits a "raw" emit takes place.
+    Count m_currentOutputOffset = 0;
+    Index m_currentLineIndex = 0;
+    Index m_currentColumnIndex = 0;
+
     bool m_needToUpdateSourceLocation = false;
 
     bool m_supressLineDirective = false;
@@ -123,6 +140,8 @@ protected:
     Dictionary<String, int> m_mapGLSLSourcePathToID;
     int m_glslSourceIDCount = 0;
 
+    RefPtr<SourceMap> m_sourceMap;
+
     LineDirectiveMode m_lineDirectiveMode;
 };
 
diff --git a/source/slang/slang-emit.cpp b/source/slang/slang-emit.cpp
index a25fae5ae..335e95c9e 100644
--- a/source/slang/slang-emit.cpp
+++ b/source/slang/slang-emit.cpp
@@ -935,7 +935,15 @@ SlangResult CodeGenContext::emitEntryPointsSourceFromIR(ComPtr<IArtifact>& outAr
         lineDirectiveMode = LineDirectiveMode::GLSL;
     }
 
-    SourceWriter sourceWriter(sourceManager, lineDirectiveMode );
+    RefPtr<SourceMap> sourceMap;
+
+    // If SourceMap is enabled, we create one and associate it with the sourceWriter
+    if (targetRequest->getLinkage()->m_generateSourceMap)
+    {
+        sourceMap = new SourceMap;
+    }
+
+    SourceWriter sourceWriter(sourceManager, lineDirectiveMode, sourceMap );
 
     CLikeSourceEmitter::Desc desc;
 
@@ -1096,6 +1104,34 @@ SlangResult CodeGenContext::emitEntryPointsSourceFromIR(ComPtr<IArtifact>& outAr
         artifact->addAssociated(metadata);
     }
 
+    if (sourceMap)
+    {
+        SourceManager sourceMapSourceManager;
+        sourceMapSourceManager.initialize(nullptr, nullptr);
+
+        // Create a sink
+        DiagnosticSink sourceMapSink(&sourceMapSourceManager, nullptr);
+
+        // Turn into JSON
+        RefPtr<JSONContainer> jsonContainer(new JSONContainer(&sourceMapSourceManager));
+
+        JSONValue jsonValue;
+        SLANG_RETURN_ON_FAIL(sourceMap->encode(jsonContainer, &sourceMapSink, jsonValue));
+
+        // Okay now convert this into a text file and then a blob
+
+        // Convert into a string
+        JSONWriter writer(JSONWriter::IndentationStyle::KNR);
+        jsonContainer->traverseRecursively(jsonValue, &writer);
+
+        auto sourceMapBlob = StringBlob::moveCreate(writer.getBuilder());
+
+        auto sourceMapArtifact = ArtifactUtil::createArtifact(ArtifactDesc::make(ArtifactKind::Text, ArtifactPayload::SourceMap, ArtifactStyle::None));
+        sourceMapArtifact->addRepresentationUnknown(sourceMapBlob);
+
+        artifact->addAssociated(sourceMapArtifact);
+    }
+
     outArtifact.swap(artifact);
     return SLANG_OK;
 }
diff --git a/source/slang/slang-options.cpp b/source/slang/slang-options.cpp
index 944162acc..f6932b9e0 100644
--- a/source/slang/slang-options.cpp
+++ b/source/slang/slang-options.cpp
@@ -669,6 +669,7 @@ struct OptionsParser
             "  -save-stdlib-bin-source <filename>: Same as -save-stdlib but output\n"
             "      the data as a C array.\n"
             "  -track-liveness: Enable liveness tracking. Places SLANG_LIVE_START, and SLANG_LIVE_END in output source to indicate value liveness.\n"
+            "  -source-map: Enables outputting of a source map. Note this is *distinct* from line-directive-mode.\n"
             "\n"
             "Deprecated options (allowed but ignored; may be removed in future):\n"
             "\n"
@@ -1176,6 +1177,10 @@ struct OptionsParser
                 {
                     getCurrentTarget()->targetFlags |= SLANG_TARGET_FLAG_PARAMETER_BLOCKS_USE_REGISTER_SPACES;
                 }
+                else if(argValue == "-source-map")
+                {
+                    requestImpl->getLinkage()->m_generateSourceMap = true;
+                }
                 else if (argValue == "-ir-compression")
                 {
                     CommandLineArg name;
diff --git a/source/slang/slang-serialize-ast.cpp b/source/slang/slang-serialize-ast.cpp
index 04b5f71b1..b51775710 100644
--- a/source/slang/slang-serialize-ast.cpp
+++ b/source/slang/slang-serialize-ast.cpp
@@ -129,14 +129,14 @@ struct ASTFieldAccess
 
         String readDump;
         {
-            SourceWriter sourceWriter(sourceManager, LineDirectiveMode::None);
+            SourceWriter sourceWriter(sourceManager, LineDirectiveMode::None, nullptr);
             ASTDumpUtil::dump(reader.getPointer(SerialIndex(1)).dynamicCast<NodeBase>(), ASTDumpUtil::Style::Hierachical, dumpFlags, &sourceWriter);
             readDump = sourceWriter.getContentAndClear();
 
         }
         String origDump;
         {
-            SourceWriter sourceWriter(sourceManager, LineDirectiveMode::None);
+            SourceWriter sourceWriter(sourceManager, LineDirectiveMode::None, nullptr);
             ASTDumpUtil::dump(node, ASTDumpUtil::Style::Hierachical, dumpFlags, &sourceWriter);
             origDump = sourceWriter.getContentAndClear();
         }
diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp
index da8fcdcd6..14b59d156 100644
--- a/source/slang/slang.cpp
+++ b/source/slang/slang.cpp
@@ -2277,7 +2277,7 @@ void FrontEndCompileRequest::parseTranslationUnit(
         if (shouldDumpAST)
         {
             StringBuilder buf;
-            SourceWriter writer(linkage->getSourceManager(), LineDirectiveMode::None);
+            SourceWriter writer(linkage->getSourceManager(), LineDirectiveMode::None, nullptr);
 
             ASTDumpUtil::dump(translationUnit->getModuleDecl(), ASTDumpUtil::Style::Flat, 0, &writer);