Round trip source map (#2810)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Make output of obfuscation locs work in a slang-module.

* Tidy up detection for writing serialized source locs.

* Support for .zip references.
Handling of obfuscated source maps read from containers.
A test to check obfuscated source map working on a module.

* When using obfuscation, always obfuscate locs instead of stripping them. We keep a source map, so we can still produce reasonable errors.

* Write out source locs if debug information is enabled.

* Check output without sourcemap.

* Small fixes.

* Small improvements around hash calculation for source map name.

* Disable test that fails on x86 gcc linux for now.

* Fix issues around obfuscated source map using lines rather than columns.
Fix some issues around encoding/decoding.

* Make column calculation of source locs take into account utf8/tabs.
Don't special case obfuscated source map for lookup for source loc.

* Support following multiple source maps.

* Small fixes/improvements around SourceMap lookup.

5 files changed, 194 insertions, 43 deletions

diff --git a/source/slang/slang-compiler.cpp b/source/slang/slang-compiler.cpp
index 43a6d238e..becd65596 100644
--- a/source/slang/slang-compiler.cpp
+++ b/source/slang/slang-compiler.cpp
@@ -1812,6 +1812,19 @@ namespace Slang
     }
 
     
+    bool _shouldWriteSourceLocs(Linkage* linkage)
+    {
+        // If debug information or source manager are not avaiable we can't/shouldn't write out locs
+        if (linkage->debugInfoLevel == DebugInfoLevel::None || 
+            linkage->getSourceManager() == nullptr)
+        {
+            return false;
+        }
+        
+        // Otherwise we do want to write out the locs
+        return true;
+    }
+
     SlangResult EndToEndCompileRequest::writeContainerToStream(Stream* stream)
     {
         auto linkage = getLinkage();
@@ -1827,7 +1840,9 @@ namespace Slang
             // Also currently only IR is needed.
             options.optionFlags &= ~SerialOptionFlag::ASTModule;
         }
-        else if (linkage->debugInfoLevel != DebugInfoLevel::None && linkage->getSourceManager())
+        
+        // If debug information is enabled, enable writing out source locs
+        if (_shouldWriteSourceLocs(linkage))
         {
             options.optionFlags |= SerialOptionFlag::SourceLocation;
             options.sourceManager = linkage->getSourceManager();
@@ -1897,7 +1912,8 @@ namespace Slang
 
             auto sourceMap = translationUnit->getModule()->getIRModule()->getObfuscatedSourceMap();
 
-            if (sourceMap)
+            // If we have a source map *and* we want to generate them for output add to the container
+            if (sourceMap && getLinkage()->m_generateSourceMap)
             {
                 // Write it out
                 String json;
diff --git a/source/slang/slang-ir-obfuscate-loc.cpp b/source/slang/slang-ir-obfuscate-loc.cpp
index 3ff2f3713..9a2f15fa4 100644
--- a/source/slang/slang-ir-obfuscate-loc.cpp
+++ b/source/slang/slang-ir-obfuscate-loc.cpp
@@ -46,6 +46,17 @@ static void _findInstsRec(IRInst* inst, List<InstWithLoc>& out)
     }
 }
 
+// We assume the root source manager is the stdlibs
+static SourceLoc _getStdLibLastLoc(SourceManager* sourceManager)
+{
+    auto rootManager = sourceManager;
+    while (rootManager->getParent())
+    {
+        rootManager = rootManager->getParent();
+    }
+    return rootManager->getNextRangeStart();
+}
+
 SlangResult obfuscateModuleLocs(IRModule* module, SourceManager* sourceManager)
 {
     // There shouldn't be an obfuscated source map set
@@ -60,26 +71,28 @@ SlangResult obfuscateModuleLocs(IRModule* module, SourceManager* sourceManager)
     instWithLocs.sort();
         
     // Lets produce a hash, so we can use as a key for random number generation.
-    // We could base it on time, or some other thing as there is no requirement for 
-    // stability or consistency.
-    // We use a hash because it avoids issues around clocks, and availability of a clock
-    // as a good source of entropy.
-    //
-    // An argument *could* be made to generate the name via some mechanism that uniquely identified the 
-    // combination of flags, options, files, names that identified the compilation, but that is 
-    // not easily achieved.
+    // 
+    // We could base it on time, or some other random seed. But it would be preferable
+    // if it was stable, and compilations of the same module on different machines
+    // produce the same hash. 
+    // 
+    // Doing so would mean that we could use the obfuscated location ouput to output 
+    // the origin.
+
     HashCode hash = 0;
 
     List<LocPair> locPairs;
 
     // We want the hash to be stable. One problem is the source locs depend on their order of inclusion.
-    // To work around this we are going 
+    // To work around this we are going to hash via offsets, not locs.
     {
         SourceView* sourceView = nullptr;
 
+        const SourceLoc endStdLibLoc = _getStdLibLastLoc(sourceManager);
+
         SourceLoc curLoc;
         for (const auto& instWithLoc : instWithLocs)
-        {
+        {            
             if (instWithLoc.loc != curLoc)
             {
                 LocPair locPair;
@@ -89,21 +102,36 @@ SlangResult obfuscateModuleLocs(IRModule* module, SourceManager* sourceManager)
                 // This is the current loc
                 curLoc = instWithLoc.loc;
             
+                // Ignore any stdlib locs in the hash
+                if (instWithLoc.loc.getRaw() < endStdLibLoc.getRaw())
+                {
+                    continue;
+                }
+
                 // If the loc isn't in the view, lookup the view it is in
                 if (sourceView == nullptr || 
                     !sourceView->getRange().contains(curLoc))
                 {
                     sourceView = sourceManager->findSourceViewRecursively(curLoc);
                     SLANG_ASSERT(sourceView);
-
+                    // If there is no source view we can't apply to the hash
+                    if (sourceView == nullptr)
+                    {
+                        continue;
+                    }
+
+                    const auto pathInfo = sourceView->getViewPathInfo();
+                    const auto name = pathInfo.getName();
+                    const auto nameHash = getHashCode(pathInfo.getName().getUnownedSlice());
+                    
                     // Combine the name
-                    hash = combineHash(hash, getHashCode(sourceView->getViewPathInfo().getName().getUnownedSlice()));
+                    hash = combineHash(hash, nameHash);
                 }
-                SLANG_ASSERT(sourceView);
 
                 // We combine the *offset* which is stable
-                hash = combineHash(hash, getHashCode(sourceView->getRange().getOffset(curLoc)));
-            }
+                const auto offset = sourceView->getRange().getOffset(curLoc);
+                hash = combineHash(hash, getHashCode(offset));
+            }    
         }
     }
 
@@ -138,16 +166,28 @@ SlangResult obfuscateModuleLocs(IRModule* module, SourceManager* sourceManager)
                 dst[i * 2 + 1] = CharUtil::getHexChar(data[i] >> 4);
             }
             buf.appendInPlace(dst, charsCount);
+
+            // Make it clear this "source" is actually just for obfuscation.
+            buf << toSlice("-obfuscated");
+
             obfusctatedPathInfo = PathInfo::makePath(buf);
         }
     }
 
     SourceFile* obfuscatedFile = sourceManager->createSourceFileWithSize(obfusctatedPathInfo, uniqueLocCount);
 
-    // We have only one line for all locs, just set up that way...
+    // We put each loc on it's own line. We do this rather than using a single line because
+    // it means the `#line` directives can still do something meaningful, since the best resolution
+    // they have is a single line.
     {
-        const uint32_t offsets[2] = { 0, uint32_t(uniqueLocCount) };
-        obfuscatedFile->setLineBreakOffsets(offsets, SLANG_COUNT_OF(offsets));
+        List<uint32_t> offsets;
+        offsets.setCount(uniqueLocCount + 1);
+        for (Index i = 0; i < uniqueLocCount + 1; ++i)
+        {
+            offsets[i] = uint32_t(i);
+        }
+
+        obfuscatedFile->setLineBreakOffsets(offsets.getBuffer(), offsets.getCount());
     }
     
     // Create the view we are going to use from the obfusctated "file".
@@ -210,10 +250,16 @@ SlangResult obfuscateModuleLocs(IRModule* module, SourceManager* sourceManager)
     RefPtr<SourceMap> sourceMap = new SourceMap;
     sourceMap->m_file = obfusctatedPathInfo.getName();
 
-    // Make sure we have line 0.
-    // We only end up with one line in the obfuscated map.
-    sourceMap->advanceToLine(0);
-
+    // Set up entries one per line
+    List<SourceMap::Entry> entries;
+    {
+        entries.setCount(uniqueLocCount);
+        for (auto& entry : entries)
+        {
+            entry.init();
+        }
+    }
+    
     {        
         // Current view, with cached "View" based sourceFileIndex
         SourceView* curView = nullptr;
@@ -275,27 +321,35 @@ SlangResult obfuscateModuleLocs(IRModule* module, SourceManager* sourceManager)
                 sourceFileIndex = curPathSourceFileIndex;
             }
 
-            // Create the entry
-            SourceMap::Entry entry;
-            entry.init();
+            // Calculate the line index associated with this loc
+            const Index generatedLineIndex = Index(obfuscatedRange.getOffset(pair.obfuscatedLoc));
+
+            // Set it up
+            SourceMap::Entry& entry = entries[generatedLineIndex];
 
             entry.sourceFileIndex = sourceFileIndex;
   
-            // Calculate the column offset, from the pair obfuscated loc
-            entry.generatedColumn = Index(obfuscatedRange.getOffset(pair.obfuscatedLoc));
-
+            // The generated has a line per loc, so the generated column is always 0
+            entry.generatedColumn = 0;
+            
             // We need to subtract 1, because handleLoc locations are 1 indexed, but SourceMap
             // entry is 0 indexed.
             entry.sourceColumn = handleLoc.column - 1;
             entry.sourceLine = handleLoc.line - 1;
-
-            // Add it to the source map
-            sourceMap->addEntry(entry);
         }
     }
 
+    // Add all of the entries in line order to the source map
+    for (Index i = 0; i < uniqueLocCount; ++i)
+    {
+        // Advance to the current line.
+        sourceMap->advanceToLine(i);
+        // Add it to the source map
+        sourceMap->addEntry(entries[i]);
+    }
+
     // Associate the sourceMap with the obfuscated file
-    obfuscatedFile->setObfuscatedSourceMap(sourceMap);
+    obfuscatedFile->setSourceMap(sourceMap);
 
     // Set the obfuscated map onto the module
     module->setObfuscatedSourceMap(sourceMap);
diff --git a/source/slang/slang-lower-to-ir.cpp b/source/slang/slang-lower-to-ir.cpp
index 8cceaff02..110f85b87 100644
--- a/source/slang/slang-lower-to-ir.cpp
+++ b/source/slang/slang-lower-to-ir.cpp
@@ -9652,7 +9652,7 @@ RefPtr<IRModule> generateIRForTranslationUnit(
         // We don't do the obfuscation remapping here, because DCE and other passes may 
         // change what locs are actually needed, we need to be sure 
         // that if we have obfuscation enabled we don't forget to obfuscate.
-        stripOptions.stripSourceLocs = linkage->m_obfuscateCode && !linkage->m_generateSourceMap;
+        stripOptions.stripSourceLocs = false;
         stripFrontEndOnlyInstructions(module, stripOptions);
     
         // Stripping out decorations could leave some dead code behind
@@ -9667,7 +9667,7 @@ RefPtr<IRModule> generateIRForTranslationUnit(
         options.keepExportsAlive = true;
         eliminateDeadCode(module, options);
 
-        if (linkage->m_obfuscateCode && linkage->m_generateSourceMap)
+        if (linkage->m_obfuscateCode)
         {
             // The obfuscated source map is stored on the module
             obfuscateModuleLocs(module, compileRequest->getSourceManager());
diff --git a/source/slang/slang-options.cpp b/source/slang/slang-options.cpp
index ae7988322..b1e1b93d2 100644
--- a/source/slang/slang-options.cpp
+++ b/source/slang/slang-options.cpp
@@ -1623,6 +1623,12 @@ struct OptionsParser
                         desc.kind = ArtifactKind::Library;
                     }
 
+                    // If its a zip we'll *assume* its a zip holding compilation results
+                    if (desc.kind == ArtifactKind::Zip)
+                    {
+                        desc.payload = ArtifactPayload::CompileResults;
+                    }
+                    
                     if (!ArtifactDescUtil::isLinkable(desc))
                     {
                         sink->diagnose(referenceModuleName.loc, Diagnostics::kindNotLinkable, Path::getPathExt(path));
diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp
index 4904abe03..c8cda0dca 100644
--- a/source/slang/slang.cpp
+++ b/source/slang/slang.cpp
@@ -15,6 +15,8 @@
 #include "../compiler-core/slang-artifact-associated-impl.h"
 #include "../compiler-core/slang-artifact-container-util.h"
 
+#include "../compiler-core/slang-json-source-map-util.h"
+
 #include "../core/slang-memory-file-system.h"
 
 #include "slang-module-library.h"
@@ -4833,10 +4835,85 @@ void EndToEndCompileRequest::setDefaultModuleName(const char* defaultModuleName)
     frontEndReq->m_defaultModuleName = namePool->getName(defaultModuleName);
 }
 
+SlangResult _addLibraryReference(EndToEndCompileRequest* req, IArtifact* artifact, ModuleLibrary* moduleLibrary)
+{
+    FrontEndCompileRequest* frontEndRequest = req->getFrontEndReq();
+    frontEndRequest->m_extraEntryPoints.addRange(moduleLibrary->m_entryPoints.getBuffer(), moduleLibrary->m_entryPoints.getCount());
+
+    // Add to the m_libModules
+    auto linkage = req->getLinkage();
+    linkage->m_libModules.add(ComPtr<IArtifact>(artifact));
+
+    return SLANG_OK;
+}
+
 SlangResult _addLibraryReference(EndToEndCompileRequest* req, IArtifact* artifact)
 {
     auto desc = artifact->getDesc();
 
+    // TODO(JS):
+    // This isn't perhaps the best way to handle this scenario, as IArtifact can 
+    // support lazy evaluation, with suitable hander.
+    // For now we just read in and strip out the bits we want.
+    if (isDerivedFrom(desc.kind, ArtifactKind::Container) && 
+        isDerivedFrom(desc.payload, ArtifactPayload::CompileResults))
+    {
+        // We want to read as a file system
+        ComPtr<IArtifact> container;
+
+        SLANG_RETURN_ON_FAIL(ArtifactContainerUtil::readContainer(artifact, container));
+
+        // Find the payload... It should be linkable
+        if (!ArtifactDescUtil::isLinkable(container->getDesc()))
+        {
+            return SLANG_FAIL;
+        }
+
+        ComPtr<IModuleLibrary> libraryIntf;
+        SLANG_RETURN_ON_FAIL(loadModuleLibrary(ArtifactKeep::Yes, container, req, libraryIntf));
+
+        auto library = as<ModuleLibrary>(libraryIntf);
+        
+        // Look for source maps
+        for (auto associated : container->getAssociated())
+        {
+            auto assocDesc = associated->getDesc();
+
+            // If we find an obfuscated source map load it and associate
+            if (isDerivedFrom(assocDesc.kind, ArtifactKind::Json) && 
+                isDerivedFrom(assocDesc.payload, ArtifactPayload::SourceMap) &&
+                isDerivedFrom(assocDesc.style, ArtifactStyle::Obfuscated))
+            {
+                ComPtr<ISlangBlob> sourceMapBlob;
+                SLANG_RETURN_ON_FAIL(associated->loadBlob(ArtifactKeep::No, sourceMapBlob.writeRef()));
+
+                RefPtr<SourceMap> sourceMap;
+                SLANG_RETURN_ON_FAIL(JSONSourceMapUtil::read(sourceMapBlob, nullptr, sourceMap));
+                
+                // I guess we add to all ir modules?
+
+                for (auto irModule : library->m_modules)
+                {
+                    irModule->setObfuscatedSourceMap(sourceMap);
+                }
+
+                // Look up the source file
+                auto sourceManager = req->getSink()->getSourceManager();
+
+                auto name = Path::getFileNameWithoutExt(associated->getName());
+
+                if (name.getLength())
+                {
+                    auto sourceFile = sourceManager->findSourceFileByPathRecursively(name);
+                    sourceFile->setSourceMap(sourceMap);
+                }
+            }
+        }
+
+        SLANG_RETURN_ON_FAIL(_addLibraryReference(req, container, library));
+        return SLANG_OK;
+    }
+
     if (desc.kind == ArtifactKind::Library && desc.payload == ArtifactPayload::SlangIR)
     {
         ComPtr<IModuleLibrary> libraryIntf;
@@ -4849,15 +4926,13 @@ SlangResult _addLibraryReference(EndToEndCompileRequest* req, IArtifact* artifac
             return SLANG_FAIL;
         }
 
-        FrontEndCompileRequest* frontEndRequest = req->getFrontEndReq();
-        frontEndRequest->m_extraEntryPoints.addRange(library->m_entryPoints.getBuffer(), library->m_entryPoints.getCount());
-    }
-    else
-    {
-        // TODO(JS):
-        // Do we want to check the path exists?
+        SLANG_RETURN_ON_FAIL(_addLibraryReference(req, artifact, library));
+        return SLANG_OK;
     }
 
+    // TODO(JS):
+    // Do we want to check the path exists?
+
     // Add to the m_libModules
     auto linkage = req->getLinkage();
     linkage->m_libModules.add(ComPtr<IArtifact>(artifact));