Improvements in -X support (#1852)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Added SourceLoc handling for command line parsing.

* Fix typo in debug.

* Fix issue around the DiagnosticSink used in options parsing not having a writer available - by having DiagnosticSink parenting.

* Small rename for clarity.

* WIP extracting command line args for downstream tools.

* Unit tests/bug fixes around extracting args.

* Use DownstreamArgs in the EndToEndCompileRequest

* Passing downstream compiler options downstream.

* Fix issue with endToEndReq being nullptr.

* Fix issue with diagnostics number change.

* Small improvements to how the source line is displayed if it's too long.
Default to 120, as suggested in previous review.

* Make render test use x-args parsing and CommandArgReader.

* Added missing diagnostics.

* More DownstreamArgs to linkage so can be seen by 'components'.
Added dxc-x-arg test.

* Used combination of name and args instead of two Lists, which whilst equivalent was perhaps a little confusing.

* Added documentation for -X support.

* Added test for x-args parsing diagnostic. Improved diagnostic with list of known names.

* Fix issues from merge.

* Fix lookup for -matrix-layout-column-major in render test.

* Remove commented out line.

9 files changed, 93 insertions, 51 deletions

diff --git a/source/compiler-core/slang-command-line-args.cpp b/source/compiler-core/slang-command-line-args.cpp
index 1d452bb10..d7a053b00 100644
--- a/source/compiler-core/slang-command-line-args.cpp
+++ b/source/compiler-core/slang-command-line-args.cpp
@@ -63,7 +63,6 @@ bool CommandLineArgs::hasArgs(const char*const* args, Index count) const
     return true;
 }
 
-
 /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
                          CommandLineReader
@@ -121,14 +120,11 @@ SlangResult CommandLineReader::expectArg(CommandLineArg& outArg)
 
 Index DownstreamArgs::addName(const String& name)
 {
-    Index index = m_names.indexOf(name);
+    Index index = findName(name);
     if (index < 0)
     {
-        index = m_names.getCount();
-        m_names.add(name);
-
-        CommandLineArgs args(m_context);
-        m_args.add(args);
+        index = m_entries.getCount();
+        m_entries.add(Entry{name, CommandLineArgs(m_context) });
     }
     return index;
 }
@@ -137,7 +133,10 @@ Index DownstreamArgs::_findOrAddName(SourceLoc loc, const UnownedStringSlice& na
 {
     if (name.getLength() <= 0)
     {
-        sink->diagnose(loc, MiscDiagnostics::downstreamToolNameNotDefined);
+        if (sink)
+        {
+            sink->diagnose(loc, MiscDiagnostics::downstreamToolNameNotDefined);
+        }
         return -1;
     }
 
@@ -152,15 +151,38 @@ Index DownstreamArgs::_findOrAddName(SourceLoc loc, const UnownedStringSlice& na
         return index;
     }
 
-    sink->diagnose(loc, MiscDiagnostics::downstreamNameNotKnown);
+    if (sink)
+    {
+        StringBuilder names;
+
+        names << "[ ";
+        for (Index i = 0; i < m_entries.getCount(); ++i)
+        {
+            if (i)
+            {
+                names << ", ";
+            }
+            names << m_entries[i].name;
+        }
+        names << " ]";
+
+        sink->diagnose(loc, MiscDiagnostics::downstreamNameNotKnown, names);
+    }
     return -1;
 }
 
 CommandLineArgs& DownstreamArgs::getArgsByName(char* name)
 {
-    Index index = findName(name);
+    const Index index = findName(name);
+    SLANG_ASSERT(index >= 0);
+    return m_entries[index].args;
+}
+
+const CommandLineArgs& DownstreamArgs::getArgsByName(char* name) const
+{
+    const Index index = findName(name);
     SLANG_ASSERT(index >= 0);
-    return m_args[index];
+    return m_entries[index].args;
 }
 
 SlangResult DownstreamArgs::stripDownstreamArgs(CommandLineArgs& ioArgs, Flags flags, DiagnosticSink* sink)
diff --git a/source/compiler-core/slang-command-line-args.h b/source/compiler-core/slang-command-line-args.h
index 7500c1a91..c18996005 100644
--- a/source/compiler-core/slang-command-line-args.h
+++ b/source/compiler-core/slang-command-line-args.h
@@ -17,6 +17,9 @@ struct CommandLineArg
     SourceLoc loc;          ///< The location of the arg
 };
 
+/* This type ends up being really just a container for the sourceManager that has the CommandLine specific SourceLocs.
+That it would perhaps be better to just have SourceManager derive from RefObject, and then we could remove this
+type. */
 class CommandLineContext : public RefObject
 {
 public:        
@@ -27,6 +30,8 @@ public:
     {
         m_sourceManager.initialize(nullptr, fileSystemExt);
         // Make range start from high value, so can be differentiated from other uses
+        // That this doesn't not assume exclusive use of this range - just that in normal use scenarios
+        // there is no confusion, and using the wrong source manager, will typically report nothing is found.
         m_sourceManager.allocateSourceRange(~(~SourceLoc::RawValue(0) >> 1));
     }
 
@@ -128,15 +133,22 @@ struct DownstreamArgs
         };
     };
 
+    struct Entry
+    {
+        String name;                        ///< The name of the 'tool' that these args are associated with
+        CommandLineArgs args;               ///< The args to be passed to the tool
+    };
+
         /// Add a name, returns the index
     Index addName(const String& name);
         /// Find the index of a name. Returns < 0 if not found.
-    Index findName(const String& name) const { return m_names.indexOf(name); }
+    Index findName(const String& name) const { return m_entries.findFirstIndex([&](const Entry& entry) -> bool { return entry.name == name; }); }
 
         /// Get the args at the nameIndex
-    CommandLineArgs& getArgsAt(Index nameIndex) { return m_args[nameIndex]; }
+    CommandLineArgs& getArgsAt(Index nameIndex) { return m_entries[nameIndex].args; }
         /// Get args by name - will assert if name isn't found
     CommandLineArgs& getArgsByName(char* name);
+    const CommandLineArgs& getArgsByName(char* name) const;
 
         /// Looks for '-X' expressions, removing them from ioArgs and putting in appropriate args 
     SlangResult stripDownstreamArgs(CommandLineArgs& ioArgs, Flags flags, DiagnosticSink* sink);
@@ -155,10 +167,9 @@ struct DownstreamArgs
 protected:
     Index _findOrAddName(SourceLoc loc, const UnownedStringSlice& name, Flags flags, DiagnosticSink* sink);
 
-    List<String> m_names;
-    List<CommandLineArgs> m_args;
+    List<Entry> m_entries;                  ///< All of the entries
 
-    RefPtr<CommandLineContext> m_context;
+    RefPtr<CommandLineContext> m_context;   ///< The context that is being used (primarily for loc tracking) across all entries/args
 };
 
 
diff --git a/source/compiler-core/slang-misc-diagnostic-defs.h b/source/compiler-core/slang-misc-diagnostic-defs.h
index e5c7c335e..aa87f02f9 100644
--- a/source/compiler-core/slang-misc-diagnostic-defs.h
+++ b/source/compiler-core/slang-misc-diagnostic-defs.h
@@ -24,7 +24,7 @@
 
 DIAGNOSTIC(-1, Note, seeTokenPasteLocation, "see token pasted location")
 
-DIAGNOSTIC(100000, Error, downstreamNameNotKnown, "downstream tool name not known")
+DIAGNOSTIC(100000, Error, downstreamNameNotKnown, "downstream tool name not known, allowed names are $0")
 DIAGNOSTIC(100001, Error, expectedArgumentForOption, "expected an argument for command-line option '$0'")
 DIAGNOSTIC(100002, Error, unbalancedDownstreamArguments, "unbalanced downstream arguments")
 DIAGNOSTIC(100003, Error, closeOfUnopenDownstreamArgs, "close of an unopen downstream argument scope")
diff --git a/source/core/slang-token-reader.cpp b/source/core/slang-token-reader.cpp
index a15dcda9c..3be010751 100644
--- a/source/core/slang-token-reader.cpp
+++ b/source/core/slang-token-reader.cpp
@@ -1,7 +1,8 @@
 #include "slang-token-reader.h"
 
-namespace Slang
-{
+namespace Slang {
+namespace Misc {
+
     enum class TokenizeErrorType
     {
         InvalidCharacter, InvalidEscapeSequence
@@ -765,4 +766,6 @@ namespace Slang
         this->tokens = TokenizeText("", text);
         tokenPtr = 0;
     }
-}
+
+} // namespace Misc
+} // namespace Slang
diff --git a/source/core/slang-token-reader.h b/source/core/slang-token-reader.h
index 18b56f31f..0d59eea76 100644
--- a/source/core/slang-token-reader.h
+++ b/source/core/slang-token-reader.h
@@ -3,8 +3,9 @@
 
 #include "slang-basic.h"
 
-namespace Slang
-{
+namespace Slang {
+namespace Misc {
+
     /* NOTE! This TokenReader is NOT used by the main slang compiler !*/
 
     enum class TokenType
@@ -293,7 +294,9 @@ namespace Slang
             result.add(lastStr);
         return result;
     }
-}
+
+} // namespace Misc
+} // namespace Slang
 
 
 #endif
diff --git a/source/slang/slang-compiler.cpp b/source/slang/slang-compiler.cpp
index 31ead4cc3..9a2a69248 100755
--- a/source/slang/slang-compiler.cpp
+++ b/source/slang/slang-compiler.cpp
@@ -967,13 +967,14 @@ namespace Slang
         /* Let's set the compiler specific options 
 
           We can only do this if the endToEndReq is set. */
-        if (endToEndReq)
         {
+            auto linkage = targetReq->getLinkage();
+        
             auto name = TypeTextUtil::getPassThroughName((SlangPassThrough)downstreamCompiler);
-            const Index nameIndex = endToEndReq->m_downstreamArgs.findName(name);
+            const Index nameIndex = linkage->m_downstreamArgs.findName(name);
             if (nameIndex >= 0)
             {
-                auto& args = endToEndReq->m_downstreamArgs.getArgsAt(nameIndex);
+                auto& args = linkage->m_downstreamArgs.getArgsAt(nameIndex);
                 for (const auto& arg : args.m_args)
                 {
                     options.compilerSpecificArguments.add(arg.value);
diff --git a/source/slang/slang-compiler.h b/source/slang/slang-compiler.h
index 603ee0bb5..e3d8e506c 100755
--- a/source/slang/slang-compiler.h
+++ b/source/slang/slang-compiler.h
@@ -1349,6 +1349,9 @@ namespace Slang
         // Determine whether to output heterogeneity-related code
         bool m_heterogeneous = false;
 
+        /// Holds any args that are destined for downstream compilers/tools etc
+        DownstreamArgs m_downstreamArgs;
+
         // Name pool for looking up names
         NamePool namePool;
 
@@ -2036,9 +2039,6 @@ namespace Slang
         };
         Dictionary<TargetRequest*, RefPtr<TargetInfo>> m_targetInfos;
 
-            /// Holds any args that are destined for downstream compilers/tools etc
-        DownstreamArgs m_downstreamArgs;
-
             /// Writes the modules in a container to the stream
         SlangResult writeContainerToStream(Stream* stream);
         
diff --git a/source/slang/slang-options.cpp b/source/slang/slang-options.cpp
index f4cd2e4ef..00372ca3c 100644
--- a/source/slang/slang-options.cpp
+++ b/source/slang/slang-options.cpp
@@ -410,7 +410,7 @@ struct OptionsParser
 
         DiagnosticSink* requestSink = requestImpl->getSink();
 
-        CommandLineContext* cmdLineContext = requestImpl->m_downstreamArgs.getContext();
+        CommandLineContext* cmdLineContext = requestImpl->getLinkage()->m_downstreamArgs.getContext();
         
         // Why create a new DiagnosticSink?
         // We *don't* want the lexer that comes as default (it's for Slang source!)
@@ -437,9 +437,6 @@ struct OptionsParser
             parseSink.setFlag(DiagnosticSink::Flag::SourceLocationLine);
         }
 
-        // We don't know how big the terminal is.. let's guess 120 for now 
-        parseSink.setSourceLineMaxLength(120);
-
         // All diagnostics will also be sent to requestSink
         parseSink.setParentSink(requestSink);
 
@@ -451,8 +448,11 @@ struct OptionsParser
         // Doing so will allocate some SourceLoc space from the CommandLineContext.
         args.setArgs(argv, argc);
 
-        // Before we do anything else lets strip out all of the downstream arguments.
-        SLANG_RETURN_ON_FAIL(requestImpl->m_downstreamArgs.stripDownstreamArgs(args, 0, sink));
+        {
+            auto linkage = requestImpl->getLinkage();
+            // Before we do anything else lets strip out all of the downstream arguments.
+            SLANG_RETURN_ON_FAIL(linkage->m_downstreamArgs.stripDownstreamArgs(args, 0, sink));
+        }
 
         CommandLineReader reader(&args, sink);
 
diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp
index 9a109f9d7..613992354 100644
--- a/source/slang/slang.cpp
+++ b/source/slang/slang.cpp
@@ -731,6 +731,24 @@ Linkage::Linkage(Session* session, ASTBuilder* astBuilder, Linkage* builtinLinka
             mapNameToLoadedModules.Add(pair.Key, pair.Value);
         }
     }
+
+    {
+        RefPtr<CommandLineContext> context = new CommandLineContext;
+        m_downstreamArgs = DownstreamArgs(context);
+
+        // Add all of the possible names we allow for downstream tools
+        {
+            for (Index i = SLANG_PASS_THROUGH_NONE + 1; i < SLANG_PASS_THROUGH_COUNT_OF; ++i)
+            {
+                m_downstreamArgs.addName(TypeTextUtil::getPassThroughName(SlangPassThrough(i)));
+            }
+
+            // Generic downstream tool
+            m_downstreamArgs.addName("downstream");
+            // Generic downstream linker
+            m_downstreamArgs.addName("linker");
+        }
+    }
 }
 
 ISlangUnknown* Linkage::getInterface(const Guid& guid)
@@ -2047,22 +2065,6 @@ void EndToEndCompileRequest::init()
     m_frontEndReq = new FrontEndCompileRequest(getLinkage(), m_writers, getSink());
 
     m_backEndReq = new BackEndCompileRequest(getLinkage(), getSink());
-
-    RefPtr<CommandLineContext> context = new CommandLineContext;
-    m_downstreamArgs = DownstreamArgs(context);
-
-    // Add all of the possible names we allow for downstream tools
-    {
-        for (Index i = SLANG_PASS_THROUGH_NONE + 1; i < SLANG_PASS_THROUGH_COUNT_OF; ++i)
-        {
-            m_downstreamArgs.addName(TypeTextUtil::getPassThroughName(SlangPassThrough(i)));
-        }
-
-        // Generic downstream tool
-        m_downstreamArgs.addName("downstream");
-        // Generic downstream linker
-        m_downstreamArgs.addName("linker");
-    }
 }
 
 SlangResult EndToEndCompileRequest::executeActionsInner()