Support for PDB output from DXC (#2693)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Add versioning to CompileOptions for DownstreamCompiler so we can add new options without breaking binary interface.

* Add support for debug info format to API/command line processing.

* Small simplification.

* Add support for adding PDB output from a compilation.

* Use builtin offset of directly.

* Fix typo in debug.

12 files changed, 198 insertions, 19 deletions

diff --git a/source/compiler-core/slang-artifact-desc-util.cpp b/source/compiler-core/slang-artifact-desc-util.cpp
index 7e274794f..fff68738c 100644
--- a/source/compiler-core/slang-artifact-desc-util.cpp
+++ b/source/compiler-core/slang-artifact-desc-util.cpp
@@ -219,6 +219,7 @@ SLANG_HIERARCHICAL_ENUM(ArtifactKind, SLANG_ARTIFACT_KIND, SLANG_ARTIFACT_KIND_E
         x(CompileResults, Base) \
         x(Metadata, Base) \
             x(DebugInfo, Metadata) \
+                x(PdbDebugInfo, DebugInfo) \
             x(Diagnostics, Metadata) \
         x(Miscellaneous, Base) \
             x(Log, Miscellaneous) \
@@ -549,6 +550,12 @@ static const KindExtension g_cpuKindExts[] =
         return ArtifactDesc::make(ArtifactKind::Assembly, ArtifactPayload::HostCPU);
     }
 
+    if (slice == toSlice("pdb"))
+    {
+        // Program database
+        return ArtifactDesc::make(ArtifactKind::Assembly, ArtifactPayload::PdbDebugInfo);
+    }
+
     for (const auto& kindExt : g_cpuKindExts)
     {
         if (slice == kindExt.ext)
@@ -613,6 +620,8 @@ static UnownedStringSlice _getPayloadExtension(ArtifactPayload payload)
         
         case Payload::MetalAIR:     return toSlice("air");
 
+        case Payload::PdbDebugInfo: return toSlice("pdb");
+
         default: break;
     }
     return UnownedStringSlice();
diff --git a/source/compiler-core/slang-artifact.h b/source/compiler-core/slang-artifact.h
index 222b902a5..df0bb21d5 100644
--- a/source/compiler-core/slang-artifact.h
+++ b/source/compiler-core/slang-artifact.h
@@ -178,6 +178,8 @@ enum class ArtifactPayload : uint8_t
     Log,            ///< Log file
     Lock,           ///< Typically some kind of 'lock' file. Contents is typically not important.
 
+    PdbDebugInfo,   ///< PDB debug info
+
     CountOf,
 };
 
diff --git a/source/compiler-core/slang-downstream-compiler.h b/source/compiler-core/slang-downstream-compiler.h
index b1f1b7b11..3cd27138f 100644
--- a/source/compiler-core/slang-downstream-compiler.h
+++ b/source/compiler-core/slang-downstream-compiler.h
@@ -252,6 +252,9 @@ struct DownstreamCompileOptions
     /// NOTE! Not all downstream compilers can use the fileSystemExt/sourceManager. This option will be ignored in those scenarios.
     ISlangFileSystemExt* fileSystemExt = nullptr;
     SourceManager* sourceManager = nullptr;
+
+    // The debug info format to use. 
+    SlangDebugInfoFormat m_debugInfoFormat = SLANG_DEBUG_INFO_FORMAT_DEFAULT;
 };
 
 #define SLANG_ALIAS_DEPRECIATED_VERSION(name, id, firstField, lastField) \
diff --git a/source/compiler-core/slang-dxc-compiler.cpp b/source/compiler-core/slang-dxc-compiler.cpp
index 9df2a4f3f..248d8d118 100644
--- a/source/compiler-core/slang-dxc-compiler.cpp
+++ b/source/compiler-core/slang-dxc-compiler.cpp
@@ -451,6 +451,8 @@ SlangResult DXCDownstreamCompiler::compile(const CompileOptions& inOptions, IArt
             break;
     }
 
+
+
     switch (options.optimizationLevel)
     {
         default:
@@ -549,7 +551,7 @@ SlangResult DXCDownstreamCompiler::compile(const CompileOptions& inOptions, IArt
 
     DxcIncludeHandler includeHandler(&searchDirectories, options.fileSystemExt, options.sourceManager);
 
-    ComPtr<IDxcOperationResult> dxcResult;
+    ComPtr<IDxcOperationResult> dxcOperationResult;
     SLANG_RETURN_ON_FAIL(dxcCompiler->Compile(dxcSourceBlob,
         wideSourcePath.begin(),
         wideEntryPointName.begin(),
@@ -559,13 +561,13 @@ SlangResult DXCDownstreamCompiler::compile(const CompileOptions& inOptions, IArt
         nullptr,            // `#define`s
         0,                  // `#define` count
         &includeHandler,    // `#include` handler
-        dxcResult.writeRef()));
+        dxcOperationResult.writeRef()));
 
     auto diagnostics = ArtifactDiagnostics::create();
     
     ComPtr<IDxcBlob> dxcResultBlob;
 
-    SLANG_RETURN_ON_FAIL(_handleOperationResult(dxcResult, diagnostics, dxcResultBlob));
+    SLANG_RETURN_ON_FAIL(_handleOperationResult(dxcOperationResult, diagnostics, dxcResultBlob));
 
     // If we have libraries then we need to link...
     if (libraries.getCount())
@@ -629,6 +631,11 @@ SlangResult DXCDownstreamCompiler::compile(const CompileOptions& inOptions, IArt
         ComPtr<IDxcBlob> linkedBlob;
         SLANG_RETURN_ON_FAIL(_handleOperationResult(linkDxcResult, diagnostics, linkedBlob));
 
+        // When we've linked we make that the overall operation result
+        // As presumably it can contain pdb and perhaps other information
+        dxcOperationResult = linkDxcResult;
+
+        // Set the result blob
         dxcResultBlob = linkedBlob;
     }
 
@@ -639,6 +646,42 @@ SlangResult DXCDownstreamCompiler::compile(const CompileOptions& inOptions, IArt
         artifact->addRepresentationUnknown((ISlangBlob*)dxcResultBlob.get());
     }
 
+    // If asking for PDB extract it.
+    if (options.m_debugInfoFormat == SLANG_DEBUG_INFO_FORMAT_PDB)
+    {
+        ComPtr<IDxcResult> dxcResult;
+        if (SLANG_SUCCEEDED(dxcOperationResult->QueryInterface(dxcResult.writeRef())))
+        {
+            if (dxcResult->HasOutput(DXC_OUT_PDB))
+            {
+                ComPtr<IDxcBlob> pdbBlob;
+                ComPtr<IDxcBlobWide> nameBlob;
+
+                if (SLANG_SUCCEEDED(dxcResult->GetOutput(DXC_OUT_PDB, __uuidof(pdbBlob), (void**)pdbBlob.writeRef(), nameBlob.writeRef())))
+                {
+                    auto pdbArtifact = ArtifactUtil::createArtifact(ArtifactDesc::make(ArtifactDesc::Kind::BinaryFormat, ArtifactDesc::Payload::PdbDebugInfo));
+
+                    if (nameBlob)
+                    {
+                        const auto wideName = (const WCHAR*)nameBlob->GetBufferPointer();
+
+                        const auto name = String::fromWString(wideName);
+                        if (name.getLength())
+                        {
+                            // Set the name on the artifact. This is the name that must be used for the PDB to be loadable as a file by other tooling.
+                            pdbArtifact->setName(name.getBuffer());
+                        }
+                    }
+
+                    pdbArtifact->addRepresentationUnknown((ISlangBlob*)pdbBlob.get());
+
+                    // Associate it
+                    artifact->addAssociated(pdbArtifact);
+                }
+            }
+        }
+    }
+
     *outArtifact = artifact.detach();
     return SLANG_OK;
 }
diff --git a/source/compiler-core/slang-visual-studio-compiler-util.cpp b/source/compiler-core/slang-visual-studio-compiler-util.cpp
index 2d5b37c50..32b4d7f25 100644
--- a/source/compiler-core/slang-visual-studio-compiler-util.cpp
+++ b/source/compiler-core/slang-visual-studio-compiler-util.cpp
@@ -66,7 +66,7 @@ static void _addFile(const String& path, const ArtifactDesc& desc, IOSFileArtifa
     if (flags & ProductFlag::Debug)
     {
         // TODO(JS): Could try and determine based on debug information
-        _addFile(modulePath + ".pdb", ArtifactDesc::make(ArtifactKind::BinaryFormat, ArtifactPayload::DebugInfo, targetDesc), lockFile, outArtifacts);
+        _addFile(modulePath + ".pdb", ArtifactDesc::make(ArtifactKind::BinaryFormat, ArtifactPayload::PdbDebugInfo, targetDesc), lockFile, outArtifacts);
     }
 
     return SLANG_OK;
diff --git a/source/core/slang-type-text-util.cpp b/source/core/slang-type-text-util.cpp
index bfa193486..2ba3011d8 100644
--- a/source/core/slang-type-text-util.cpp
+++ b/source/core/slang-type-text-util.cpp
@@ -1,5 +1,6 @@
 
 #include "slang-type-text-util.h"
+#include "slang-array-view.h"
 
 #include "slang-string-util.h"
 
@@ -30,6 +31,14 @@ namespace Slang
         x(nvrtc, NVRTC) \
         x(llvm, LLVM)
 
+#define SLANG_DEBUG_INFO_FORMATS(x) \
+    x(default-format, DEFAULT) \
+    x(c7, C7) \
+    x(pdb, PDB) \
+    x(stabs, STABS) \
+    x(coff, COFF) \
+    x(dwarf, DWARF) 
+
 namespace { // anonymous
 
 struct ScalarTypeInfo
@@ -109,6 +118,40 @@ static const ArchiveTypeInfo s_archiveTypeInfos[] =
     return SLANG_ARCHIVE_TYPE_UNDEFINED;
 }
 
+struct DebugInfoFormatTable
+{
+    UnownedStringSlice entries[SLANG_DEBUG_INFO_FORMAT_COUNT_OF];
+
+    static DebugInfoFormatTable _makeTable()
+    {
+        DebugInfoFormatTable dst;
+#define SLANG_DEBUG_INFO_FORMAT_ENTRY(name, value) \
+        dst.entries[SLANG_DEBUG_INFO_FORMAT_##value] = toSlice(#name);
+        SLANG_DEBUG_INFO_FORMATS(SLANG_DEBUG_INFO_FORMAT_ENTRY)
+        return dst;
+    }
+
+    static Index findIndex(const UnownedStringSlice slice) { return makeConstArrayView(table.entries).indexOf(slice); }
+    static UnownedStringSlice getSlice(SlangDebugInfoFormat format) { return table.entries[Index(format)]; }
+
+    static const DebugInfoFormatTable table;
+};
+
+/* static */const DebugInfoFormatTable DebugInfoFormatTable::table = DebugInfoFormatTable::_makeTable();
+
+/* static */SlangResult TypeTextUtil::findDebugInfoFormat(const Slang::UnownedStringSlice& text, SlangDebugInfoFormat& out)
+{
+    const auto index = DebugInfoFormatTable::findIndex(text);
+    if (index >= 0)
+    {
+        out = SlangDebugInfoFormat(index);
+        return SLANG_OK;
+    }
+    return SLANG_FAIL;
+}
+
+/* static */UnownedStringSlice TypeTextUtil::getDebugInfoFormatName(SlangDebugInfoFormat format) { return DebugInfoFormatTable::getSlice(format); }
+
 /* static */UnownedStringSlice TypeTextUtil::getScalarTypeName(slang::TypeReflection::ScalarType scalarType)
 {    
     typedef slang::TypeReflection::ScalarType ScalarType;
diff --git a/source/core/slang-type-text-util.h b/source/core/slang-type-text-util.h
index f5534ec72..df9b83013 100644
--- a/source/core/slang-type-text-util.h
+++ b/source/core/slang-type-text-util.h
@@ -18,6 +18,12 @@ struct TypeTextUtil
         // Converts text to scalar type. Returns 'none' if not determined
     static slang::TypeReflection::ScalarType findScalarType(const Slang::UnownedStringSlice& text);
 
+        /// Given a slice finds the associated debug info format
+    static SlangResult findDebugInfoFormat(const Slang::UnownedStringSlice& text, SlangDebugInfoFormat& out);
+
+        /// Get the text name for a format
+    static UnownedStringSlice getDebugInfoFormatName(SlangDebugInfoFormat format);
+
         /// As human readable text
     static UnownedStringSlice getPassThroughAsHumanText(SlangPassThrough type);
         /// Gets pass through from human text (as from getPassThroughAsHumanText)
diff --git a/source/slang/slang-api.cpp b/source/slang/slang-api.cpp
index 45c583060..a03aa9071 100644
--- a/source/slang/slang-api.cpp
+++ b/source/slang/slang-api.cpp
@@ -385,6 +385,14 @@ SLANG_API void spSetDebugInfoLevel(
     request->setDebugInfoLevel(level);
 }
 
+SLANG_API void spSetDebugInfoFormat(
+    slang::ICompileRequest* request,
+    SlangDebugInfoFormat format)
+{
+    SLANG_ASSERT(request);
+    request->setDebugInfoFormat(format);
+}
+
 SLANG_API void spSetOptimizationLevel(
     slang::ICompileRequest*    request,
     SlangOptimizationLevel  level)
diff --git a/source/slang/slang-compiler.h b/source/slang/slang-compiler.h
index f377d4882..e6b173e2a 100755
--- a/source/slang/slang-compiler.h
+++ b/source/slang/slang-compiler.h
@@ -130,6 +130,19 @@ namespace Slang
         Maximal     = SLANG_DEBUG_INFO_LEVEL_MAXIMAL,
     };
 
+    enum class DebugInfoFormat : SlangDebugInfoFormatIntegral
+    {
+        Default     = SLANG_DEBUG_INFO_FORMAT_DEFAULT,
+        C7          = SLANG_DEBUG_INFO_FORMAT_C7,
+        Pdb         = SLANG_DEBUG_INFO_FORMAT_PDB,
+
+        Stabs       = SLANG_DEBUG_INFO_FORMAT_STABS,
+        Coff        = SLANG_DEBUG_INFO_FORMAT_COFF,
+        Dwarf       = SLANG_DEBUG_INFO_FORMAT_DWARF,
+
+        CountOf     = SLANG_DEBUG_INFO_FORMAT_COUNT_OF,
+    };
+
     enum class OptimizationLevel : SlangOptimizationLevelIntegral
     {
         None    = SLANG_OPTIMIZATION_LEVEL_NONE,
@@ -1884,6 +1897,7 @@ namespace Slang
         MatrixLayoutMode getDefaultMatrixLayoutMode() { return defaultMatrixLayoutMode; }
 
         DebugInfoLevel debugInfoLevel = DebugInfoLevel::None;
+        DebugInfoFormat debugInfoFormat = DebugInfoFormat::Default;
 
         OptimizationLevel optimizationLevel = OptimizationLevel::Default;
 
@@ -2595,6 +2609,7 @@ namespace Slang
             SlangSeverity overrideSeverity) SLANG_OVERRIDE;
         virtual SLANG_NO_THROW SlangDiagnosticFlags SLANG_MCALL getDiagnosticFlags() SLANG_OVERRIDE;
         virtual SLANG_NO_THROW void SLANG_MCALL setDiagnosticFlags(SlangDiagnosticFlags flags) SLANG_OVERRIDE;
+        virtual SLANG_NO_THROW void SLANG_MCALL setDebugInfoFormat(SlangDebugInfoFormat format) SLANG_OVERRIDE;
 
         EndToEndCompileRequest(
             Session* session);
diff --git a/source/slang/slang-diagnostic-defs.h b/source/slang/slang-diagnostic-defs.h
index aaf09a8be..2401b6e58 100644
--- a/source/slang/slang-diagnostic-defs.h
+++ b/source/slang/slang-diagnostic-defs.h
@@ -130,6 +130,7 @@ DIAGNOSTIC(    95, Error, unknownLibraryKind, "unknown library kind '$0'")
 DIAGNOSTIC(    96, Error, kindNotLinkable, "not a known linkable kind '$0'")
 DIAGNOSTIC(    97, Error, libraryDoesNotExist, "library '$0' does not exist")
 DIAGNOSTIC(    98, Error, cannotAccessAsBlob, "cannot access as a blob")
+DIAGNOSTIC(    99, Error, unknownDebugOption, "unknown debug option, known options are ($0)")
 
 //
 // 001xx - Downstream Compilers
diff --git a/source/slang/slang-options.cpp b/source/slang/slang-options.cpp
index 5ac0d7663..60bc087de 100644
--- a/source/slang/slang-options.cpp
+++ b/source/slang/slang-options.cpp
@@ -1497,25 +1497,69 @@ struct OptionsParser
                  
                     compileRequest->setOptimizationLevel(level);
                 }
-
                 // Note: unlike with `-O` above, we have to consider that other
                 // options might have names that start with `-g` and so cannot
                 // just detect it as a prefix.
-                else if( argValue == "-g" || argValue == "-g2" )
-                {
-                    compileRequest->setDebugInfoLevel(SLANG_DEBUG_INFO_LEVEL_STANDARD);
-                }
-                else if( argValue == "-g0" )
-                {
-                    compileRequest->setDebugInfoLevel(SLANG_DEBUG_INFO_LEVEL_NONE);
-                }
-                else if( argValue == "-g1" )
-                {
-                    compileRequest->setDebugInfoLevel(SLANG_DEBUG_INFO_LEVEL_MINIMAL);
-                }
-                else if( argValue == "-g3" )
+                else if (argValue.startsWith("-g"))
                 {
-                    compileRequest->setDebugInfoLevel(SLANG_DEBUG_INFO_LEVEL_MAXIMAL);
+                    if (argValue == toSlice("-g"))
+                    {
+                        // The default is standard
+                        compileRequest->setDebugInfoLevel(SLANG_DEBUG_INFO_LEVEL_STANDARD);
+                    }
+                    else if (argValue.getLength() == 3 && argValue[2] >= '0' && argValue[2] <= '3')
+                    {
+                        // Extract the digit into an index
+                        const Index levelIndex = argValue[2] - '0';
+                        SLANG_ASSERT(levelIndex >= 0 && levelIndex <= 3);
+
+                        // Map indices to enum values
+                        const SlangDebugInfoLevel levels[] = 
+                        {
+                            SLANG_DEBUG_INFO_LEVEL_NONE,
+                            SLANG_DEBUG_INFO_LEVEL_MINIMAL,
+                            SLANG_DEBUG_INFO_LEVEL_STANDARD,
+                            SLANG_DEBUG_INFO_LEVEL_MAXIMAL
+                        };
+
+                        const auto level = levels[levelIndex];
+                        compileRequest->setDebugInfoLevel(level);
+                    }
+                    else
+                    {
+                        // Perhaps it's trying to specify a format
+                        auto formatName = argValue.getUnownedSlice().tail(2);
+
+                        SlangDebugInfoFormat format;
+                        if (SLANG_FAILED(TypeTextUtil::findDebugInfoFormat(formatName, format)))
+                        {
+                            List<String> debugOptions;
+
+                            debugOptions.add(toSlice("-g"));
+
+                            for (Int i = 0; i <= 3; ++i)
+                            {
+                                StringBuilder buf;
+                                buf << toSlice("-g") << i;
+                                debugOptions.add(buf);
+                            }
+
+                            for (Index i = 0; i < SLANG_DEBUG_INFO_FORMAT_COUNT_OF; ++i)
+                            {
+                                StringBuilder buf;
+                                buf << toSlice("-g") << TypeTextUtil::getDebugInfoFormatName(SlangDebugInfoFormat(i));
+                                debugOptions.add(buf);
+                            }
+
+                            StringBuilder buf;
+                            StringUtil::join(debugOptions, toSlice(", "), buf);
+
+                            sink->diagnose(arg.loc, Diagnostics::unknownDebugOption, buf);
+                            return SLANG_FAIL;
+                        }
+                       
+                        compileRequest->setDebugInfoFormat(format);
+                    }
                 }
                 else if( argValue == "-default-image-format-unknown" )
                 {
diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp
index 3a11d3831..da8fcdcd6 100644
--- a/source/slang/slang.cpp
+++ b/source/slang/slang.cpp
@@ -4737,6 +4737,11 @@ void EndToEndCompileRequest::setDebugInfoLevel(SlangDebugInfoLevel level)
     getLinkage()->debugInfoLevel = DebugInfoLevel(level);
 }
 
+void EndToEndCompileRequest::setDebugInfoFormat(SlangDebugInfoFormat format)
+{
+    getLinkage()->debugInfoFormat = DebugInfoFormat(format);
+}
+
 void EndToEndCompileRequest::setOptimizationLevel(SlangOptimizationLevel level)
 {
     getLinkage()->optimizationLevel = OptimizationLevel(level);