Add command line option for separate debug info (#7178)

* Add command line option for separate debug info

Add command line arg -separate-debug-info which, if provided, produces
both a .spv and a .dbg.spv file. The .dbg.spv file contains full debug
info and the .spv file has all debug info stripped out.

Also add a DebugBuildIdentifier instruction to store a unique hash in
both the output files, so they can be more easily matched together.

A matching API is provided to allow using the Slang API to retrieve a
base and debug SPIRV as well as the debug build identifier string.

1 files changed, 328 insertions, 13 deletions

diff --git a/source/slang/slang-emit.cpp b/source/slang/slang-emit.cpp
index 88533d938..c0c1ba75a 100644
--- a/source/slang/slang-emit.cpp
+++ b/source/slang/slang-emit.cpp
@@ -356,6 +356,7 @@ void calcRequiredLoweringPassSet(
     case kIROp_DebugScope:
     case kIROp_DebugNoScope:
     case kIROp_DebugFunction:
+    case kIROp_DebugBuildIdentifier:
         result.debugInfo = true;
         break;
     case kIROp_ResultType:
@@ -616,6 +617,21 @@ static void unexportNonEmbeddableIR(CodeGenTarget target, IRModule* irModule)
     }
 }
 
+// Helper function to convert a 20 byte SHA1 to a hexadecimal string,
+// needed for the build identifier instruction.
+String getBuildIdentifierString(ComponentType* component)
+{
+    ComPtr<ISlangBlob> hashBlob;
+    component->getEntryPointHash(0, 0, hashBlob.writeRef());
+
+    const uint8_t* data = reinterpret_cast<const uint8_t*>(hashBlob->getBufferPointer());
+    size_t size = hashBlob->getBufferSize();
+    StringBuilder sb;
+    for (size_t i = 0; i < size; ++i)
+        sb << StringUtil::makeStringWithFormat("%02x", data[i]);
+    return sb.produceString();
+}
+
 Result linkAndOptimizeIR(
     CodeGenContext* codeGenContext,
     LinkingAndOptimizationOptions const& options,
@@ -644,6 +660,19 @@ Result linkAndOptimizeIR(
     auto irModule = outLinkedIR.module;
     auto irEntryPoints = outLinkedIR.entryPoints;
 
+    // For now, only emit the debug build identifier if separate debug info is enabled
+    // and only if there are targets.
+    // TODO: We will ultimately need to change this to always emit the instruction.
+    if (targetCompilerOptions.shouldEmitSeparateDebugInfo())
+    {
+        // Build identifier is a hash of the source code and compile options.
+        String buildIdentifier = getBuildIdentifierString(codeGenContext->getProgram());
+        int buildIdentifierFlags = 0;
+        IRBuilder builder(irModule);
+        builder.setInsertInto(irModule->getModuleInst());
+        builder.emitDebugBuildIdentifier(buildIdentifier.getUnownedSlice(), buildIdentifierFlags);
+    }
+
 #if 0
     dumpIRIfEnabled(codeGenContext, irModule, "LINKED");
 #endif
@@ -2104,22 +2133,256 @@ SlangResult emitSPIRVFromIR(
     const List<IRFunc*>& irEntryPoints,
     List<uint8_t>& spirvOut);
 
-SlangResult emitSPIRVForEntryPointsDirectly(
-    CodeGenContext* codeGenContext,
-    ComPtr<IArtifact>& outArtifact)
+// Helper class to assist in stepping through the SPIRV instructions.
+class SpirvInstructionHelper
 {
-    // Outside because we want to keep IR in scope whilst we are processing emits
-    LinkedIR linkedIR;
-    LinkingAndOptimizationOptions linkingAndOptimizationOptions;
-    SLANG_RETURN_ON_FAIL(
-        linkAndOptimizeIR(codeGenContext, linkingAndOptimizationOptions, linkedIR));
+public:
+    // The index of the SPIRV words in the blob.
+    // The first 5 words are the header as defined by the SPIRV spec.
+    enum SpvWordIndex
+    {
+        SPV_INDEX_MAGIC_NUMBER,
+        SPV_INDEX_VERSION_NUMBER,
+        SPV_INDEX_GENERATOR_NUMBER,
+        SPV_INDEX_BOUND,
+        SPV_INDEX_SCHEMA,
+        SPV_INDEX_INSTRUCTION_START,
+    };
+
+    // An instruction in the SPIRV blob. This points to the first word of the instruction.
+    struct SpvInstruction
+    {
+        SpvInstruction(SpvWord* word)
+            : word(word)
+        {
+        }
 
-    auto irModule = linkedIR.module;
-    auto irEntryPoints = linkedIR.entryPoints;
+        uint16_t getOpCode() const { return word[0] & 0xFFFF; }
+        uint16_t getWordCountForInst() const { return (word[0] >> 16) & 0xFFFF; }
+        SpvWord getOperand(uint32_t index) const { return word[index + 1]; }
+
+        // Helper function to interpret the instruction as a string and
+        // extract the string value.
+        String getStringFromInst() const
+        {
+            String result;
+            for (uint32_t i = 1; i < getWordCountForInst(); i++)
+            {
+                SpvWord op = getOperand(i);
+                for (int b = 0; b < 4; ++b)
+                {
+                    char c = (char)((op >> (b * 8)) & 0xFF);
+                    if (c == '\0')
+                        return result;
+                    result.append(c);
+                }
+            }
+            return result;
+        }
+
+        SpvWord* word = nullptr;
+    };
+
+    SpirvInstructionHelper() {}
+
+    // Load the SPIRV instructions from the artifact into a data blob that
+    // we can read.
+    SlangResult loadBlob(ComPtr<IArtifact>& artifact)
+    {
+        ComPtr<ISlangBlob> spirvBlob;
+        SlangResult res = artifact->loadBlob(ArtifactKeep::Yes, spirvBlob.writeRef());
+        if (SLANG_FAILED(res) || !spirvBlob)
+            return SLANG_FAIL;
+
+        // Populate the full array of SPIR-V words.
+        m_words.clear();
+        m_words.addRange(
+            reinterpret_cast<const SpvWord*>(spirvBlob->getBufferPointer()),
+            spirvBlob->getBufferSize() / sizeof(SpvWord));
+
+        // Populate the header words. These are the first 5 words of the SPIR-V
+        // blob and are treated differently from the rest of the instructions.
+        m_headerWords.clear();
+        m_headerWords.addRange(m_words.getBuffer(), SPV_INDEX_INSTRUCTION_START);
+
+        return SLANG_OK;
+    }
+
+    // Get the header words.
+    List<SpvWord> getHeaderWords() const { return m_headerWords; }
+
+    // Visit all SPIRV instructions (excluding header words), invoking the callback for each
+    // instruction. The callback should be a function or lambda with signature: void(const
+    // SpvInstruction&).
+    template<typename Func>
+    void visitInstructions(Func&& callback)
+    {
+        // Instructions start after the header (first 5 words)
+        constexpr size_t kHeaderWordCount =
+            static_cast<size_t>(SpvWordIndex::SPV_INDEX_INSTRUCTION_START);
+        size_t i = kHeaderWordCount;
+        while (i < (size_t)m_words.getCount())
+        {
+            SpvWord* wordPtr = m_words.getBuffer() + i;
+            SpvInstruction inst(wordPtr);
+            callback(inst);
+            uint16_t wordCount = inst.getWordCountForInst();
+            if (wordCount == 0)
+                break; // Prevent infinite loop on malformed input
+            i += wordCount;
+        }
+    }
+
+private:
+    // The full array of SPIRV words.
+    List<SpvWord> m_words;
+
+    // The header words.
+    List<SpvWord> m_headerWords;
+};
+
+// Helper function that takes an artifact populated with SPIRV instructions
+// after the spirv-opt step, and a previously created but empty
+// strippedArtifact. The artifact is unmodified, and the strippedArtifact
+// will contain all the artifact's instructions except for debug instructions.
+static SlangResult stripDbgSpirvFromArtifact(
+    ComPtr<IArtifact>& artifact,
+    ComPtr<IArtifact>& strippedArtifact)
+{
+    // Standard debug opcodes to strip out. This mimics the behavior of
+    // spirv-opt.
+    static const uint16_t debugOpCodeVals[] = {
+        SpvOpSourceContinued,
+        SpvOpSource,
+        SpvOpSourceExtension,
+        SpvOpString,
+        SpvOpName,
+        SpvOpMemberName,
+        SpvOpModuleProcessed,
+        SpvOpLine,
+        SpvOpNoLine};
+    // If the instruction is an extended instruction, then we also need
+    // to check if the instruction number is for a debug instruction as
+    // listed in slang-emit-spirv-ops-debug-info-ext.h
+    static const uint32_t debugExtInstVals[] = {
+        NonSemanticShaderDebugInfo100DebugCompilationUnit,
+        NonSemanticShaderDebugInfo100DebugTypeBasic,
+        NonSemanticShaderDebugInfo100DebugTypePointer,
+        NonSemanticShaderDebugInfo100DebugTypeQualifier,
+        NonSemanticShaderDebugInfo100DebugTypeArray,
+        NonSemanticShaderDebugInfo100DebugTypeVector,
+        NonSemanticShaderDebugInfo100DebugTypeFunction,
+        NonSemanticShaderDebugInfo100DebugTypeComposite,
+        NonSemanticShaderDebugInfo100DebugTypeMember,
+        NonSemanticShaderDebugInfo100DebugFunction,
+        NonSemanticShaderDebugInfo100DebugScope,
+        NonSemanticShaderDebugInfo100DebugNoScope,
+        NonSemanticShaderDebugInfo100DebugInlinedAt,
+        NonSemanticShaderDebugInfo100DebugLocalVariable,
+        NonSemanticShaderDebugInfo100DebugInlinedVariable,
+        NonSemanticShaderDebugInfo100DebugDeclare,
+        NonSemanticShaderDebugInfo100DebugValue,
+        NonSemanticShaderDebugInfo100DebugExpression,
+        NonSemanticShaderDebugInfo100DebugSource,
+        NonSemanticShaderDebugInfo100DebugFunctionDefinition,
+        NonSemanticShaderDebugInfo100DebugSourceContinued,
+        NonSemanticShaderDebugInfo100DebugLine,
+        NonSemanticShaderDebugInfo100DebugEntryPoint,
+        NonSemanticShaderDebugInfo100DebugTypeMatrix,
+    };
+
+    // Hash sets for easier lookup.
+    HashSet<uint16_t> debugOpCodes;
+    for (auto val : debugOpCodeVals)
+        debugOpCodes.add(val);
+    HashSet<uint32_t> debugExtInstNumbers;
+    for (auto val : debugExtInstVals)
+        debugExtInstNumbers.add(val);
+
+    SpirvInstructionHelper spirvInstructionHelper;
+    SLANG_RETURN_ON_FAIL(spirvInstructionHelper.loadBlob(artifact));
+
+    auto headerWords = spirvInstructionHelper.getHeaderWords();
+
+    List<uint8_t> spirvWordsList;
+    spirvWordsList.addRange(
+        reinterpret_cast<const uint8_t*>(headerWords.getBuffer()),
+        headerWords.getCount() * sizeof(SpvWord));
+
+    // First find the DebugBuildIdentifier instruction, and keep track of which string
+    // it refers to, this string needs to be kept in the final output.
+    SpvWord debugStringId = 0;
+    spirvInstructionHelper.visitInstructions(
+        [&](const SpirvInstructionHelper::SpvInstruction& inst)
+        {
+            if (inst.getOpCode() == SpvOpExtInst)
+            {
+                if (inst.getOperand(3) == NonSemanticShaderDebugInfo100DebugBuildIdentifier)
+                {
+                    debugStringId = inst.getOperand(4);
+                    return;
+                }
+            }
+        });
+
+    // Iterate over the instructions from the artifact and add them to the list
+    // only if they are not debug instructions. We also get the debug build hash
+    // to use as the filename for the debug spirv file.
+    String debugBuildHash;
+    spirvInstructionHelper.visitInstructions(
+        [&](const SpirvInstructionHelper::SpvInstruction& inst)
+        {
+            if (debugOpCodes.contains(inst.getOpCode()))
+            {
+                // We can only strip strings if they are not being used by the
+                // DebugBuildIdentifier instruction.
+                bool foundDebugString = false;
+                if (inst.getOpCode() == SpvOpString && inst.getOperand(0) == debugStringId)
+                {
+                    debugBuildHash = inst.getStringFromInst();
+                    foundDebugString = true;
+                }
+                if (!foundDebugString)
+                    return;
+            }
+            // Also check if the instruction is an extended instruction containing DebugInfo.
+            if (inst.getOpCode() == SpvOpExtInst)
+            {
+                // Ignore this if the instruction contains DebugInfo.
+                if (debugExtInstNumbers.contains(inst.getOperand(3)))
+                    return;
+            }
+            // Otherwise this is a non-debug instruction and should be included.
+            spirvWordsList.addRange(
+                reinterpret_cast<const uint8_t*>(inst.word),
+                inst.getWordCountForInst() * sizeof(SpvWord));
+        });
 
+    // Create the stripped artifact using the above created instruction list.
+    strippedArtifact->addRepresentationUnknown(ListBlob::moveCreate(spirvWordsList));
+
+    // Set the name of the artifact to the debug build hash so it can be used
+    // as the filename for the debug spirv file.
+    artifact->setName(debugBuildHash.getBuffer());
+
+    return SLANG_OK;
+}
+
+// Helper function to create an artifact from IR used internally by
+// emitSPIRVForEntryPointsDirectly.
+static SlangResult createArtifactFromIR(
+    CodeGenContext* codeGenContext,
+    IRModule* irModule,
+    List<IRFunc*> irEntryPoints,
+    ComPtr<IArtifact>& artifact,
+    ComPtr<IArtifact>& dbgArtifact)
+{
     List<uint8_t> spirv, outSpirv;
     emitSPIRVFromIR(codeGenContext, irModule, irEntryPoints, spirv);
 
+    auto targetRequest = codeGenContext->getTargetReq();
+    auto targetCompilerOptions = targetRequest->getOptionSet();
+
 #if 0
     String optErr;
     if (SLANG_FAILED(optimizeSPIRV(spirv, optErr, outSpirv)))
@@ -2128,8 +2391,7 @@ SlangResult emitSPIRVForEntryPointsDirectly(
         spirv = _Move(outSpirv);
     }
 #endif
-    auto artifact =
-        ArtifactUtil::createArtifactForCompileTarget(asExternal(codeGenContext->getTargetFormat()));
+
     artifact->addRepresentationUnknown(ListBlob::moveCreate(spirv));
 
     IDownstreamCompiler* compiler = codeGenContext->getSession()->getOrLoadDownstreamCompiler(
@@ -2252,7 +2514,19 @@ SlangResult emitSPIRVForEntryPointsDirectly(
         auto downstreamStartTime = std::chrono::high_resolution_clock::now();
         if (SLANG_SUCCEEDED(compiler->compile(downstreamOptions, optimizedArtifact.writeRef())))
         {
-            artifact = _Move(optimizedArtifact);
+            // Check if we need to output a separate SPIRV file containing debug info. If so
+            // then strip all debug instructions from the artifact. The dbgArtifact will still
+            // contain all instructions.
+            if (targetCompilerOptions.shouldEmitSeparateDebugInfo())
+            {
+                auto strippedArtifact = ArtifactUtil::createArtifactForCompileTarget(SLANG_SPIRV);
+                SLANG_RETURN_ON_FAIL(
+                    stripDbgSpirvFromArtifact(optimizedArtifact, strippedArtifact));
+                artifact = _Move(strippedArtifact);
+                dbgArtifact = _Move(optimizedArtifact);
+            }
+            else
+                artifact = _Move(optimizedArtifact);
         }
         auto downstreamElapsedTime =
             (std::chrono::high_resolution_clock::now() - downstreamStartTime).count() * 0.000000001;
@@ -2262,8 +2536,49 @@ SlangResult emitSPIRVForEntryPointsDirectly(
             passthroughDownstreamDiagnostics(codeGenContext->getSink(), compiler, artifact));
     }
 
+    return SLANG_OK;
+}
+
+SlangResult emitSPIRVForEntryPointsDirectly(
+    CodeGenContext* codeGenContext,
+    ComPtr<IArtifact>& outArtifact)
+{
+    // Outside because we want to keep IR in scope whilst we are processing emits
+    LinkedIR linkedIR;
+    LinkingAndOptimizationOptions linkingAndOptimizationOptions;
+    SLANG_RETURN_ON_FAIL(
+        linkAndOptimizeIR(codeGenContext, linkingAndOptimizationOptions, linkedIR));
+
+    auto irModule = linkedIR.module;
+    auto irEntryPoints = linkedIR.entryPoints;
+
+    dumpIRIfEnabled(codeGenContext, irModule, "POST LINK AND OPTIMIZE");
+
+    auto targetRequest = codeGenContext->getTargetReq();
+    auto targetCompilerOptions = targetRequest->getOptionSet();
+
+    // Create the artifact containing the main SPIRV data, and the debug SPIRV
+    // data if requested by the command line arg -separate-debug-info.
+    Slang::ComPtr<Slang::IArtifact> dbgArtifact;
+    auto artifact =
+        ArtifactUtil::createArtifactForCompileTarget(asExternal(codeGenContext->getTargetFormat()));
+    SLANG_RETURN_ON_FAIL(
+        createArtifactFromIR(codeGenContext, irModule, irEntryPoints, artifact, dbgArtifact));
     ArtifactUtil::addAssociated(artifact, linkedIR.metadata);
 
+    // Associate the debug artifact with the main artifact.
+    // EndToEndCompileRequest::generateOutput will read this data
+    // and produce a .dbg.spv file for this child artifact.
+    if (targetCompilerOptions.shouldEmitSeparateDebugInfo())
+    {
+        artifact->addAssociated(dbgArtifact);
+
+        auto artifactPostEmitMetadata =
+            static_cast<ArtifactPostEmitMetadata*>(linkedIR.metadata.get());
+        artifactPostEmitMetadata->addRef();
+        artifactPostEmitMetadata->m_debugBuildIdentifier = dbgArtifact->getName();
+    }
+
     outArtifact.swap(artifact);
 
     return SLANG_OK;