Add utility to trace creation of problematic IRInsts to assist LLM in debugging (#7820)

* Initial plan

* Add SLANG_DEBUG_IR_BREAK environment variable support

Co-authored-by: csyonghe <2652293+csyonghe@users.noreply.github.com>

* Apply code formatting to SLANG_DEBUG_IR_BREAK implementation

Co-authored-by: csyonghe <2652293+csyonghe@users.noreply.github.com>

* Improve stack trace debugging with -rdynamic flag and backtrace_symbols

Co-authored-by: csyonghe <2652293+csyonghe@users.noreply.github.com>

* Address PR feedback: use PlatformUtil::getEnvironmentVariable, remove -rdynamic flag, and delete fallback branch

Co-authored-by: csyonghe <2652293+csyonghe@users.noreply.github.com>

* Address PR feedback: simplify env var parsing, move backtrace to PlatformUtil, use #if for SLANG_LINUX_FAMILY

Co-authored-by: csyonghe <2652293+csyonghe@users.noreply.github.com>

* Address PR feedback: remove unneeded include, make backtrace() more generic by removing uid parameter

Co-authored-by: csyonghe <2652293+csyonghe@users.noreply.github.com>

* Fix and clone source tracking.

* Add python script to dump traces.

* Update instructions.

* Batch calls to addr2line

* Cleanup claude instructions.

* update claude action.

* Remove duplicated build instructions from claude.yml workflow

Co-authored-by: csyonghe <2652293+csyonghe@users.noreply.github.com>

* fix build error.

* Fix build errors

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: csyonghe <2652293+csyonghe@users.noreply.github.com>
Co-authored-by: Yong He <yonghe@outlook.com>
Co-authored-by: Gangzheng Tong <tonggangzheng@gmail.com>

7 files changed, 90 insertions, 2 deletions

diff --git a/source/core/slang-platform.cpp b/source/core/slang-platform.cpp
index 2c2bdd25e..aab1f3044 100644
--- a/source/core/slang-platform.cpp
+++ b/source/core/slang-platform.cpp
@@ -15,6 +15,11 @@
 #include <dlfcn.h>
 #endif
 
+
+#if SLANG_LINUX_FAMILY
+#include <execinfo.h>
+#endif
+
 namespace Slang
 {
 // SharedLibrary
@@ -331,4 +336,25 @@ static const PlatformFlags s_familyFlags[int(PlatformFamily::CountOf)] = {
 #endif
 }
 
+/* static */ void PlatformUtil::backtrace()
+{
+#if SLANG_LINUX_FAMILY
+    // Print stack trace for debugging assistance
+    void* stackTrace[64];
+    int stackDepth = ::backtrace(stackTrace, 64);
+    char** symbols = ::backtrace_symbols(stackTrace, stackDepth);
+    if (symbols)
+    {
+        for (int i = 0; i < stackDepth; ++i)
+        {
+            fprintf(stdout, "%s\n", symbols[i]);
+        }
+        free(symbols);
+    }
+    fprintf(stdout, "\n");
+#else
+    fprintf(stdout, "Stack trace not available on this platform.\n");
+#endif
+}
+
 } // namespace Slang
diff --git a/source/core/slang-platform.h b/source/core/slang-platform.h
index 0b97aca6d..04559cbcf 100644
--- a/source/core/slang-platform.h
+++ b/source/core/slang-platform.h
@@ -150,6 +150,10 @@ struct PlatformUtil
     /// @param text Text to be displayed in 'debugger output'
     /// @return SLANG_E_NOT_AVAILABLE if not on this platform, and potentially other errors
     static SlangResult outputDebugMessage(const char* text);
+
+    /// Print a stack trace to stderr for debugging purposes.
+    /// Only available on Linux family platforms.
+    static void backtrace();
 };
 
 #ifndef _MSC_VER
diff --git a/source/slang/slang-ir-clone.cpp b/source/slang/slang-ir-clone.cpp
index 74a972c1d..a7b68efe2 100644
--- a/source/slang/slang-ir-clone.cpp
+++ b/source/slang/slang-ir-clone.cpp
@@ -54,6 +54,10 @@ IRInst* cloneInstAndOperands(IRCloneEnv* env, IRBuilder* builder, IRInst* oldIns
     SLANG_ASSERT(builder);
     SLANG_ASSERT(oldInst);
 
+#if SLANG_ENABLE_IR_BREAK_ALLOC
+    _debugSetInstBeingCloned(oldInst->_debugUID);
+    SLANG_DEFER(_debugResetInstBeingCloned());
+#endif
     // We start by mapping the type of the orignal instruction
     // to its replacement value, if any.
     //
diff --git a/source/slang/slang-ir-link.cpp b/source/slang/slang-ir-link.cpp
index b874b9f28..a3466c8c7 100644
--- a/source/slang/slang-ir-link.cpp
+++ b/source/slang/slang-ir-link.cpp
@@ -1332,6 +1332,11 @@ IRInst* cloneInst(
     IRInst* originalInst,
     IROriginalValuesForClone const& originalValues)
 {
+#if SLANG_ENABLE_IR_BREAK_ALLOC
+    _debugSetInstBeingCloned(originalInst->_debugUID);
+    SLANG_DEFER(_debugResetInstBeingCloned());
+#endif
+
     switch (originalInst->getOp())
     {
         // We need to special-case any instruction that is not
@@ -1427,7 +1432,6 @@ IRInst* cloneInst(
     }
     auto funcType = cloneType(context, originalInst->getFullType());
     context->builder = oldBuilder;
-
     IRInst* clonedInst = builder->createIntrinsicInst(
         funcType,
         originalInst->getOp(),
diff --git a/source/slang/slang-ir.cpp b/source/slang/slang-ir.cpp
index 3c2d5d2d1..63d3766ab 100644
--- a/source/slang/slang-ir.cpp
+++ b/source/slang/slang-ir.cpp
@@ -2,6 +2,7 @@
 #include "slang-ir.h"
 
 #include "../core/slang-basic.h"
+#include "../core/slang-platform.h"
 #include "../core/slang-writer.h"
 #include "slang-ir-dominators.h"
 #include "slang-ir-insts.h"
@@ -1738,8 +1739,21 @@ void IRBuilder::_maybeSetSourceLoc(IRInst* inst)
 }
 
 #if SLANG_ENABLE_IR_BREAK_ALLOC
-SLANG_API uint32_t _slangIRAllocBreak = 0xFFFFFFFF;
+uint32_t _slangIRAllocBreak = 0xFFFFFFFF;
+bool _slangIRPrintStackAtBreak = false;
 static bool _slangIRAllocBreakFirst = true;
+static uint32_t _slangInstBeingCloned = 0xFFFFFFFF;
+
+void _debugSetInstBeingCloned(uint32_t uid)
+{
+    _slangInstBeingCloned = uid;
+}
+
+void _debugResetInstBeingCloned()
+{
+    _slangInstBeingCloned = 0xFFFFFFFF;
+}
+
 uint32_t& _debugGetIRAllocCounter()
 {
     static uint32_t counter = 0;
@@ -1758,6 +1772,20 @@ uint32_t _debugGetAndIncreaseInstCounter()
 #if _WIN32 && defined(_MSC_VER)
         __debugbreak();
 #endif
+        if (_slangIRPrintStackAtBreak)
+        {
+            fprintf(stdout, "BEGIN IR Trace\nInstruction #%u created at:\n", _slangIRAllocBreak);
+            PlatformUtil::backtrace();
+            if (_slangInstBeingCloned != 0xFFFFFFFF)
+            {
+                fprintf(
+                    stdout,
+                    "Inst #%u is a clone of Inst #%u.\n",
+                    _slangIRAllocBreak,
+                    _slangInstBeingCloned);
+            }
+            fprintf(stdout, "END IR Trace\n");
+        }
     }
     return _debugGetIRAllocCounter()++;
 }
diff --git a/source/slang/slang-ir.h b/source/slang/slang-ir.h
index 54fc3d8de..0f4da4f0d 100644
--- a/source/slang/slang-ir.h
+++ b/source/slang/slang-ir.h
@@ -2640,6 +2640,10 @@ bool isMovableInst(IRInst* inst);
 
 #if SLANG_ENABLE_IR_BREAK_ALLOC
 uint32_t& _debugGetIRAllocCounter();
+extern uint32_t _slangIRAllocBreak;
+extern bool _slangIRPrintStackAtBreak;
+void _debugSetInstBeingCloned(uint32_t uid);
+void _debugResetInstBeingCloned();
 #endif
 
 // TODO: Ellie, comment and move somewhere more appropriate?
diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp
index f65681e4b..29ca2328b 100644
--- a/source/slang/slang.cpp
+++ b/source/slang/slang.cpp
@@ -4,8 +4,10 @@
 #include "../core/slang-castable.h"
 #include "../core/slang-io.h"
 #include "../core/slang-performance-profiler.h"
+#include "../core/slang-platform.h"
 #include "../core/slang-shared-library.h"
 #include "../core/slang-string-util.h"
+#include "../core/slang-string.h"
 #include "../core/slang-type-convert-util.h"
 #include "../core/slang-type-text-util.h"
 // Artifact
@@ -24,6 +26,7 @@
 #include "slang-check.h"
 #include "slang-doc-ast.h"
 #include "slang-doc-markdown-writer.h"
+#include "slang-ir.h"
 #include "slang-lookup.h"
 #include "slang-lower-to-ir.h"
 #include "slang-mangle.h"
@@ -160,6 +163,21 @@ void Session::init()
 {
     SLANG_ASSERT(BaseTypeInfo::check());
 
+#if SLANG_ENABLE_IR_BREAK_ALLOC
+    // Read environment variable for IR debugging
+    StringBuilder irBreakEnv;
+    if (SLANG_SUCCEEDED(PlatformUtil::getEnvironmentVariable(
+            UnownedStringSlice("SLANG_DEBUG_IR_BREAK"),
+            irBreakEnv)))
+    {
+        String envValue = irBreakEnv.produceString();
+        if (envValue.getLength())
+        {
+            _slangIRAllocBreak = stringToInt(envValue);
+            _slangIRPrintStackAtBreak = true;
+        }
+    }
+#endif
 
     _initCodeGenTransitionMap();