Split debug info for all targets (#5732)

* Split debug info for all targets

Work towards https://github.com/shader-slang/slang/issues/5724

* release separate debug info

Closes https://github.com/shader-slang/slang/issues/5724

* Add split debug info support for MacOS

* Add SLANG_ENABLE_SPLIT_DEBUG_INFO option

* Sign and package debug info on MacOS

* Set --build-id where available

* Correct debug info installing

* Keep cpack macos signing workaround

* Neaten cmake

* Disable sccache if building split debug info on Windows

* Only repack necessary files on MacOS releases

6 files changed, 221 insertions, 32 deletions

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 2d51b76a1..1d435bed8 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -109,23 +109,18 @@ jobs:
           brew install Bearer/tap/gon
           security find-identity -v
           brew install coreutils
-
           # create variables
           CERTIFICATE_PATH=$RUNNER_TEMP/build_certificate.p12
           KEYCHAIN_PATH=$RUNNER_TEMP/app-signing.keychain-db
-
           # import certificate and provisioning profile from secrets
           echo -n "$BUILD_CERTIFICATE_BASE64" | base64 --decode --output "$CERTIFICATE_PATH"
-
           # create temporary keychain
           security create-keychain -p "$KEYCHAIN_PASSWORD" "$KEYCHAIN_PATH"
           security set-keychain-settings -lut 21600 "$KEYCHAIN_PATH"
           security unlock-keychain -p "$KEYCHAIN_PASSWORD" "$KEYCHAIN_PATH"
-
           # import certificate to keychain
           security import "$CERTIFICATE_PATH" -P "$P12_PASSWORD" -A -t cert -f pkcs12 -k "$KEYCHAIN_PATH"
           security list-keychain -d user -s "$KEYCHAIN_PATH"
-
           security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k "${KEYCHAIN_PASSWORD}" "$KEYCHAIN_PATH"
 
           binaries=(
@@ -137,6 +132,8 @@ jobs:
             "${bin_dir}/slangd"
             "${bin_dir}/slangc"
           )
+
+          # Sign main binaries
           for b in "${binaries[@]}"; do
             if [[ -f "$b" ]]; then
               echo "Signing binary '$b'..."
@@ -152,16 +149,28 @@ jobs:
       - name: Package Slang
         id: package
         run: |
-          # For the release, also generate a tar.gz file
+          # Package main binaries
           cpack --preset "$config" -G ZIP
           cpack --preset "$config" -G TGZ
+          # Package debug info
+          cpack --preset "$config-debug-info" -G ZIP
+          cpack --preset "$config-debug-info" -G TGZ
+
           triggering_ref=${{ github.ref_name }}
           base=slang-${triggering_ref#v}-${{matrix.os}}-${{matrix.platform}}
+
+          # Move main packages
           mv "$(pwd)/build/dist-${config}/slang.zip" "${base}.zip"
           echo "SLANG_BINARY_ARCHIVE_ZIP=${base}.zip" >> "$GITHUB_OUTPUT"
           mv "$(pwd)/build/dist-${config}/slang.tar.gz" "${base}.tar.gz"
           echo "SLANG_BINARY_ARCHIVE_TAR=${base}.tar.gz" >> "$GITHUB_OUTPUT"
 
+          # Move debug info packages
+          mv "$(pwd)/build/dist-${config}-debug-info/slang-debug-info.zip" "${base}-debug-info.zip"
+          echo "SLANG_DEBUG_INFO_ARCHIVE_ZIP=${base}-debug-info.zip" >> "$GITHUB_OUTPUT"
+          mv "$(pwd)/build/dist-${config}-debug-info/slang-debug-info.tar.gz" "${base}-debug-info.tar.gz"
+          echo "SLANG_DEBUG_INFO_ARCHIVE_TAR=${base}-debug-info.tar.gz" >> "$GITHUB_OUTPUT"
+
           # For some reason, the binaries packed by cpack for macos is modified
           # by cpack and considered damanged by macos. For now we workaround this
           # by repacking all the binaries into the release package.
@@ -169,8 +178,14 @@ jobs:
             mkdir ./ttmp
             unzip "${base}.zip" -d ./ttmp
 
-            /bin/cp -rf build/$cmake_config/bin/* ./ttmp/bin/
-            /bin/cp -rf build/$cmake_config/lib/* ./ttmp/lib/
+            # Copy only existing files from build directory
+            find ./ttmp/{bin,lib} -type f | while read -r file; do
+              src_file="build/$cmake_config/${file#./ttmp/}"
+              if [ -f "$src_file" ]; then
+                cp "$src_file" "$file"
+              fi
+            done
+
             rm ${base}.zip
             rm ${base}.tar.gz
             cd ./ttmp
@@ -178,6 +193,7 @@ jobs:
             tar -czvf ../${base}.tar.gz .
             cd ../
           fi
+
       - name: File check
         run: |
           find "build/dist-$config" -print0 ! -iname '*.md' ! -iname '*.h' -type f | xargs -0 file
@@ -191,6 +207,8 @@ jobs:
           files: |
             ${{ steps.package.outputs.SLANG_BINARY_ARCHIVE_ZIP }}
             ${{ steps.package.outputs.SLANG_BINARY_ARCHIVE_TAR }}
+            ${{ steps.package.outputs.SLANG_DEBUG_INFO_ARCHIVE_ZIP }}
+            ${{ steps.package.outputs.SLANG_DEBUG_INFO_ARCHIVE_TAR }}
             ${{ steps.notarize.outputs.SLANG_NOTARIZED_DIST }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 475780764..436f8e69d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -185,6 +185,12 @@ enum_option(
     "Build slang as a static library"
 )
 
+option(
+    SLANG_ENABLE_SPLIT_DEBUG_INFO
+    "Generate split debug info for debug builds"
+    ON
+)
+
 set(SLANG_GENERATORS_PATH
     ""
     CACHE PATH
diff --git a/CMakePresets.json b/CMakePresets.json
index 15bac9f44..b81309af4 100644
--- a/CMakePresets.json
+++ b/CMakePresets.json
@@ -119,7 +119,15 @@
       "generators": ["ZIP"],
       "variables": {
         "CPACK_PACKAGE_FILE_NAME": "slang",
-        "CPACK_COMPONENTS_ALL": "Unspecified;metadata;debug-info;slang-llvm"
+        "CPACK_COMPONENTS_ALL": "Unspecified;metadata;slang-llvm"
+      }
+    },
+    {
+      "name": "base-debug-info",
+      "inherits": "base",
+      "variables": {
+        "CPACK_PACKAGE_FILE_NAME": "slang-debug-info",
+        "CPACK_COMPONENTS_ALL": "debug-info"
       }
     },
     {
@@ -141,6 +149,24 @@
       "packageDirectory": "dist-debug"
     },
     {
+      "name": "release-debug-info",
+      "inherits": "base-debug-info",
+      "configurations": ["Release"],
+      "packageDirectory": "dist-release-debug-info"
+    },
+    {
+      "name": "releaseWithDebugInfo-debug-info",
+      "inherits": "base-debug-info",
+      "configurations": ["RelWithDebInfo"],
+      "packageDirectory": "dist-releaseWithDebugInfo-debug-info"
+    },
+    {
+      "name": "debug-debug-info",
+      "inherits": "base-debug-info",
+      "configurations": ["Debug"],
+      "packageDirectory": "dist-debug-debug-info"
+    },
+    {
       "name": "generators",
       "inherits": "release",
       "configurePreset": "generators",
diff --git a/cmake/CompilerFlags.cmake b/cmake/CompilerFlags.cmake
index cd9021cd0..e3ddaa7e0 100644
--- a/cmake/CompilerFlags.cmake
+++ b/cmake/CompilerFlags.cmake
@@ -152,8 +152,14 @@ function(set_default_compile_options target)
 
     add_supported_cxx_flags(${target} PRIVATE ${warning_flags})
 
-    # Don't assume that symbols will be resolved at runtime
-    add_supported_cxx_linker_flags(${target} PRIVATE "-Wl,--no-undefined")
+    add_supported_cxx_linker_flags(
+        ${target}
+        PRIVATE
+        # Don't assume that symbols will be resolved at runtime
+        "-Wl,--no-undefined"
+        # No reason not to do this? Useful when using split debug info
+        "-Wl,--build-id"
+    )
 
     set_target_properties(
         ${target}
diff --git a/cmake/SlangTarget.cmake b/cmake/SlangTarget.cmake
index 4da2cab3b..59d12ab7a 100644
--- a/cmake/SlangTarget.cmake
+++ b/cmake/SlangTarget.cmake
@@ -1,5 +1,5 @@
 #
-# A function to make target creation a little more declarative
+# A function to make target specification a little more declarative
 #
 # See the comments on the options below for usage
 #
@@ -21,6 +21,8 @@ function(slang_add_target dir type)
         # Don't include any source in this target, this is a complement to
         # EXPLICIT_SOURCE, and doesn't interact with EXTRA_SOURCE_DIRS
         NO_SOURCE
+        # Don't generate split debug info for this target
+        NO_SPLIT_DEBUG_INFO
     )
     set(single_value_args
         # Set the target name, useful for multiple targets from the same
@@ -49,7 +51,7 @@ function(slang_add_target dir type)
         DEBUG_DIR
         # Install this target as part of a component
         INSTALL_COMPONENT
-        # Don't install debug info by default for this target and use this
+        # Override the debug info component name for installation
         # explicit name instead, used for externally built things such as
         # slang-glslang and slang-llvm which have large pdb files
         DEBUG_INFO_INSTALL_COMPONENT
@@ -198,6 +200,15 @@ function(slang_add_target dir type)
             PDB_OUTPUT_DIRECTORY "${output_dir}/${runtime_subdir}"
     )
 
+    if(NOT MSVC)
+        set_target_properties(
+            ${target}
+            PROPERTIES
+                COMPILE_OPTIONS
+                    "$<$<CONFIG:Debug,RelWithDebInfo>:-fdebug-prefix-map=${CMAKE_CURRENT_BINARY_DIR}=${output_dir}>"
+        )
+    endif()
+
     #
     # Set common compile options and properties
     #
@@ -209,6 +220,118 @@ function(slang_add_target dir type)
         set_default_compile_options(${target})
     endif()
 
+    # Set debug info options if not disabled
+    # Determine if this target produces a binary that can have debug info
+    if(
+        NOT ARG_NO_SPLIT_DEBUG_INFO
+        AND type MATCHES "^(EXECUTABLE|SHARED|MODULE)$"
+        AND SLANG_ENABLE_SPLIT_DEBUG_INFO
+    )
+        set(generate_split_debug_info TRUE)
+    else()
+        set(generate_split_debug_info FALSE)
+    endif()
+
+    if(generate_split_debug_info)
+        if(MSVC)
+            get_target_property(
+                c_compiler_launcher
+                ${target}
+                C_COMPILER_LAUNCHER
+            )
+            get_target_property(
+                cxx_compiler_launcher
+                ${target}
+                CXX_COMPILER_LAUNCHER
+            )
+
+            if(
+                c_compiler_launcher MATCHES "ccache"
+                OR cxx_compiler_launcher MATCHES "ccache"
+            )
+                message(
+                    WARNING
+                    "(s)ccache detected for target ${target}. Removing launcher as it's incompatible with split debug info compiled with MSVC."
+                )
+                set_target_properties(
+                    ${target}
+                    PROPERTIES C_COMPILER_LAUNCHER "" CXX_COMPILER_LAUNCHER ""
+                )
+            endif()
+
+            get_target_property(
+                msvc_debug_information_format
+                ${target}
+                MSVC_DEBUG_INFORMATION_FORMAT
+            )
+            if(
+                NOT msvc_debug_information_format
+                    MATCHES
+                    "(ProgramDatabase|EditAndContinue)"
+            )
+                message(
+                    WARNING
+                    "Debug format must be ProgramDatabase or EditAndContinue to generate split debug info with MSVC"
+                )
+            endif()
+
+            set_target_properties(
+                ${target}
+                PROPERTIES
+                    # While it would be nice to set this here, we don't know if
+                    # the user wants ProgramDatabase or EditAndContinue, so
+                    # just check above
+                    # MSVC_DEBUG_INFORMATION_FORMAT
+                    #     "$<$<CONFIG:Debug,RelWithDebInfo>:ProgramDatabase>"
+                    COMPILE_PDB_NAME "${target}"
+                    COMPILE_PDB_OUTPUT_DIRECTORY "${output_dir}"
+            )
+        else()
+            # Common debug flags for GCC/Clang
+            target_compile_options(
+                ${target}
+                PRIVATE
+                    $<$<CONFIG:Debug,RelWithDebInfo>:
+                    -g
+                    -fdebug-prefix-map=${CMAKE_SOURCE_DIR}=.
+                    -fdebug-prefix-map=${CMAKE_BINARY_DIR}=.
+                    >
+            )
+
+            if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
+                # macOS - use dsymutil with --flat to create separate debug file
+                add_custom_command(
+                    TARGET ${target}
+                    POST_BUILD
+                    COMMAND
+                        dsymutil --flat $<TARGET_FILE:${target}> -o
+                        $<TARGET_FILE:${target}>.dwarf
+                    COMMAND chmod 644 $<TARGET_FILE:${target}>.dwarf
+                    COMMAND ${CMAKE_STRIP} -S $<TARGET_FILE:${target}>
+                    WORKING_DIRECTORY ${output_dir}
+                    VERBATIM
+                )
+            else()
+                add_custom_command(
+                    TARGET ${target}
+                    POST_BUILD
+                    COMMAND
+                        ${CMAKE_OBJCOPY} --only-keep-debug
+                        $<TARGET_FILE:${target}> $<TARGET_FILE:${target}>.dwarf
+                    COMMAND chmod 644 $<TARGET_FILE:${target}>.dwarf
+                    COMMAND
+                        ${CMAKE_STRIP} --strip-debug $<TARGET_FILE:${target}>
+                    COMMAND
+                        ${CMAKE_OBJCOPY}
+                        --add-gnu-debuglink=$<TARGET_FILE:${target}>.dwarf
+                        $<TARGET_FILE:${target}>
+                    WORKING_DIRECTORY ${output_dir}
+                    VERBATIM
+                )
+            endif()
+        endif()
+    endif()
+
     set_target_properties(
         ${target}
         PROPERTIES EXCLUDE_FROM_ALL ${ARG_EXCLUDE_FROM_ALL}
@@ -429,32 +552,41 @@ function(slang_add_target dir type)
             PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} ${ARGN}
         )
     endmacro()
+
     if(ARG_INSTALL_COMPONENT)
         i(EXCLUDE_FROM_ALL COMPONENT ${ARG_INSTALL_COMPONENT})
-        set(pdb_component "${ARG_INSTALL_COMPONENT}-debug-info")
+        set(debug_component "${ARG_INSTALL_COMPONENT}-debug-info")
     elseif(ARG_INSTALL)
         i()
-        set(pdb_component "debug-info")
+        set(debug_component "debug-info")
     endif()
-    if(ARG_DEBUG_INFO_INSTALL_COMPONENT)
-        set(pdb_component ${ARG_DEBUG_INFO_INSTALL_COMPONENT})
+
+    if(DEFINED ARG_DEBUG_INFO_INSTALL_COMPONENT)
+        set(debug_component "${ARG_DEBUG_INFO_INSTALL_COMPONENT}")
     endif()
-    if(MSVC AND DEFINED pdb_component)
-        if(
-            type STREQUAL "EXECUTABLE"
-            OR type STREQUAL "SHARED"
-            OR type STREQUAL "MODULE"
-        )
-            install(
-                FILES $<TARGET_PDB_FILE:${target}>
-                DESTINATION ${runtime_subdir}
-                # Optional, because if we're building without debug info (like
-                # a release build) then we don't want to fail here.
-                OPTIONAL
-                COMPONENT ${pdb_component}
-                EXCLUDE_FROM_ALL
-            )
+
+    # Install debug info only if target is being installed
+    if((ARG_INSTALL OR ARG_INSTALL_COMPONENT) AND generate_split_debug_info)
+        if(type STREQUAL "EXECUTABLE" OR WIN32)
+            set(debug_dest ${runtime_subdir})
+        else()
+            set(debug_dest ${library_subdir})
         endif()
+
+        if(MSVC)
+            set(debug_file $<TARGET_PDB_FILE:${target}>)
+        else()
+            set(debug_file "$<TARGET_FILE:${target}>.dwarf")
+        endif()
+
+        install(
+            FILES ${debug_file}
+            DESTINATION ${debug_dest}
+            CONFIGURATIONS Debug RelWithDebInfo
+            COMPONENT ${debug_component}
+            EXCLUDE_FROM_ALL
+            OPTIONAL
+        )
     endif()
 endfunction()
 
diff --git a/docs/building.md b/docs/building.md
index ea40be212..cc7ac6eda 100644
--- a/docs/building.md
+++ b/docs/building.md
@@ -163,6 +163,7 @@ See the [documentation on testing](../tools/slang-test/README.md) for more infor
 | `SLANG_ENABLE_TESTS`              | `TRUE`                     | Enable test targets, requires SLANG_ENABLE_GFX, SLANG_ENABLE_SLANGD and SLANG_ENABLE_SLANGRT |
 | `SLANG_ENABLE_EXAMPLES`           | `TRUE`                     | Enable example targets, requires SLANG_ENABLE_GFX                                            |
 | `SLANG_LIB_TYPE`                  | `SHARED`                   | How to build the slang library                                                               |
+| `SLANG_ENABLE_SPLIT_DEBUG_INFO`   | `TRUE`                     | Enable generating split debug info for Debug and RelWithDebInfo configs                      |
 | `SLANG_SLANG_LLVM_FLAVOR`         | `FETCH_BINARY_IF_POSSIBLE` | How to set up llvm support                                                                   |
 | `SLANG_SLANG_LLVM_BINARY_URL`     | System dependent           | URL specifying the location of the slang-llvm prebuilt library                               |
 | `SLANG_GENERATORS_PATH`           | ``                         | Path to an installed `all-generators` target for cross compilation                           |