Add backtraces to examples (#5973)

* examples: Log stack trace on exceptions

For now, this is only implemented on Windows.
This helps to address #5520.

* examples: Print log file if there is any

* format code

---------

Co-authored-by: slangbot <186143334+slangbot@users.noreply.github.com>
Co-authored-by: Yong He <yonghe@outlook.com>

20 files changed, 316 insertions, 14 deletions

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index e46f41e7a..759d99994 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -1,4 +1,6 @@
 function(example dir)
+    cmake_parse_arguments(ARG "WIN32_EXECUTABLE" "" "" ${ARGN})
+
     set(debug_dir ${CMAKE_CURRENT_BINARY_DIR}/${dir})
 
     file(
@@ -30,6 +32,22 @@ function(example dir)
         )
     endif()
 
+    # Libraries providing a main function that prints stack traces on exceptions
+    if(CMAKE_SYSTEM_NAME MATCHES "Windows")
+        # On Windows we have two different versions: main for "console applications" and
+        # WinMain for normal Windows applications.
+        if(${ARG_WIN32_EXECUTABLE})
+            set(main_wrapper_libraries example-winmain)
+        else()
+            set(main_wrapper_libraries example-main)
+        endif()
+        # Add stack printing support
+        set(main_wrapper_libraries ${main_wrapper_libraries} stacktrace-windows)
+        set(main_wrapper_libraries ${main_wrapper_libraries} dbghelp.lib)
+    else()
+        set(main_wrapper_libraries example-main)
+    endif()
+
     slang_add_target(
         ${dir}
         EXECUTABLE
@@ -42,7 +60,9 @@ function(example dir)
             gfx-util
             platform
             $<$<BOOL:${SLANG_ENABLE_CUDA}>:CUDA::cuda_driver>
+            ${main_wrapper_libraries}
         EXTRA_COMPILE_DEFINITIONS_PRIVATE
+            SLANG_EXAMPLE_NAME=${dir}
             $<$<BOOL:${SLANG_ENABLE_XLIB}>:SLANG_ENABLE_XLIB>
         REQUIRED_BY all-examples
         OPTIONAL_REQUIRES ${copy_assets_target} copy-prebuilt-binaries
@@ -68,6 +88,9 @@ if(SLANG_ENABLE_EXAMPLES)
             $<$<BOOL:${SLANG_ENABLE_CUDA}>:CUDA::cuda_driver>
         FOLDER examples
     )
+    slang_add_target(example-main STATIC FOLDER examples)
+    slang_add_target(example-winmain STATIC FOLDER examples EXCLUDE_FROM_ALL)
+    slang_add_target(stacktrace-windows STATIC FOLDER examples EXCLUDE_FROM_ALL)
 
     add_custom_target(
         all-examples
diff --git a/examples/autodiff-texture/main.cpp b/examples/autodiff-texture/main.cpp
index d0c35d003..d99f9f341 100644
--- a/examples/autodiff-texture/main.cpp
+++ b/examples/autodiff-texture/main.cpp
@@ -823,4 +823,4 @@ struct AutoDiffTexture : public WindowedAppBase
     }
 };
 
-PLATFORM_UI_MAIN(innerMain<AutoDiffTexture>)
+EXAMPLE_MAIN(innerMain<AutoDiffTexture>);
diff --git a/examples/cpu-com-example/main.cpp b/examples/cpu-com-example/main.cpp
index 382b3cacd..6c67215b4 100644
--- a/examples/cpu-com-example/main.cpp
+++ b/examples/cpu-com-example/main.cpp
@@ -175,7 +175,7 @@ static SlangResult _innerMain(int argc, char** argv)
     return SLANG_OK;
 }
 
-int main(int argc, char** argv)
+int exampleMain(int argc, char** argv)
 {
     return SLANG_SUCCEEDED(_innerMain(argc, argv)) ? 0 : -1;
 }
diff --git a/examples/cpu-hello-world/main.cpp b/examples/cpu-hello-world/main.cpp
index 60a24fa8c..76ca5af1d 100644
--- a/examples/cpu-hello-world/main.cpp
+++ b/examples/cpu-hello-world/main.cpp
@@ -217,7 +217,7 @@ static SlangResult _innerMain(int argc, char** argv)
     return SLANG_OK;
 }
 
-int main(int argc, char** argv)
+int exampleMain(int argc, char** argv)
 {
     return SLANG_SUCCEEDED(_innerMain(argc, argv)) ? 0 : -1;
 }
diff --git a/examples/example-base/example-base.h b/examples/example-base/example-base.h
index 6988d613b..9aabac8d4 100644
--- a/examples/example-base/example-base.h
+++ b/examples/example-base/example-base.h
@@ -10,6 +10,19 @@
 void _Win32OutputDebugString(const char* str);
 #endif
 
+#define SLANG_STRINGIFY(x) #x
+#define SLANG_EXPAND_STRINGIFY(x) SLANG_STRINGIFY(x)
+
+#ifdef _WIN32
+#define EXAMPLE_MAIN(innerMain)                                   \
+    extern const char* const g_logFileName =                      \
+        "log-" SLANG_EXPAND_STRINGIFY(SLANG_EXAMPLE_NAME) ".txt"; \
+    PLATFORM_UI_MAIN(innerMain);
+
+#else
+#define EXAMPLE_MAIN(innerMain) PLATFORM_UI_MAIN(innerMain)
+#endif // _WIN32
+
 struct WindowedAppBase : public TestBase
 {
 protected:
diff --git a/examples/example-main/main.cpp b/examples/example-main/main.cpp
new file mode 100644
index 000000000..46ffc7278
--- /dev/null
+++ b/examples/example-main/main.cpp
@@ -0,0 +1,32 @@
+#include "../stacktrace-windows/common.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+extern int exampleMain(int argc, char** argv);
+
+#if defined(_WIN32)
+
+#include <windows.h>
+
+int main(int argc, char** argv)
+{
+    __try
+    {
+        return exampleMain(argc, argv);
+    }
+    __except (exceptionFilter(stdout, GetExceptionInformation()))
+    {
+        ::exit(1);
+    }
+}
+
+#else // defined(_WIN32)
+
+int main(int argc, char** argv)
+{
+    // TODO: Catch exception and print stack trace also on non-Windows platforms.
+    return exampleMain(argc, argv);
+}
+
+#endif
diff --git a/examples/example-winmain/main.cpp b/examples/example-winmain/main.cpp
new file mode 100644
index 000000000..8094e7fc4
--- /dev/null
+++ b/examples/example-winmain/main.cpp
@@ -0,0 +1,28 @@
+#include "../stacktrace-windows/common.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <windows.h>
+
+extern int exampleMain(int argc, char** argv);
+extern const char* const g_logFileName;
+
+int WinMain(
+    HINSTANCE /* instance */,
+    HINSTANCE /* prevInstance */,
+    LPSTR /* commandLine */,
+    int /*showCommand*/)
+
+{
+    FILE* logFile = fopen(g_logFileName, "w");
+    __try
+    {
+        int argc = 0;
+        char** argv = nullptr;
+        return exampleMain(argc, argv);
+    }
+    __except (exceptionFilter(logFile, GetExceptionInformation()))
+    {
+        ::exit(1);
+    }
+}
diff --git a/examples/gpu-printing/main.cpp b/examples/gpu-printing/main.cpp
index bbc300dba..27a77a82b 100644
--- a/examples/gpu-printing/main.cpp
+++ b/examples/gpu-printing/main.cpp
@@ -152,7 +152,7 @@ struct ExampleProgram : public TestBase
     }
 };
 
-int main(int argc, char* argv[])
+int exampleMain(int argc, char** argv)
 {
     ExampleProgram app;
     if (SLANG_FAILED(app.execute(argc, argv)))
diff --git a/examples/hello-world/main.cpp b/examples/hello-world/main.cpp
index 7e4211b12..fbf67569d 100644
--- a/examples/hello-world/main.cpp
+++ b/examples/hello-world/main.cpp
@@ -66,7 +66,8 @@ struct HelloWorldExample : public TestBase
     ~HelloWorldExample();
 };
 
-int main(int argc, char* argv[])
+
+int exampleMain(int argc, char** argv)
 {
     initDebugCallback();
     HelloWorldExample example;
diff --git a/examples/model-viewer/main.cpp b/examples/model-viewer/main.cpp
index ecca818f1..8bbc8ec88 100644
--- a/examples/model-viewer/main.cpp
+++ b/examples/model-viewer/main.cpp
@@ -969,4 +969,4 @@ struct ModelViewer : WindowedAppBase
 
 // This macro instantiates an appropriate main function to
 // run the application defined above.
-PLATFORM_UI_MAIN(innerMain<ModelViewer>)
+EXAMPLE_MAIN(innerMain<ModelViewer>);
diff --git a/examples/nv-aftermath-example/main.cpp b/examples/nv-aftermath-example/main.cpp
index 9d85f1ff4..ed6db43a2 100644
--- a/examples/nv-aftermath-example/main.cpp
+++ b/examples/nv-aftermath-example/main.cpp
@@ -599,4 +599,4 @@ void AftermathCrashExample::renderFrame(int frameBufferIndex)
 
 // This macro instantiates an appropriate main function to
 // run the application defined above.
-PLATFORM_UI_MAIN(innerMain<AftermathCrashExample>)
+EXAMPLE_MAIN(innerMain<AftermathCrashExample>)
diff --git a/examples/platform-test/main.cpp b/examples/platform-test/main.cpp
index 159e26c55..865e4eab7 100644
--- a/examples/platform-test/main.cpp
+++ b/examples/platform-test/main.cpp
@@ -122,4 +122,4 @@ struct PlatformTest : public WindowedAppBase
 
 // This macro instantiates an appropriate main function to
 // run the application defined above.
-PLATFORM_UI_MAIN(innerMain<PlatformTest>)
+EXAMPLE_MAIN(innerMain<PlatformTest>);
diff --git a/examples/ray-tracing-pipeline/main.cpp b/examples/ray-tracing-pipeline/main.cpp
index a288e75b5..a3d468db1 100644
--- a/examples/ray-tracing-pipeline/main.cpp
+++ b/examples/ray-tracing-pipeline/main.cpp
@@ -712,4 +712,4 @@ struct RayTracing : public WindowedAppBase
 
 // This macro instantiates an appropriate main function to
 // run the application defined above.
-PLATFORM_UI_MAIN(innerMain<RayTracing>)
+EXAMPLE_MAIN(innerMain<RayTracing>);
diff --git a/examples/ray-tracing/main.cpp b/examples/ray-tracing/main.cpp
index 6b908a14e..6a0abf8b4 100644
--- a/examples/ray-tracing/main.cpp
+++ b/examples/ray-tracing/main.cpp
@@ -676,4 +676,4 @@ struct RayTracing : public WindowedAppBase
 
 // This macro instantiates an appropriate main function to
 // run the application defined above.
-PLATFORM_UI_MAIN(innerMain<RayTracing>)
+EXAMPLE_MAIN(innerMain<RayTracing>);
diff --git a/examples/reflection-api/main.cpp b/examples/reflection-api/main.cpp
index 5c157b797..c072c641b 100644
--- a/examples/reflection-api/main.cpp
+++ b/examples/reflection-api/main.cpp
@@ -1469,7 +1469,7 @@ struct ExampleProgram : public TestBase
     }
 };
 
-int main(int argc, char* argv[])
+int exampleMain(int argc, char** argv)
 {
     ExampleProgram app;
     if (SLANG_FAILED(app.execute(argc, argv)))
diff --git a/examples/shader-object/main.cpp b/examples/shader-object/main.cpp
index f5c02141f..1010cdcb9 100644
--- a/examples/shader-object/main.cpp
+++ b/examples/shader-object/main.cpp
@@ -131,7 +131,7 @@ Result loadShaderProgram(
 }
 
 // Main body of the example.
-int main(int argc, char* argv[])
+int exampleMain(int argc, char** argv)
 {
     testBase.parseOption(argc, argv);
 
diff --git a/examples/shader-toy/main.cpp b/examples/shader-toy/main.cpp
index 185d18246..42054beae 100644
--- a/examples/shader-toy/main.cpp
+++ b/examples/shader-toy/main.cpp
@@ -408,4 +408,4 @@ struct ShaderToyApp : public WindowedAppBase
 
 // This macro instantiates an appropriate main function to
 // run the application defined above.
-PLATFORM_UI_MAIN(innerMain<ShaderToyApp>)
+EXAMPLE_MAIN(innerMain<ShaderToyApp>);
diff --git a/examples/stacktrace-windows/common.cpp b/examples/stacktrace-windows/common.cpp
new file mode 100644
index 000000000..b07f78d0a
--- /dev/null
+++ b/examples/stacktrace-windows/common.cpp
@@ -0,0 +1,201 @@
+#include "common.h"
+
+#include <inttypes.h>
+#include <string>
+#include <vector>
+#include <windows.h>
+
+// dbghelp.h needs to be included after windows.h
+#include <dbghelp.h>
+
+#define SLANG_EXAMPLE_LOG_ERROR(...)                      \
+    fprintf(file, "error: %s: %d: ", __FILE__, __LINE__); \
+    print(file, __VA_ARGS__);                             \
+    fprintf(file, "\n");
+
+static void print(FILE* /* file */) {}
+static void print(FILE* file, unsigned int n)
+{
+    fprintf(file, "%u", n);
+}
+
+
+static bool getModuleFileNameAtAddress(FILE* file, DWORD64 const address, std::string& fileName)
+{
+    HMODULE module = NULL;
+    {
+        BOOL result = GetModuleHandleEx(
+            GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
+            (LPCTSTR)address,
+            &module);
+        if (result == 0)
+        {
+            SLANG_EXAMPLE_LOG_ERROR(GetLastError());
+            return false;
+        }
+        if (module == NULL)
+        {
+            SLANG_EXAMPLE_LOG_ERROR();
+            return false;
+        }
+    }
+
+    std::vector<char> buffer(1U << 8U);
+    uint32_t constexpr maxBufferSize = 1U << 20;
+    while (buffer.size() < maxBufferSize)
+    {
+        DWORD result = GetModuleFileNameA(module, buffer.data(), buffer.size());
+        if (result == 0)
+        {
+            SLANG_EXAMPLE_LOG_ERROR(GetLastError());
+            return false;
+        }
+        else if (result == ERROR_INSUFFICIENT_BUFFER)
+        {
+            buffer.resize(buffer.size() << 1U);
+        }
+        else
+        {
+            break;
+        }
+    }
+    if (buffer.size() == maxBufferSize)
+    {
+        SLANG_EXAMPLE_LOG_ERROR();
+        return false;
+    }
+
+    fileName = std::string(buffer.data(), buffer.data() + buffer.size());
+    return true;
+}
+
+// NOTE: This function is not thread-safe, due to usage of StackWalk64 and static buffers.
+static bool printStack(FILE* file, HANDLE process, HANDLE thread, CONTEXT const& context)
+{
+#if defined(_M_AMD64)
+    DWORD constexpr machineType = IMAGE_FILE_MACHINE_AMD64;
+#else
+#error Unsupported machine type
+#endif
+
+    static char symbolBuffer[sizeof(SYMBOL_INFO) + MAX_SYM_NAME * sizeof(TCHAR)];
+
+    // StackWalk64 may modify the context record
+    CONTEXT contextCopy;
+    memcpy(&contextCopy, &context, sizeof(CONTEXT));
+
+    STACKFRAME64 frame = {};
+    constexpr uint32_t maxFrameCount = 1U << 10;
+    uint32_t frameIndex = 0U;
+    while (frameIndex < maxFrameCount)
+    {
+        // Use the default routine
+        PREAD_PROCESS_MEMORY_ROUTINE64 readMemoryRoutine = NULL;
+        // Not sure what this is for, but documentation says most callers can pass NULL
+        PTRANSLATE_ADDRESS_ROUTINE64 translateAddressRoutine = NULL;
+        {
+            BOOL result = StackWalk64(
+                machineType,
+                process,
+                thread,
+                &frame,
+                &contextCopy,
+                readMemoryRoutine,
+                SymFunctionTableAccess64,
+                SymGetModuleBase64,
+                translateAddressRoutine);
+            if (result == FALSE)
+                break;
+        }
+
+        PSYMBOL_INFO maybeSymbol = (PSYMBOL_INFO)symbolBuffer;
+        {
+            maybeSymbol->SizeOfStruct = sizeof(SYMBOL_INFO);
+            maybeSymbol->MaxNameLen = MAX_SYM_NAME;
+            DWORD64 address = frame.AddrPC.Offset;
+            // Not required, we want to look up the symbol exactly at the address
+            PDWORD64 displacement = NULL;
+            BOOL result = SymFromAddr(process, address, displacement, maybeSymbol);
+            if (result == FALSE)
+            {
+                SLANG_EXAMPLE_LOG_ERROR(GetLastError());
+                maybeSymbol = NULL;
+            }
+        }
+
+        fprintf(file, "%u", frameIndex);
+
+        std::string moduleFileName;
+        if (getModuleFileNameAtAddress(file, frame.AddrPC.Offset, moduleFileName))
+            fprintf(file, ": %s", moduleFileName.c_str());
+
+        if (maybeSymbol)
+        {
+            PSYMBOL_INFO& symbol = maybeSymbol;
+
+            IMAGEHLP_LINE64 line = {};
+            line.SizeOfStruct = sizeof(IMAGEHLP_LINE64);
+
+            DWORD displacement;
+            if (SymGetLineFromAddr64(process, frame.AddrPC.Offset, &displacement, &line))
+            {
+                fprintf(file, ": %s: %s: %lu", symbol->Name, line.FileName, line.LineNumber);
+            }
+            else
+            {
+                fprintf(file, ": %s", symbol->Name);
+            }
+
+            fprintf(file, ": 0x%.16" PRIXPTR, symbol->Address);
+        }
+        fprintf(file, "\n");
+
+        frameIndex++;
+    }
+
+    return frameIndex < maxFrameCount;
+}
+
+int exceptionFilter(FILE* logFile, _EXCEPTION_POINTERS* exception)
+{
+    FILE* file = logFile ? logFile : stdout;
+    fprintf(
+        file,
+        "error: Exception 0x%x occurred. Stack trace:\n",
+        exception->ExceptionRecord->ExceptionCode);
+
+    HANDLE process = GetCurrentProcess();
+    HANDLE thread = GetCurrentThread();
+
+    bool symbolsLoaded = false;
+    {
+        // The default search paths should suffice
+        PCSTR symbolFileSearchPath = NULL;
+        BOOL loadSymbolsOfLoadedModules = TRUE;
+        BOOL result = SymInitialize(process, symbolFileSearchPath, loadSymbolsOfLoadedModules);
+        if (result == FALSE)
+        {
+            fprintf(file, "warning: Failed to load symbols\n");
+        }
+        else
+        {
+            symbolsLoaded = true;
+        }
+    }
+
+    if (!printStack(file, process, thread, *exception->ContextRecord))
+    {
+        fprintf(file, "warning: Failed to print complete stack trace!\n");
+    }
+
+    if (symbolsLoaded)
+    {
+        BOOL result = SymCleanup(process);
+        if (result == FALSE)
+        {
+            SLANG_EXAMPLE_LOG_ERROR(GetLastError());
+        }
+    }
+
+    return EXCEPTION_EXECUTE_HANDLER;
+}
diff --git a/examples/stacktrace-windows/common.h b/examples/stacktrace-windows/common.h
new file mode 100644
index 000000000..0f375c431
--- /dev/null
+++ b/examples/stacktrace-windows/common.h
@@ -0,0 +1,4 @@
+#pragma once
+#include <stdio.h>
+
+int exceptionFilter(FILE* logFile, struct _EXCEPTION_POINTERS* exception);
diff --git a/examples/triangle/main.cpp b/examples/triangle/main.cpp
index f757b59c7..6fd36f72d 100644
--- a/examples/triangle/main.cpp
+++ b/examples/triangle/main.cpp
@@ -405,4 +405,4 @@ struct HelloWorld : public WindowedAppBase
 
 // This macro instantiates an appropriate main function to
 // run the application defined above.
-PLATFORM_UI_MAIN(innerMain<HelloWorld>)
+EXAMPLE_MAIN(innerMain<HelloWorld>);