Add slang example tests to CI (#5879)

* Examples: Don't proceed if 'initializeBase' fails

* Examples: Only access gWindow if it's been initialized

* Examples: Free memory from CommandLineToArgvW

* Add example run step to CI

Lots of examples are still unexpectedly failing, but is one small step towards addressing
issue #5520.

11 files changed, 277 insertions, 23 deletions

diff --git a/.github/workflows/ci-examples.sh b/.github/workflows/ci-examples.sh
new file mode 100755
index 000000000..c5fad28a6
--- /dev/null
+++ b/.github/workflows/ci-examples.sh
@@ -0,0 +1,200 @@
+#!/usr/bin/env bash
+set -e
+
+show_help() {
+  me=$(basename "$0")
+  cat <<EOF
+$me: Run all of the examples in test mode
+
+Usage: $me --os <os> --config <config> --bin-dir <path> --skip-file <path> [--dry-run]
+
+Options:
+  --help                 Show this help message
+  --dry-run              Skip running the examples.
+  --bin-dir   <path>     Path to binary directory.
+                         Must contain all of the example binaries.
+  --skip-file <path>     Path to file containing skip patterns.
+                         See the 'Skip file' section, below.
+  --os        <os>       Operating system.
+                         Valid <os> values: 'macos', 'windows', 'linux'
+  --config    <config>   Build configuration.
+                         Valid <config> values: 'debug', 'release'.
+
+Skip file:
+
+  The skip patterns are regexp patterns on the following format:
+  <os>:<config>:<sample> # Some comment describing why test is disabled
+
+  For example:
+  # Bug 123: foo-example fails (both debug and release)
+  windows:(debug|release):foo-example
+  # Bug 456: bar-example fails in release mode on Linux
+  linux:release:bar-example
+
+EOF
+}
+
+while [[ "$#" -gt 0 ]]; do
+  case $1 in
+  -h | --help)
+    show_help
+    exit 0
+    ;;
+  --dry-run)
+    dry_run=true
+    ;;
+  --bin-dir)
+    bin_dir="$2"
+    shift
+    ;;
+  --skip-file)
+    skip_file="$2"
+    shift
+    ;;
+  --os)
+    case $2 in
+    windows | linux | macos) ;;
+    *)
+      printf "error: Unrecognized os: '$2'\n\n" >&2
+      show_help >&2
+      exit 1
+      ;;
+    esac
+    os="$2"
+    shift
+    ;;
+  --config)
+    case $2 in
+    debug | release) ;;
+    *)
+      printf "error: Unrecognized config: '$2'\n\n" >&2
+      show_help >&2
+      exit 1
+      ;;
+    esac
+    config="$2"
+    shift
+    ;;
+  *)
+    echo "unrecognized argument: $1" >&2
+    show_help >&2
+    exit 1
+    ;;
+  esac
+  shift
+done
+
+if [[ "$os" == "" ]]; then
+  echo "error: No OS specified.\n\n"
+  show_help >&2
+  exit 1
+fi
+
+if [[ "$config" == "" ]]; then
+  printf "error: No build configuration specified.\n\n" >&2
+  show_help >&2
+  exit 1
+fi
+
+if [[ "$bin_dir" == "" ]]; then
+  printf "error: No binary directory specified.\n\n" >&2
+  show_help >&2
+  exit 1
+fi
+
+if [[ "$skip_file" == "" ]]; then
+  printf "error: No skip file specified.\n\n" >&2
+  show_help >&2
+  exit 1
+fi
+
+if [[ ! -f "$skip_file" ]]; then
+  printf "error: Skip file '$skip_file' does not exist.\n\n" >&2
+fi
+
+if [[ ! -d "$bin_dir" ]]; then
+  printf "error: Binary directory '$bin_dir' does not exist.\n\n" >&2
+fi
+
+summary=""
+failure_count=0
+skip_count=0
+sample_count=0
+
+function skip {
+  local p
+  local line_index
+  p="$1"
+  line_index=1
+  while read pattern; do
+    pat=$pattern
+    if [[ ! $pat =~ .*# ]]; then
+      echo "error: Skip pattern on line $line_index is missing a comment!"
+      exit 1
+    fi
+    pat="${pattern%% *#*}"
+    if [[ $p =~ ^$pat$ ]]; then
+      return 0
+    fi
+    line_index=$((line_index + 1))
+  done <$skip_file
+
+  return 1
+}
+
+function run_sample {
+  local command
+  local sample
+  command=$@
+  shift
+  sample="${command%% *}"
+  sample_count=$((sample_count + 1))
+  summary+="$sample: "
+  if skip "$os:$config:$sample"; then
+    echo "Skipping $sample..."
+    summary+="\n  skipped\n"
+    skip_count=$((skip_count + 1))
+    return
+  fi
+  echo "Running '$command'..."
+  result=0
+  pushd $bin_dir 1>/dev/null 2>&1
+  if [[ ! "$dry_run" = true ]]; then
+    ./$command || result=$?
+  fi
+  if [[ $result -eq 0 ]]; then
+    summary+="\n  success\n"
+  else
+    summary+="\n  failure (exit code: $result)\n"
+    failure_count=$((failure_count + 1))
+  fi
+  popd 1>/dev/null 2>&1
+}
+
+sample_commands=(
+  'platform-test --test-mode'
+  'ray-tracing-pipeline --test-mode'
+  'ray-tracing --test-mode'
+  'shader-toy --test-mode'
+  'triangle --test-mode'
+  'model-viewer --test-mode'
+  'shader-object'
+  'reflection-api'
+  'hello-world'
+  'gpu-printing'
+  'cpu-hello-world'
+  'cpu-com-example'
+)
+
+for sample_command in "${sample_commands[@]}"; do
+  run_sample $sample_command
+  echo ""
+done
+
+echo ""
+echo "Summary: "
+printf "\n$summary\n\n"
+echo "$failure_count failed, and $skip_count skipped, out of $sample_count tests"
+if [[ $failure_count -ne 0 ]]; then
+  exit 1
+fi
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 357b18137..6289d291f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -176,6 +176,14 @@ jobs:
               -expected-failure-list tests/expected-failure-github.txt \
               -expected-failure-list tests/expected-failure-record-replay-tests.txt
           fi
+      - name: Run Slang examples
+        if: steps.filter.outputs.should-run == 'true' && matrix.platform != 'wasm'
+        run: |
+          .github/workflows/ci-examples.sh \
+           --bin-dir "$bin_dir" \
+           --os "${{matrix.os}}" \
+           --config "${{matrix.config}}" \
+           --skip-file tests/expected-example-failure-github.txt
       - name: Test Slang via glsl
         if: steps.filter.outputs.should-run == 'true' && matrix.full-gpu-tests && matrix.platform != 'wasm'
         run: |
diff --git a/examples/autodiff-texture/main.cpp b/examples/autodiff-texture/main.cpp
index 647929e70..d0c35d003 100644
--- a/examples/autodiff-texture/main.cpp
+++ b/examples/autodiff-texture/main.cpp
@@ -285,7 +285,7 @@ struct AutoDiffTexture : public WindowedAppBase
     }
     Slang::Result initialize()
     {
-        initializeBase("autodiff-texture", 1024, 768);
+        SLANG_RETURN_ON_FAIL(initializeBase("autodiff-texture", 1024, 768));
         srand(20421);
 
         if (!isTestMode())
diff --git a/examples/example-base/test-base.cpp b/examples/example-base/test-base.cpp
index 5d40f2d80..bfbfd6b15 100644
--- a/examples/example-base/test-base.cpp
+++ b/examples/example-base/test-base.cpp
@@ -27,6 +27,11 @@ int TestBase::parseOption(int argc, char** argv)
             m_isTestMode = true;
         }
     }
+
+#ifdef _WIN32
+    LocalFree(szArglist);
+#endif
+
     return 0;
 }
 
diff --git a/examples/model-viewer/main.cpp b/examples/model-viewer/main.cpp
index 8b75381be..ecca818f1 100644
--- a/examples/model-viewer/main.cpp
+++ b/examples/model-viewer/main.cpp
@@ -745,7 +745,7 @@ struct ModelViewer : WindowedAppBase
     //
     Result initialize()
     {
-        initializeBase("Model Viewer", 1024, 768);
+        SLANG_RETURN_ON_FAIL(initializeBase("Model Viewer", 1024, 768));
         if (!isTestMode())
         {
             gWindow->events.mouseMove = [this](const platform::MouseEventArgs& e)
diff --git a/examples/platform-test/main.cpp b/examples/platform-test/main.cpp
index 373c15e09..159e26c55 100644
--- a/examples/platform-test/main.cpp
+++ b/examples/platform-test/main.cpp
@@ -71,18 +71,26 @@ struct PlatformTest : public WindowedAppBase
 
     Slang::Result initialize()
     {
-        initializeBase("platform-test", 1024, 768);
-
-        gWindow->events.sizeChanged = [this]() { onSizeChanged(); };
-        gWindow->events.focus = [this]() { onFocus(); };
-        gWindow->events.lostFocus = [this]() { onLostFocus(); };
-        gWindow->events.keyDown = [this](const platform::KeyEventArgs& e) { onKeyDown(e); };
-        gWindow->events.keyUp = [this](const platform::KeyEventArgs& e) { onKeyUp(e); };
-        gWindow->events.keyPress = [this](const platform::KeyEventArgs& e) { onKeyPress(e); };
-        gWindow->events.mouseMove = [this](const platform::MouseEventArgs& e) { onMouseMove(e); };
-        gWindow->events.mouseDown = [this](const platform::MouseEventArgs& e) { onMouseDown(e); };
-        gWindow->events.mouseUp = [this](const platform::MouseEventArgs& e) { onMouseUp(e); };
-        gWindow->events.mouseWheel = [this](const platform::MouseEventArgs& e) { onMouseWheel(e); };
+        SLANG_RETURN_ON_FAIL(initializeBase("platform-test", 1024, 768));
+
+        // We may not have a window if we're running in test mode
+        SLANG_ASSERT(isTestMode() || gWindow);
+        if (gWindow)
+        {
+            gWindow->events.sizeChanged = [this]() { onSizeChanged(); };
+            gWindow->events.focus = [this]() { onFocus(); };
+            gWindow->events.lostFocus = [this]() { onLostFocus(); };
+            gWindow->events.keyDown = [this](const platform::KeyEventArgs& e) { onKeyDown(e); };
+            gWindow->events.keyUp = [this](const platform::KeyEventArgs& e) { onKeyUp(e); };
+            gWindow->events.keyPress = [this](const platform::KeyEventArgs& e) { onKeyPress(e); };
+            gWindow->events.mouseMove = [this](const platform::MouseEventArgs& e)
+            { onMouseMove(e); };
+            gWindow->events.mouseDown = [this](const platform::MouseEventArgs& e)
+            { onMouseDown(e); };
+            gWindow->events.mouseUp = [this](const platform::MouseEventArgs& e) { onMouseUp(e); };
+            gWindow->events.mouseWheel = [this](const platform::MouseEventArgs& e)
+            { onMouseWheel(e); };
+        }
 
         return SLANG_OK;
     }
@@ -105,7 +113,10 @@ struct PlatformTest : public WindowedAppBase
         commandBuffer->close();
         gQueue->executeCommandBuffer(commandBuffer);
 
-        gSwapchain->present();
+        // We may not have a swapchain if we're running in test mode
+        SLANG_ASSERT(isTestMode() || gSwapchain);
+        if (gSwapchain)
+            gSwapchain->present();
     }
 };
 
diff --git a/examples/ray-tracing-pipeline/main.cpp b/examples/ray-tracing-pipeline/main.cpp
index 1e55c7eb1..0fbb857f0 100644
--- a/examples/ray-tracing-pipeline/main.cpp
+++ b/examples/ray-tracing-pipeline/main.cpp
@@ -298,7 +298,7 @@ struct RayTracing : public WindowedAppBase
 
     Slang::Result initialize()
     {
-        initializeBase("Ray Tracing Pipeline", 1024, 768);
+        SLANG_RETURN_ON_FAIL(initializeBase("Ray Tracing Pipeline", 1024, 768));
         if (!isTestMode())
         {
             gWindow->events.mouseMove = [this](const platform::MouseEventArgs& e)
diff --git a/examples/ray-tracing/main.cpp b/examples/ray-tracing/main.cpp
index 7878550f8..11529a753 100644
--- a/examples/ray-tracing/main.cpp
+++ b/examples/ray-tracing/main.cpp
@@ -289,7 +289,7 @@ struct RayTracing : public WindowedAppBase
 
     Slang::Result initialize()
     {
-        initializeBase("Ray Tracing", 1024, 768);
+        SLANG_RETURN_ON_FAIL(initializeBase("Ray Tracing", 1024, 768));
 
         if (!isTestMode())
         {
diff --git a/examples/shader-toy/main.cpp b/examples/shader-toy/main.cpp
index 1ab85c0b2..185d18246 100644
--- a/examples/shader-toy/main.cpp
+++ b/examples/shader-toy/main.cpp
@@ -282,10 +282,18 @@ struct ShaderToyApp : public WindowedAppBase
 
     Result initialize()
     {
-        initializeBase("Shader Toy", 1024, 768);
-        gWindow->events.mouseMove = [this](const platform::MouseEventArgs& e) { handleEvent(e); };
-        gWindow->events.mouseUp = [this](const platform::MouseEventArgs& e) { handleEvent(e); };
-        gWindow->events.mouseDown = [this](const platform::MouseEventArgs& e) { handleEvent(e); };
+        SLANG_RETURN_ON_FAIL(initializeBase("Shader Toy", 1024, 768));
+
+        // We may not have a window if we're running in test mode
+        SLANG_ASSERT(isTestMode() || gWindow);
+        if (gWindow)
+        {
+            gWindow->events.mouseMove = [this](const platform::MouseEventArgs& e)
+            { handleEvent(e); };
+            gWindow->events.mouseUp = [this](const platform::MouseEventArgs& e) { handleEvent(e); };
+            gWindow->events.mouseDown = [this](const platform::MouseEventArgs& e)
+            { handleEvent(e); };
+        }
 
         InputElementDesc inputElements[] = {
             {"POSITION", 0, Format::R32G32_FLOAT, offsetof(FullScreenTriangle::Vertex, position)},
@@ -383,7 +391,11 @@ struct ShaderToyApp : public WindowedAppBase
         commandBuffer->close();
 
         gQueue->executeCommandBuffer(commandBuffer);
-        gSwapchain->present();
+
+        // We may not have a swapchain if we're running in test mode
+        SLANG_ASSERT(isTestMode() || gSwapchain);
+        if (gSwapchain)
+            gSwapchain->present();
     }
 
     void handleEvent(const platform::MouseEventArgs& event)
diff --git a/examples/triangle/main.cpp b/examples/triangle/main.cpp
index e77f488cc..f757b59c7 100644
--- a/examples/triangle/main.cpp
+++ b/examples/triangle/main.cpp
@@ -223,7 +223,7 @@ struct HelloWorld : public WindowedAppBase
     {
         // Create a window for our application to render into.
         //
-        initializeBase("hello-world", 1024, 768);
+        SLANG_RETURN_ON_FAIL(initializeBase("hello-world", 1024, 768));
 
         // We will create objects needed to configur the "input assembler"
         // (IA) stage of the D3D pipeline.
diff --git a/tests/expected-example-failure-github.txt b/tests/expected-example-failure-github.txt
new file mode 100644
index 000000000..f793383cc
--- /dev/null
+++ b/tests/expected-example-failure-github.txt
@@ -0,0 +1,18 @@
+linux:(debug|release):gpu-printing           # See issue 5879
+linux:(debug|release):hello-world            # See issue 5879
+linux:(debug|release):model-viewer           # See issue 5879
+linux:(debug|release):platform-test          # See issue 5879
+linux:(debug|release):ray-tracing            # See issue 5879
+linux:(debug|release):ray-tracing-pipeline   # See issue 5879
+linux:(debug|release):reflection-api         # See issue 5879
+linux:(debug|release):shader-object          # See issue 5879
+linux:(debug|release):shader-toy             # See issue 5879
+linux:(debug|release):triangle               # See issue 5879
+macos:(debug|release):hello-world            # See issue 5879
+macos:(debug|release):model-viewer           # See issue 5879
+macos:(debug|release):ray-tracing            # See issue 5879
+macos:(debug|release):ray-tracing-pipeline   # See issue 5879
+windows:(debug|release):ray-tracing          # See issue 5879
+windows:(debug|release):ray-tracing-pipeline # See issue 5879
+windows:(debug|release):reflection-api       # See issue 5879
+windows:debug:hello-world                    # See issue 5879