Enhances CI reliability and debug logging (#8393)

1. Adds retry logic - Implements 3-attempt retry mechanism for
intermittent test failures
2. Reduces parallelism for Linux - Changes server counts back to 1 for
more stable execution, given the test is short in Linux for now
3. Adds detailed error logging - Enhanced diagnostic information for
test parsing failures
4. Add Python 3.10 setup for slangpy tests for github runners
5. Removes submodule checkout for slang and slasngpy tests
6. Adds check-ci job - Implements consolidated CI status for simplified
branch protection rule

---------

Co-authored-by: slangbot <ellieh+slangbot@nvidia.com>
Co-authored-by: slangbot <186143334+slangbot@users.noreply.github.com>

3 files changed, 64 insertions, 12 deletions

diff --git a/.github/actions/common-test-setup/action.yml b/.github/actions/common-test-setup/action.yml
index 3fc6dd617..1b1698e1a 100644
--- a/.github/actions/common-test-setup/action.yml
+++ b/.github/actions/common-test-setup/action.yml
@@ -72,11 +72,20 @@ runs:
       run: |
         echo "Checking supported backends"
         # Capture the output of slang-test while also displaying it
-        if ! smokeResult=$("$bin_dir/slang-test" tests/render/check-backend-support-on-ci.slang 2>&1); then
-          echo "❌ ERROR: slang-test failed to run"
-          echo "Output: $smokeResult"
-          exit 1
-        fi
+        # Add retry logic for intermittent failures
+        for attempt in 1 2 3; do
+          if smokeResult=$("$bin_dir/slang-test" "tests/render/check-backend-support-on-ci.slang" 2>&1); then
+            break
+          else
+            echo "⚠️  Attempt $attempt failed, retrying..."
+            if [ $attempt -eq 3 ]; then
+              echo "❌ ERROR: slang-test failed to run after 3 attempts"
+              echo "Output: $smokeResult"
+              exit 1
+            fi
+            sleep 2
+          fi
+        done
         supportedBackends="$(echo "$smokeResult" | grep 'Supported backends: ')"
         echo "$smokeResult"
         echo "$supportedBackends"
diff --git a/.github/workflows/ci-slang-test.yml b/.github/workflows/ci-slang-test.yml
index 4e4a880d9..04d11bdde 100644
--- a/.github/workflows/ci-slang-test.yml
+++ b/.github/workflows/ci-slang-test.yml
@@ -62,8 +62,6 @@ jobs:
           fi
           # Build common slang-test arguments
           slang_test_args=(
-            "-use-test-server"
-            "-server-count" "${{ inputs.server-count }}"
             "-category" "${{ inputs.test-category }}"
             "-expected-failure-list" "tests/expected-failure-github.txt"
             "-skip-reference-image-generation"
@@ -71,6 +69,12 @@ jobs:
             "-enable-debug-layers" "${{ inputs.enable-debug-layers }}"
           )
 
+          # Add test server arguments only if server count > 1
+          if [ "${{ inputs.server-count }}" -gt 1 ]; then
+            slang_test_args+=("-use-test-server")
+            slang_test_args+=("-server-count" "${{ inputs.server-count }}")
+          fi
+
           # Add no-GPU failure list for non-GPU tests
           if [[ "${{ inputs.full-gpu-tests }}" != "true" ]]; then
             slang_test_args+=("-expected-failure-list" "tests/expected-failure-no-gpu.txt")
@@ -154,9 +158,6 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-        with:
-          submodules: "recursive"
-          fetch-depth: "1"
 
       - name: Common Test Setup
         uses: ./.github/actions/common-test-setup
@@ -166,6 +167,12 @@ jobs:
           platform: ${{ inputs.platform }}
           config: ${{ inputs.config }}
 
+      - name: Setup Python
+        if: ${{ runner.environment != 'self-hosted' }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
       - name: Run slangpy tests
         run: |
           python --version
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index cbf5fe77e..f155cc6d9 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -135,7 +135,7 @@ jobs:
       config: debug
       runs-on: '["ubuntu-22.04"]'
       test-category: smoke
-      server-count: 2
+      server-count: 1
 
   test-linux-release-gcc-x86_64:
     needs: [filter, build-linux-release-gcc-x86_64]
@@ -148,7 +148,7 @@ jobs:
       config: release
       runs-on: '["ubuntu-22.04"]'
       test-category: full
-      server-count: 4
+      server-count: 1
 
   # macOS tests
   test-macos-debug-clang-aarch64:
@@ -204,3 +204,39 @@ jobs:
       runs-on: '["Windows", "self-hosted", "GCP-T4"]'
       test-category: full
       full-gpu-tests: true
+
+  check-ci:
+    needs:
+      [
+        test-windows-release-cl-x86_64-gpu,
+        test-windows-debug-cl-x86_64-gpu,
+        test-macos-release-clang-aarch64,
+        test-macos-debug-clang-aarch64,
+        test-linux-release-gcc-x86_64,
+        test-linux-debug-gcc-x86_64,
+      ]
+    runs-on: ubuntu-latest
+    if: always() # Always run, even if dependencies fail
+    steps:
+      - name: Check CI Results
+        run: |
+          echo "=== CI Results Summary ==="
+          echo "Windows Release GPU: ${{ needs.test-windows-release-cl-x86_64-gpu.result }}"
+          echo "Windows Debug GPU: ${{ needs.test-windows-debug-cl-x86_64-gpu.result }}"
+          echo "macOS Release ARM64: ${{ needs.test-macos-release-clang-aarch64.result }}"
+          echo "macOS Debug ARM64: ${{ needs.test-macos-debug-clang-aarch64.result }}"
+          echo "Linux Release x64: ${{ needs.test-linux-release-gcc-x86_64.result }}"
+          echo "Linux Debug x64: ${{ needs.test-linux-debug-gcc-x86_64.result }}"
+
+          # Check if all required jobs succeeded
+          if [[ "${{ needs.test-windows-release-cl-x86_64-gpu.result }}" != "success" ]] || \
+             [[ "${{ needs.test-windows-debug-cl-x86_64-gpu.result }}" != "success" ]] || \
+             [[ "${{ needs.test-macos-release-clang-aarch64.result }}" != "success" ]] || \
+             [[ "${{ needs.test-macos-debug-clang-aarch64.result }}" != "success" ]] || \
+             [[ "${{ needs.test-linux-release-gcc-x86_64.result }}" != "success" ]] || \
+             [[ "${{ needs.test-linux-debug-gcc-x86_64.result }}" != "success" ]]; then
+            echo "❌ One or more CI jobs failed or were cancelled"
+            exit 1
+          fi
+
+          echo "✅ All CI jobs passed successfully!"