Enhances CI reliability and debug logging (#8393)

1. Adds retry logic - Implements 3-attempt retry mechanism for
intermittent test failures
2. Reduces parallelism for Linux - Changes server counts back to 1 for
more stable execution, given the test is short in Linux for now
3. Adds detailed error logging - Enhanced diagnostic information for
test parsing failures
4. Add Python 3.10 setup for slangpy tests for github runners
5. Removes submodule checkout for slang and slasngpy tests
6. Adds check-ci job - Implements consolidated CI status for simplified
branch protection rule

---------

Co-authored-by: slangbot <ellieh+slangbot@nvidia.com>
Co-authored-by: slangbot <186143334+slangbot@users.noreply.github.com>

4 files changed, 157 insertions, 26 deletions

diff --git a/.github/actions/common-test-setup/action.yml b/.github/actions/common-test-setup/action.yml
index 3fc6dd617..1b1698e1a 100644
--- a/.github/actions/common-test-setup/action.yml
+++ b/.github/actions/common-test-setup/action.yml
@@ -72,11 +72,20 @@ runs:
       run: |
         echo "Checking supported backends"
         # Capture the output of slang-test while also displaying it
-        if ! smokeResult=$("$bin_dir/slang-test" tests/render/check-backend-support-on-ci.slang 2>&1); then
-          echo "❌ ERROR: slang-test failed to run"
-          echo "Output: $smokeResult"
-          exit 1
-        fi
+        # Add retry logic for intermittent failures
+        for attempt in 1 2 3; do
+          if smokeResult=$("$bin_dir/slang-test" "tests/render/check-backend-support-on-ci.slang" 2>&1); then
+            break
+          else
+            echo "⚠️  Attempt $attempt failed, retrying..."
+            if [ $attempt -eq 3 ]; then
+              echo "❌ ERROR: slang-test failed to run after 3 attempts"
+              echo "Output: $smokeResult"
+              exit 1
+            fi
+            sleep 2
+          fi
+        done
         supportedBackends="$(echo "$smokeResult" | grep 'Supported backends: ')"
         echo "$smokeResult"
         echo "$supportedBackends"
diff --git a/.github/workflows/ci-slang-test.yml b/.github/workflows/ci-slang-test.yml
index 4e4a880d9..04d11bdde 100644
--- a/.github/workflows/ci-slang-test.yml
+++ b/.github/workflows/ci-slang-test.yml
@@ -62,8 +62,6 @@ jobs:
           fi
           # Build common slang-test arguments
           slang_test_args=(
-            "-use-test-server"
-            "-server-count" "${{ inputs.server-count }}"
             "-category" "${{ inputs.test-category }}"
             "-expected-failure-list" "tests/expected-failure-github.txt"
             "-skip-reference-image-generation"
@@ -71,6 +69,12 @@ jobs:
             "-enable-debug-layers" "${{ inputs.enable-debug-layers }}"
           )
 
+          # Add test server arguments only if server count > 1
+          if [ "${{ inputs.server-count }}" -gt 1 ]; then
+            slang_test_args+=("-use-test-server")
+            slang_test_args+=("-server-count" "${{ inputs.server-count }}")
+          fi
+
           # Add no-GPU failure list for non-GPU tests
           if [[ "${{ inputs.full-gpu-tests }}" != "true" ]]; then
             slang_test_args+=("-expected-failure-list" "tests/expected-failure-no-gpu.txt")
@@ -154,9 +158,6 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-        with:
-          submodules: "recursive"
-          fetch-depth: "1"
 
       - name: Common Test Setup
         uses: ./.github/actions/common-test-setup
@@ -166,6 +167,12 @@ jobs:
           platform: ${{ inputs.platform }}
           config: ${{ inputs.config }}
 
+      - name: Setup Python
+        if: ${{ runner.environment != 'self-hosted' }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
       - name: Run slangpy tests
         run: |
           python --version
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index cbf5fe77e..f155cc6d9 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -135,7 +135,7 @@ jobs:
       config: debug
       runs-on: '["ubuntu-22.04"]'
       test-category: smoke
-      server-count: 2
+      server-count: 1
 
   test-linux-release-gcc-x86_64:
     needs: [filter, build-linux-release-gcc-x86_64]
@@ -148,7 +148,7 @@ jobs:
       config: release
       runs-on: '["ubuntu-22.04"]'
       test-category: full
-      server-count: 4
+      server-count: 1
 
   # macOS tests
   test-macos-debug-clang-aarch64:
@@ -204,3 +204,39 @@ jobs:
       runs-on: '["Windows", "self-hosted", "GCP-T4"]'
       test-category: full
       full-gpu-tests: true
+
+  check-ci:
+    needs:
+      [
+        test-windows-release-cl-x86_64-gpu,
+        test-windows-debug-cl-x86_64-gpu,
+        test-macos-release-clang-aarch64,
+        test-macos-debug-clang-aarch64,
+        test-linux-release-gcc-x86_64,
+        test-linux-debug-gcc-x86_64,
+      ]
+    runs-on: ubuntu-latest
+    if: always() # Always run, even if dependencies fail
+    steps:
+      - name: Check CI Results
+        run: |
+          echo "=== CI Results Summary ==="
+          echo "Windows Release GPU: ${{ needs.test-windows-release-cl-x86_64-gpu.result }}"
+          echo "Windows Debug GPU: ${{ needs.test-windows-debug-cl-x86_64-gpu.result }}"
+          echo "macOS Release ARM64: ${{ needs.test-macos-release-clang-aarch64.result }}"
+          echo "macOS Debug ARM64: ${{ needs.test-macos-debug-clang-aarch64.result }}"
+          echo "Linux Release x64: ${{ needs.test-linux-release-gcc-x86_64.result }}"
+          echo "Linux Debug x64: ${{ needs.test-linux-debug-gcc-x86_64.result }}"
+
+          # Check if all required jobs succeeded
+          if [[ "${{ needs.test-windows-release-cl-x86_64-gpu.result }}" != "success" ]] || \
+             [[ "${{ needs.test-windows-debug-cl-x86_64-gpu.result }}" != "success" ]] || \
+             [[ "${{ needs.test-macos-release-clang-aarch64.result }}" != "success" ]] || \
+             [[ "${{ needs.test-macos-debug-clang-aarch64.result }}" != "success" ]] || \
+             [[ "${{ needs.test-linux-release-gcc-x86_64.result }}" != "success" ]] || \
+             [[ "${{ needs.test-linux-debug-gcc-x86_64.result }}" != "success" ]]; then
+            echo "❌ One or more CI jobs failed or were cancelled"
+            exit 1
+          fi
+
+          echo "✅ All CI jobs passed successfully!"
diff --git a/tools/slang-test/slang-test-main.cpp b/tools/slang-test/slang-test-main.cpp
index 87441c6c4..0297227d6 100644
--- a/tools/slang-test/slang-test-main.cpp
+++ b/tools/slang-test/slang-test-main.cpp
@@ -549,13 +549,36 @@ static void applyMacroSubstitution(String filePath, TestDetails& details)
 static SlangResult _gatherTestsForFile(
     TestCategorySet* categorySet,
     String filePath,
-    FileTestList* outTestList)
+    FileTestList* outTestList,
+    TestContext* context = nullptr)
 {
     outTestList->tests.clear();
 
     String fileContents;
 
-    SLANG_RETURN_ON_FAIL(Slang::File::readAllText(filePath, fileContents));
+    SlangResult readResult = Slang::File::readAllText(filePath, fileContents);
+    if (SLANG_FAILED(readResult))
+    {
+        // Log file reading failure with details (thread-safe)
+        if (context && context->getTestReporter())
+        {
+            context->getTestReporter()->messageFormat(
+                TestMessageType::RunError,
+                "Failed to read test file '%s' (error: 0x%08X)",
+                filePath.getBuffer(),
+                (unsigned int)readResult);
+        }
+        else
+        {
+            // Fallback to stderr if no context available
+            fprintf(
+                stderr,
+                "Failed to read test file '%s' (error: 0x%08X)\n",
+                filePath.getBuffer(),
+                (unsigned int)readResult);
+        }
+        return readResult;
+    }
 
     // Walk through the lines of the file, looking for test commands
     char const* cursor = fileContents.begin();
@@ -614,9 +637,27 @@ static SlangResult _gatherTestsForFile(
         {
             SlangResult res = _parseCategories(categorySet, &cursor, fileOptions);
 
-            // If if failed we are done, unless it was just 'not available'
+            // If it failed we are done, unless it was just 'not available'
             if (SLANG_FAILED(res) && res != SLANG_E_NOT_AVAILABLE)
+            {
+                if (context && context->getTestReporter())
+                {
+                    context->getTestReporter()->messageFormat(
+                        TestMessageType::RunError,
+                        "Failed to parse TEST_CATEGORY in file '%s' (error: 0x%08X)",
+                        filePath.getBuffer(),
+                        (unsigned int)res);
+                }
+                else
+                {
+                    fprintf(
+                        stderr,
+                        "Failed to parse TEST_CATEGORY in file '%s' (error: 0x%08X)\n",
+                        filePath.getBuffer(),
+                        (unsigned int)res);
+                }
                 return res;
+            }
 
             skipToEndOfLine(&cursor);
             continue;
@@ -624,7 +665,27 @@ static SlangResult _gatherTestsForFile(
 
         if (command == "TEST")
         {
-            SLANG_RETURN_ON_FAIL(_gatherTestOptions(categorySet, &cursor, testDetails.options));
+            SlangResult testRes = _gatherTestOptions(categorySet, &cursor, testDetails.options);
+            if (SLANG_FAILED(testRes))
+            {
+                if (context && context->getTestReporter())
+                {
+                    context->getTestReporter()->messageFormat(
+                        TestMessageType::RunError,
+                        "Failed to parse TEST directive in file '%s' (error: 0x%08X)",
+                        filePath.getBuffer(),
+                        (unsigned int)testRes);
+                }
+                else
+                {
+                    fprintf(
+                        stderr,
+                        "Failed to parse TEST directive in file '%s' (error: 0x%08X)\n",
+                        filePath.getBuffer(),
+                        (unsigned int)testRes);
+                }
+                return testRes;
+            }
             applyMacroSubstitution(filePath, testDetails);
 
             // See if the type of test needs certain APIs available
@@ -639,7 +700,27 @@ static SlangResult _gatherTestsForFile(
         }
         else if (command == "DIAGNOSTIC_TEST")
         {
-            SLANG_RETURN_ON_FAIL(_gatherTestOptions(categorySet, &cursor, testDetails.options));
+            SlangResult diagRes = _gatherTestOptions(categorySet, &cursor, testDetails.options);
+            if (SLANG_FAILED(diagRes))
+            {
+                if (context && context->getTestReporter())
+                {
+                    context->getTestReporter()->messageFormat(
+                        TestMessageType::RunError,
+                        "Failed to parse DIAGNOSTIC_TEST directive in file '%s' (error: 0x%08X)",
+                        filePath.getBuffer(),
+                        (unsigned int)diagRes);
+                }
+                else
+                {
+                    fprintf(
+                        stderr,
+                        "Failed to parse DIAGNOSTIC_TEST directive in file '%s' (error: 0x%08X)\n",
+                        filePath.getBuffer(),
+                        (unsigned int)diagRes);
+                }
+                return diagRes;
+            }
             applyMacroSubstitution(filePath, testDetails);
 
             // Apply the file wide options
@@ -1637,7 +1718,7 @@ String getOutput(const ExecuteResult& exeRes, bool removeEmbeddedSource = false)
     String debugLayer = exeRes.debugLayer;
 
     // Apply embedded source removal to standard output if requested
-    if (removeEmbeddedSource)
+    if (removeEmbeddedSource && standardOuptut.getLength() > 0)
     {
         standardOuptut = removeEmbeddedSourceFromSPIRV(standardOuptut);
     }
@@ -4482,7 +4563,7 @@ static SlangResult _runTestsOnFile(TestContext* context, String filePath)
     // Gather a list of tests to run
     FileTestList testList;
 
-    SLANG_RETURN_ON_FAIL(_gatherTestsForFile(&context->categorySet, filePath, &testList));
+    SLANG_RETURN_ON_FAIL(_gatherTestsForFile(&context->categorySet, filePath, &testList, context));
 
     if (testList.tests.getCount() == 0)
     {
@@ -4772,17 +4853,15 @@ void runTestsInDirectory(TestContext* context)
     {
         if (shouldRunTest(context, file))
         {
-            if (context->options.verbosity >= VerbosityLevel::Info)
-            {
-                printf("found test: '%s'\n", file.getBuffer());
-            }
-            if (SLANG_FAILED(_runTestsOnFile(context, file)))
+            SlangResult result = _runTestsOnFile(context, file);
+            if (SLANG_FAILED(result))
             {
                 {
                     TestReporter::TestScope scope(context->getTestReporter(), file);
-                    context->getTestReporter()->message(
+                    context->getTestReporter()->messageFormat(
                         TestMessageType::RunError,
-                        "slang-test: unable to parse test");
+                        "slang-test: unable to parse test (error code: 0x%08X)",
+                        (unsigned int)result);
 
                     context->getTestReporter()->addResult(TestResult::Fail);
                 }