Run vk tests on spirv backend with expected failure list. (#3128)

13 files changed, 297 insertions, 31 deletions

diff --git a/.github/workflows/windows-selfhosted.yml b/.github/workflows/windows-selfhosted.yml
index 414f7515c..9b7c7b6eb 100644
--- a/.github/workflows/windows-selfhosted.yml
+++ b/.github/workflows/windows-selfhosted.yml
@@ -36,9 +36,13 @@ jobs:
         .\make-slang-tag-version.bat
         
         MSBuild.exe slang.sln -v:m -m -property:Configuration=${{matrix.configuration}} -property:Platform=${{matrix.platform}} -property:WindowsTargetPlatformVersion=10.0.19041.0 -maxcpucount:12
+    - name: test-spirv-direct
+      run: |
+        set PATH=%PATH%;.\external\slang-binaries\spirv-tools\windows-${{matrix.testPlatform}}\bin\
+        ".\bin\windows-${{matrix.testPlatform}}\${{matrix.configuration}}\slang-test.exe" tests/ -use-test-server -emit-spirv-directly -expected-failure-list tests/expected-failure.txt -api vk 2>&1
+      shell: cmd
     - name: test
       run: |
-        $slangTestBinDir = ".\bin\windows-${{matrix.testPlatform}}\${{matrix.configuration}}\";
-        $spirvToolsBinDir = ".\external\slang-binaries\spirv-tools\windows-${{matrix.testPlatform}}\bin\";
-        $env:Path += ";$slangTestBinDir;$spirvToolsBinDir";
-        & "$slangTestBinDir\slang-test.exe" -appveyor -bindir "$slangTestBinDir\" -platform ${{matrix.testPlatform}} -configuration ${{matrix.configuration}} -category ${{matrix.testCategory}} -api all-cpu 2>&1;
+        set PATH=%PATH%;.\external\slang-binaries\spirv-tools\windows-${{matrix.testPlatform}}\bin\
+        ".\bin\windows-${{matrix.testPlatform}}\${{matrix.configuration}}\slang-test.exe" -use-test-server -api vk 2>&1
+      shell: cmd
diff --git a/source/core/slang-process-util.cpp b/source/core/slang-process-util.cpp
index ef22c0392..99c81ee5c 100644
--- a/source/core/slang-process-util.cpp
+++ b/source/core/slang-process-util.cpp
@@ -57,7 +57,8 @@ static String _getText(const ConstArrayView<Byte>& bytes)
         const auto preCount = _getCount(outStdOut) + _getCount(outStdError);
 
         SLANG_RETURN_ON_FAIL(StreamUtil::readOrDiscard(stdOutStream, 0, outStdOut));
-        SLANG_RETURN_ON_FAIL(StreamUtil::readOrDiscard(stdErrorStream, 0, outStdError));
+        if (stdErrorStream)
+            SLANG_RETURN_ON_FAIL(StreamUtil::readOrDiscard(stdErrorStream, 0, outStdError));
 
         const auto postCount = _getCount(outStdOut) + _getCount(outStdError);
 
@@ -69,9 +70,16 @@ static String _getText(const ConstArrayView<Byte>& bytes)
     }
 
     // Read anything remaining
-    SLANG_RETURN_ON_FAIL(StreamUtil::readOrDiscardAll(stdOutStream, 0, outStdOut));
-    SLANG_RETURN_ON_FAIL(StreamUtil::readOrDiscardAll(stdErrorStream, 0, outStdError));
-
+    for(;;)
+    {
+        const auto preCount = _getCount(outStdOut) + _getCount(outStdError);
+        StreamUtil::readOrDiscard(stdOutStream, 0, outStdOut);
+        if (stdErrorStream)
+            StreamUtil::readOrDiscard(stdErrorStream, 0, outStdError);
+        const auto postCount = _getCount(outStdOut) + _getCount(outStdError);
+        if (preCount == postCount)
+            break;
+    }
     return SLANG_OK;
 }
 
diff --git a/source/core/slang-process.h b/source/core/slang-process.h
index ab54f36c9..0beb78c06 100644
--- a/source/core/slang-process.h
+++ b/source/core/slang-process.h
@@ -23,6 +23,7 @@ public:
         {
             // Ignored on non-Windows platforms
             AttachDebugger = 0x01,
+            DisableStdErrRedirection = 0x02
         };
     };
 
diff --git a/source/core/windows/slang-win-process.cpp b/source/core/windows/slang-win-process.cpp
index 93bff3eb3..e577da0fa 100644
--- a/source/core/windows/slang-win-process.cpp
+++ b/source/core/windows/slang-win-process.cpp
@@ -419,8 +419,11 @@ void WinProcess::kill(int32_t returnCode)
             WinHandle childStdInWriteTmp;
             // create stdout pipe for child process
             SLANG_RETURN_FAIL_ON_FALSE(CreatePipe(childStdOutReadTmp.writeRef(), childStdOutWrite.writeRef(), &securityAttributes, bufferSize));
-            // create stderr pipe for child process
-            SLANG_RETURN_FAIL_ON_FALSE(CreatePipe(childStdErrReadTmp.writeRef(), childStdErrWrite.writeRef(), &securityAttributes, bufferSize));
+            if ((flags & Process::Flag::DisableStdErrRedirection) == 0)
+            {
+                // create stderr pipe for child process
+                SLANG_RETURN_FAIL_ON_FALSE(CreatePipe(childStdErrReadTmp.writeRef(), childStdErrWrite.writeRef(), &securityAttributes, bufferSize));
+            }
             // create stdin pipe for child process        
             SLANG_RETURN_FAIL_ON_FALSE(CreatePipe(childStdInRead.writeRef(), childStdInWriteTmp.writeRef(), &securityAttributes, bufferSize));
 
@@ -431,7 +434,8 @@ void WinProcess::kill(int32_t returnCode)
             // create a non-inheritable duplicate of the stdout reader        
             SLANG_RETURN_FAIL_ON_FALSE(DuplicateHandle(currentProcess, childStdOutReadTmp, currentProcess, childStdOutRead.writeRef(), 0, FALSE, DUPLICATE_SAME_ACCESS));
             // create a non-inheritable duplicate of the stderr reader
-            SLANG_RETURN_FAIL_ON_FALSE(DuplicateHandle(currentProcess, childStdErrReadTmp, currentProcess, childStdErrRead.writeRef(), 0, FALSE, DUPLICATE_SAME_ACCESS));
+            if (childStdErrReadTmp)
+                SLANG_RETURN_FAIL_ON_FALSE(DuplicateHandle(currentProcess, childStdErrReadTmp, currentProcess, childStdErrRead.writeRef(), 0, FALSE, DUPLICATE_SAME_ACCESS));
             // create a non-inheritable duplicate of the stdin writer
             SLANG_RETURN_FAIL_ON_FALSE(DuplicateHandle(currentProcess, childStdInWriteTmp, currentProcess, childStdInWrite.writeRef(), 0, FALSE, DUPLICATE_SAME_ACCESS));
         }
@@ -522,7 +526,8 @@ void WinProcess::kill(int32_t returnCode)
     }
 
     RefPtr<Stream> streams[Index(StdStreamType::CountOf)];
-    streams[Index(StdStreamType::ErrorOut)] = new WinPipeStream(childStdErrRead.detach(), FileAccess::Read);
+    if (childStdErrRead)
+        streams[Index(StdStreamType::ErrorOut)] = new WinPipeStream(childStdErrRead.detach(), FileAccess::Read);
     streams[Index(StdStreamType::Out)] = new WinPipeStream(childStdOutRead.detach(), FileAccess::Read);
     streams[Index(StdStreamType::In)] = new WinPipeStream(childStdInWrite.detach(), FileAccess::Write);
 
diff --git a/test.ps1 b/test.ps1
new file mode 100644
index 000000000..9a53035ba
--- /dev/null
+++ b/test.ps1
@@ -0,0 +1 @@
+& ".\bin\windows-x64\release\slang-test.exe" tests/  -use-test-server -emit-spirv-directly -expected-failure-list tests/expected-failure.txt -api vk 2>&1;
\ No newline at end of file
diff --git a/tests/expected-failure.txt b/tests/expected-failure.txt
new file mode 100644
index 000000000..af19025e4
--- /dev/null
+++ b/tests/expected-failure.txt
@@ -0,0 +1,179 @@
+tests/autodiff/array-param.slang.1 (vk)
+tests/autodiff/bool-return-control-flow.slang.1 (vk)
+tests/autodiff/dynamic-dispatch-generic-member-2.slang.1 (vk)
+tests/autodiff/dynamic-object-bwd-diff.slang.1 (vk)
+tests/autodiff/float-cast.slang.1 (vk)
+tests/autodiff/fwd-array-out-param.slang.1 (vk)
+tests/autodiff/global-param-hoisting.slang.1 (vk)
+tests/autodiff/high-order-backward-diff-3.slang.2 (vk)
+tests/autodiff/high-order-backward-diff-4.slang.2 (vk)
+tests/autodiff/high-order-builtins-2.slang.2 (vk)
+tests/autodiff/long-loop-branching-addition.slang.1 (vk)
+tests/autodiff/long-loop-chained-addition.slang.1 (vk)
+tests/autodiff/long-loop-multiple.slang.1 (vk)
+tests/autodiff/long-loop-noninductive.slang.1 (vk)
+tests/autodiff/long-loop.slang.1 (vk)
+tests/autodiff/long-while-loop.slang.1 (vk)
+tests/autodiff/matrix-arithmetic-fwd.slang.1 (vk)
+tests/autodiff/nested-jvp.slang.1 (vk)
+tests/autodiff/reverse-addr-eliminate.slang.1 (vk)
+tests/autodiff/reverse-array-out-param.slang.1 (vk)
+tests/autodiff/reverse-continue-loop.slang.1 (vk)
+tests/autodiff/reverse-control-flow-3.slang (vk)
+tests/autodiff/reverse-do-while.slang.1 (vk)
+tests/autodiff/reverse-hybrid-control-flow.slang.1 (vk)
+tests/autodiff/reverse-inout-param-1.slang.1 (vk)
+tests/autodiff/reverse-inout-param-2.slang.1 (vk)
+tests/autodiff/reverse-loop.slang.1 (vk)
+tests/autodiff/reverse-matrix-ops.slang.1 (vk)
+tests/autodiff/reverse-more-loops.slang (vk)
+tests/autodiff/reverse-nested-loop.slang.1 (vk)
+tests/autodiff/reverse-switch-case.slang.1 (vk)
+tests/autodiff/reverse-vector-arithmetic.slang.1 (vk)
+tests/autodiff/reverse-while-loop-2.slang.1 (vk)
+tests/autodiff/reverse-while-loop.slang.1 (vk)
+tests/autodiff/select.slang.1 (vk)
+tests/autodiff/bsdf/bsdf-sample.slang (vk)
+tests/autodiff/material/diff-bwd-falcor-material-system.slang (vk)
+tests/autodiff/material/diff-falcor-material-system.slang (vk)
+tests/autodiff/material2/diff-bwd-falcor-material-system.slang (vk)
+tests/autodiff/material2/diff-falcor-material-system.slang (vk)
+tests/autodiff/path-tracer/pt-loop.slang.1 (vk)
+tests/autodiff-dstdlib/determinant.slang (vk)
+tests/autodiff-dstdlib/dstdlib-abs.slang (vk)
+tests/autodiff-dstdlib/dstdlib-max.slang (vk)
+tests/autodiff-dstdlib/dstdlib-mul-mat-mat.slang (vk)
+tests/autodiff-dstdlib/dstdlib-mul-mat-vec.slang (vk)
+tests/autodiff-dstdlib/dstdlib-mul-vec-mat.slang (vk)
+tests/autodiff-dstdlib/dstdlib-sqrt.slang (vk)
+tests/autodiff-dstdlib/vector-length.slang (vk)
+tests/bugs/atomic-coerce.slang.1 (vk)
+tests/bugs/bool-op.slang.1 (vk)
+tests/bugs/buffer-swizzle-store.slang.1 (vk)
+tests/bugs/byte-address-buffer-interlocked-add-f32.slang (vk)
+tests/bugs/dxbc-double-problem.slang.1 (vk)
+tests/bugs/generic-type-duplication.slang.1 (vk)
+tests/bugs/gh-3075.slang.2 (vk)
+tests/bugs/glsl-static-const-array.slang (vk)
+tests/bugs/loop-optimize.slang.1 (vk)
+tests/bugs/matrix-reshape.slang.1 (vk)
+tests/bugs/nested-switch.slang.1 (vk)
+tests/bugs/parameter-block-load.slang (vk)
+tests/bugs/string-inline.slang.4 (vk)
+tests/bugs/vec-compare.slang.2 (vk)
+tests/bugs/vec-init.slang.2 (vk)
+tests/bugs/inlining/global-const-inline.slang.1 (vk)
+tests/compute/buffer-layout.slang.2 (vk)
+tests/compute/byte-address-buffer.slang.2 (vk)
+tests/compute/column-major.slang.3 (vk)
+tests/compute/dynamic-dispatch-13.slang.2 (vk)
+tests/compute/dynamic-dispatch-14.slang.3 (vk)
+tests/compute/dynamic-dispatch-16.slang (vk)
+tests/compute/dynamic-dispatch-17.slang (vk)
+tests/compute/dynamic-dispatch-18.slang.2 (vk)
+tests/compute/entry-point-uniform-params.slang.2 (vk)
+tests/compute/frem.slang.2 (vk)
+tests/compute/func-cbuffer-param.slang.2 (vk)
+tests/compute/half-calc.slang.1 (vk)
+tests/compute/half-rw-texture-convert.slang.4 (vk)
+tests/compute/half-rw-texture-convert2.slang.4 (vk)
+tests/compute/half-structured-buffer.slang (vk)
+tests/compute/half-vector-calc.slang.1 (vk)
+tests/compute/half-vector-compare.slang.1 (vk)
+tests/compute/interface-shader-param-in-struct.slang.2 (vk)
+tests/compute/kernel-context-threading.slang.5 (vk)
+tests/compute/loop-unroll.slang.5 (vk)
+tests/compute/non-square-column-major.slang.3 (vk)
+tests/compute/non-square-row-major.slang.3 (vk)
+tests/compute/pack-any-value-16bit.slang (vk)
+tests/compute/parameter-block.slang.2 (vk)
+tests/compute/ray-tracing-inline.slang.1 (vk)
+tests/compute/row-major.slang.3 (vk)
+tests/compute/rw-texture-simple.slang.4 (vk)
+tests/compute/semantic.slang.3 (vk)
+tests/compute/static-const-array.slang.1 (vk)
+tests/compute/static-const-matrix-array.slang.1 (vk)
+tests/compute/static-const-vector-array.slang.1 (vk)
+tests/compute/structured-buffer-of-matrices.slang.3 (vk)
+tests/compute/texture-sample-grad-offset-clamp.slang (vk)
+tests/compute/texture-simple.slang.4 (vk)
+tests/compute/texture-simpler.slang (vk)
+tests/compute/vector-scalar-compare.slang.1 (vk)
+tests/cross-compile/fmod.slang.1 (vk)
+tests/cross-compile/get-dimensions.slang.1 (vk)
+tests/cross-compile/glsl-bool-ops.slang.1 (vk)
+tests/hlsl/byte-buffer-load-std430.slang (vk)
+tests/hlsl/glsl-matrix-layout.slang (vk)
+tests/hlsl/packoffset.slang.1 (vk)
+tests/hlsl-intrinsic/asfloat16.slang.3 (vk)
+tests/hlsl-intrinsic/bit-cast-double.slang.3 (vk)
+tests/hlsl-intrinsic/classify-double.slang.3 (vk)
+tests/hlsl-intrinsic/classify-float.slang.3 (vk)
+tests/hlsl-intrinsic/f16tof32.slang.3 (vk)
+tests/hlsl-intrinsic/f32tof16.slang.3 (vk)
+tests/hlsl-intrinsic/literal-int64.slang.4 (vk)
+tests/hlsl-intrinsic/scalar-double-d3d-intrinsic.slang.4 (vk)
+tests/hlsl-intrinsic/scalar-double-simple.slang.4 (vk)
+tests/hlsl-intrinsic/scalar-double-vk-intrinsic.slang.1 (vk)
+tests/hlsl-intrinsic/scalar-float.slang.3 (vk)
+tests/hlsl-intrinsic/scalar-int.slang.3 (vk)
+tests/hlsl-intrinsic/scalar-int64.slang.4 (vk)
+tests/hlsl-intrinsic/scalar-uint.slang.3 (vk)
+tests/hlsl-intrinsic/scalar-uint64.slang.4 (vk)
+tests/hlsl-intrinsic/vector-double-reduced-intrinsic.slang.3 (vk)
+tests/hlsl-intrinsic/vector-float.slang.3 (vk)
+tests/hlsl-intrinsic/vector-int-runtime-index.slang.3 (vk)
+tests/hlsl-intrinsic/vector-int.slang.3 (vk)
+tests/hlsl-intrinsic/wave-active-count-bits.slang.3 (vk)
+tests/hlsl-intrinsic/wave-active-product.slang.3 (vk)
+tests/hlsl-intrinsic/wave-broadcast-lane-at-vk.slang.1 (vk)
+tests/hlsl-intrinsic/wave-diverge.slang.3 (vk)
+tests/hlsl-intrinsic/wave-equality.slang.3 (vk)
+tests/hlsl-intrinsic/wave-get-lane-index.slang.3 (vk)
+tests/hlsl-intrinsic/wave-is-first-lane.slang.3 (vk)
+tests/hlsl-intrinsic/wave-prefix-count-bits.slang.3 (vk)
+tests/hlsl-intrinsic/wave-prefix-product.slang.3 (vk)
+tests/hlsl-intrinsic/wave-prefix-sum.slang.3 (vk)
+tests/hlsl-intrinsic/wave-read-lane-at-vk.slang.1 (vk)
+tests/hlsl-intrinsic/wave-shuffle-vk.slang.3 (vk)
+tests/hlsl-intrinsic/wave-vector.slang.3 (vk)
+tests/hlsl-intrinsic/wave.slang.3 (vk)
+tests/hlsl-intrinsic/active-mask/switch.slang.3 (vk)
+tests/hlsl-intrinsic/bit-cast/bit-cast-16-bit.slang.1 (vk)
+tests/hlsl-intrinsic/byte-address-buffer/byte-address-16bit-vector.slang.2 (vk)
+tests/hlsl-intrinsic/byte-address-buffer/byte-address-16bit.slang.2 (vk)
+tests/hlsl-intrinsic/byte-address-buffer/byte-address-struct.slang.3 (vk)
+tests/hlsl-intrinsic/size-of/align-of-3.slang.3 (vk)
+tests/hlsl-intrinsic/size-of/size-of-3.slang.3 (vk)
+tests/hlsl-intrinsic/wave-mask/wave-active-product.slang.3 (vk)
+tests/hlsl-intrinsic/wave-mask/wave-broadcast-lane-at-vk.slang.1 (vk)
+tests/hlsl-intrinsic/wave-mask/wave-diverge.slang.3 (vk)
+tests/hlsl-intrinsic/wave-mask/wave-equality.slang.3 (vk)
+tests/hlsl-intrinsic/wave-mask/wave-get-active.slang.3 (vk)
+tests/hlsl-intrinsic/wave-mask/wave-get-converged.slang.3 (vk)
+tests/hlsl-intrinsic/wave-mask/wave-is-first-lane.slang.3 (vk)
+tests/hlsl-intrinsic/wave-mask/wave-prefix-product.slang.3 (vk)
+tests/hlsl-intrinsic/wave-mask/wave-prefix-sum.slang.3 (vk)
+tests/hlsl-intrinsic/wave-mask/wave-read-lane-at-vk.slang.1 (vk)
+tests/hlsl-intrinsic/wave-mask/wave-shuffle-vk.slang.3 (vk)
+tests/hlsl-intrinsic/wave-mask/wave-vector.slang.3 (vk)
+tests/hlsl-intrinsic/wave-mask/wave.slang.3 (vk)
+tests/ir/loop-unroll-0.slang.1 (vk)
+tests/ir/string-literal-hash.slang.1 (vk)
+tests/language-feature/general-inline.slang.1 (vk)
+tests/language-feature/simple-inline.slang.1 (vk)
+tests/language-feature/initializer-lists/default-init-16bit-types.slang (vk)
+tests/language-feature/shader-params/interface-shader-param-ordinary.slang.2 (vk)
+tests/language-feature/swizzles/matrix-swizzle-write-array.slang.1 (vk)
+tests/language-feature/swizzles/matrix-swizzle-write-single.slang.1 (vk)
+tests/language-feature/swizzles/matrix-swizzle-write-swizzle.slang.1 (vk)
+tests/language-feature/swizzles/matrix-swizzle-write.slang.1 (vk)
+tests/optimization/func-resource-result/func-resource-result-complex.slang.1 (vk)
+tests/slang-extension/atomic-float-byte-address-buffer.slang.2 (vk)
+tests/slang-extension/atomic-int64-byte-address-buffer.slang.4 (vk)
+tests/slang-extension/atomic-min-max-u64-byte-address-buffer.slang.4 (vk)
+tests/slang-extension/cas-int64-byte-address-buffer.slang.4 (vk)
+tests/slang-extension/exchange-int64-byte-address-buffer.slang.4 (vk)
+tests/slang-extension/realtime-clock.slang.2 (vk)
+tests/spirv/spirv-instruction.slang (vk)
+tests/type/texture-sampler/texture-sampler-2d.slang (vk)
\ No newline at end of file
diff --git a/tools/slang-test/options.cpp b/tools/slang-test/options.cpp
index 8ab9e9ae2..3fb71dd4a 100644
--- a/tools/slang-test/options.cpp
+++ b/tools/slang-test/options.cpp
@@ -3,7 +3,6 @@
 
 #include "../../source/core/slang-string-util.h"
 #include "../../source/core/slang-io.h"
-
 #include <stdio.h>
 #include <stdlib.h>
 
@@ -291,6 +290,27 @@ static bool _isSubCommand(const char* arg)
         {
             optionsOut->skipApiDetection = true;
         }
+        else if (strcmp(arg, "-emit-spirv-directly") == 0)
+        {
+            optionsOut->emitSPIRVDirectly = true;
+        }
+        else if (strcmp(arg, "-expected-failure-list") == 0)
+        {
+            if (argCursor == argEnd)
+            {
+                stdError.print("error: expected operand for '%s'\n", arg);
+                return SLANG_FAIL;
+            }
+            auto fileName = *argCursor++;
+            String text;
+            File::readAllText(fileName, text);
+            List<UnownedStringSlice> lines;
+            StringUtil::split(text.getUnownedSlice(), '\n', lines);
+            for (auto line : lines)
+            {
+                optionsOut->expectedFailureList.add(line);
+            }
+        }
         else
         {
             stdError.print("unknown option '%s'\n", arg);
diff --git a/tools/slang-test/options.h b/tools/slang-test/options.h
index e786a0a74..398334e46 100644
--- a/tools/slang-test/options.h
+++ b/tools/slang-test/options.h
@@ -113,7 +113,11 @@ struct Options
     Slang::String adapter;
 
     // Maximum number of test servers to run.
-    int serverCount = 4;
+    int serverCount = 1;
+
+    bool emitSPIRVDirectly = false;
+
+    Slang::HashSet<Slang::String> expectedFailureList;
 
         /// Parse the args, report any errors into stdError, and write the results into optionsOut
     static SlangResult parse(int argc, char** argv, TestCategorySet* categorySet, Slang::WriterHelper stdError, Options* optionsOut);
diff --git a/tools/slang-test/slang-test-main.cpp b/tools/slang-test/slang-test-main.cpp
index 95ef4d55d..cb2db064d 100644
--- a/tools/slang-test/slang-test-main.cpp
+++ b/tools/slang-test/slang-test-main.cpp
@@ -2992,6 +2992,10 @@ static void _addRenderTestOptions(const Options& options, CommandLine& ioCmdLine
         ioCmdLine.addArg("-adapter");
         ioCmdLine.addArg(options.adapter);
     }
+    if (options.emitSPIRVDirectly)
+    {
+        ioCmdLine.addArg("-emit-spirv-directly");
+    }
 }
 
 static SlangResult _extractProfileTime(const UnownedStringSlice& text, double& timeOut)
@@ -4117,7 +4121,7 @@ void runTestsInDirectory(
         auto threadFunc = [&](int threadId)
         {
             TestReporter reporter;
-            reporter.init(context->options.outputMode, true);
+            reporter.init(context->options.outputMode, context->options.expectedFailureList, true);
             TestReporter::SuiteScope suiteScope(&reporter, "tests");
             context->setThreadIndex(threadId);
             context->setTestReporter(&reporter);
@@ -4275,10 +4279,6 @@ static SlangResult runUnitTestModule(TestContext* context, TestOptions& testOpti
 
 SlangResult innerMain(int argc, char** argv)
 {
-    // Disable buffering for out and std out
-    StreamUtil::setStreamBufferStyle(StdStreamType::Out, StreamBufferStyle::None);
-    StreamUtil::setStreamBufferStyle(StdStreamType::ErrorOut, StreamBufferStyle::None);
-
     auto stdWriters = StdWriters::initDefaultSingleton();
 
     // The context holds useful things used during testing
@@ -4457,7 +4457,7 @@ SlangResult innerMain(int argc, char** argv)
     {
         // Setup the reporter
         TestReporter reporter;
-        SLANG_RETURN_ON_FAIL(reporter.init(options.outputMode));
+        SLANG_RETURN_ON_FAIL(reporter.init(options.outputMode, options.expectedFailureList));
 
         context.setTestReporter(&reporter);
 
diff --git a/tools/slang-test/test-context.cpp b/tools/slang-test/test-context.cpp
index e0f2749b9..39f5c92ed 100644
--- a/tools/slang-test/test-context.cpp
+++ b/tools/slang-test/test-context.cpp
@@ -186,11 +186,12 @@ SlangResult TestContext::_createJSONRPCConnection(RefPtr<JSONRPCConnection>& out
     {
         CommandLine cmdLine;
         cmdLine.setExecutableLocation(ExecutableLocation(exeDirectoryPath, "test-server"));
-        SLANG_RETURN_ON_FAIL(Process::create(cmdLine, Process::Flag::AttachDebugger, process));
+        SLANG_RETURN_ON_FAIL(Process::create(cmdLine, Process::Flag::AttachDebugger | Process::Flag::DisableStdErrRedirection, process));
     }
 
     Stream* writeStream = process->getStream(StdStreamType::In);
     RefPtr<BufferedReadStream> readStream(new BufferedReadStream(process->getStream(StdStreamType::Out)));
+    RefPtr<BufferedReadStream> readErrStream(new BufferedReadStream(process->getStream(StdStreamType::ErrorOut)));
 
     RefPtr<HTTPPacketConnection> connection = new HTTPPacketConnection(readStream, writeStream);
     RefPtr<JSONRPCConnection> rpcConnection = new JSONRPCConnection;
diff --git a/tools/slang-test/test-reporter.cpp b/tools/slang-test/test-reporter.cpp
index ede5823c7..3ac8d90ea 100644
--- a/tools/slang-test/test-reporter.cpp
+++ b/tools/slang-test/test-reporter.cpp
@@ -76,7 +76,7 @@ TestReporter::TestReporter() :
     m_passedTestCount = 0;
     m_failedTestCount = 0;
     m_ignoredTestCount = 0;
-
+    m_expectedFailedTestCount = 0;
     m_maxFailTestResults = 10;
 
     m_inTest = false;
@@ -84,10 +84,11 @@ TestReporter::TestReporter() :
     m_isVerbose = false;
 }
 
-Result TestReporter::init(TestOutputMode outputMode, bool isSubReporter)
+Result TestReporter::init(TestOutputMode outputMode, const HashSet<String>& expectedFailureList, bool isSubReporter)
 {
     m_outputMode = outputMode;
     m_isSubReporter = isSubReporter;
+    m_expectedFailureList = expectedFailureList;
     return SLANG_OK;
 }
 
@@ -141,7 +142,8 @@ void TestReporter::addResult(TestResult result)
     assert(m_inTest);
 
     std::lock_guard<std::recursive_mutex> lock(m_mutex);
-
+    if (result == TestResult::Fail && m_expectedFailureList.contains(m_currentInfo.name))
+        result = TestResult::ExpectedFail;
     m_currentInfo.testResult = combine(m_currentInfo.testResult, result);
     m_numCurrentResults++;
 }
@@ -158,6 +160,7 @@ void TestReporter::addResultWithLocation(TestResult result, const char* testText
     assert(m_inTest);
 
     std::lock_guard<std::recursive_mutex> lock(m_mutex);
+    result = adjustResult(m_currentInfo.name.getUnownedSlice(), result);
 
     m_numCurrentResults++;
 
@@ -207,6 +210,7 @@ void TestReporter::consolidateWith(TestReporter* other)
     m_failedTestCount += other->m_failedTestCount;
     m_ignoredTestCount += other->m_ignoredTestCount;
     m_passedTestCount += other->m_passedTestCount;
+    m_expectedFailedTestCount += other->m_expectedFailedTestCount;
     m_totalTestCount += other->m_totalTestCount;
 }
 
@@ -295,12 +299,13 @@ static void _appendTime(double timeInSec, StringBuilder& out)
     out << timeInSec << "ns";
 }
 
-void TestReporter::_addResult(const TestInfo& info)
+void TestReporter::_addResult(TestInfo info)
 {
     if (info.testResult == TestResult::Ignored && m_hideIgnored)
     {
         return;
     }
+    info.testResult = adjustResult(info.name.getUnownedSlice(), info.testResult);
 
     m_totalTestCount++;
 
@@ -313,6 +318,9 @@ void TestReporter::_addResult(const TestInfo& info)
         case TestResult::Pass:
             m_passedTestCount++;
             break;
+        case TestResult::ExpectedFail:
+            m_expectedFailedTestCount++;
+            break;
 
         case TestResult::Ignored:
             m_ignoredTestCount++;
@@ -333,6 +341,9 @@ void TestReporter::_addResult(const TestInfo& info)
         case TestResult::Fail:
             resultString = "FAILED";
             break;
+        case TestResult::ExpectedFail:
+            resultString = "failed(expected)";
+            break;
         case TestResult::Pass:
             resultString = "passed";
             break;
@@ -383,7 +394,8 @@ void TestReporter::_addResult(const TestInfo& info)
                     }
                     break;
                 }
-                case TestResult::Pass:     
+                case TestResult::Pass:
+                case TestResult::ExpectedFail:
                 {
                     StringBuilder message;
                     message << info.message;
@@ -443,6 +455,8 @@ void TestReporter::_addResult(const TestInfo& info)
                 case TestResult::Fail:      resultString = "Failed";  break;
                 case TestResult::Pass:      resultString = "Passed";  break;
                 case TestResult::Ignored:   resultString = "Ignored"; break;
+                case TestResult::ExpectedFail:   resultString = "ExpectedFail"; break;
+
                 default:
                     assert(!"unexpected");
                     break;
@@ -560,7 +574,7 @@ void TestReporter::message(TestMessageType type, const char* messageContent)
 
 bool TestReporter::didAllSucceed() const
 {
-    return m_passedTestCount == (m_totalTestCount - m_ignoredTestCount);
+    return m_failedTestCount == 0;
 }
 
 void TestReporter::outputSummary()
@@ -592,8 +606,25 @@ void TestReporter::outputSummary()
             {
                 printf(", %d tests ignored", ignoredCount);
             }
+            if (m_expectedFailedTestCount)
+            {
+                printf(", %d tests failed expectedly", m_expectedFailedTestCount);
+                printf("\n===\n\n");
+                printf("\npassing tests that are expected to fail:\n");
+                printf("---\n");
+                for (const auto& testInfo : m_testInfos)
+                {
+                    if (testInfo.testResult == TestResult::Pass)
+                    {
+                        if (m_expectedFailureList.contains(testInfo.name))
+                        {
+                            printf("%s\n", testInfo.name.getBuffer());
+                        }
+                    }
+                }
+                printf("---\n");
+            }
             printf("\n===\n\n");
-
             if (m_failedTestCount)
             {
                 printf("failing tests:\n");
@@ -607,6 +638,7 @@ void TestReporter::outputSummary()
                 }
                 printf("---\n");
             }
+            
             break;
         }
         
@@ -714,3 +746,10 @@ void TestReporter::endSuite()
     
     m_suiteStack.removeLast();
 }
+
+TestResult TestReporter::adjustResult(UnownedStringSlice testName, TestResult result)
+{
+    if (result == TestResult::Fail && m_expectedFailureList.contains(testName))
+        result = TestResult::ExpectedFail;
+    return result;
+}
diff --git a/tools/slang-test/test-reporter.h b/tools/slang-test/test-reporter.h
index b92865f8a..1d2857463 100644
--- a/tools/slang-test/test-reporter.h
+++ b/tools/slang-test/test-reporter.h
@@ -70,6 +70,8 @@ class TestReporter : public ITestReporter
     void startSuite(const Slang::String& name);
     void endSuite();
 
+    TestResult adjustResult(Slang::UnownedStringSlice testName, TestResult result);
+
     virtual SLANG_NO_THROW void SLANG_MCALL startTest(const char* testName) override;
     virtual SLANG_NO_THROW void SLANG_MCALL addResult(TestResult result) override;
     virtual SLANG_NO_THROW void SLANG_MCALL addResultWithLocation(TestResult result, const char* testText, const char* file, int line) override;
@@ -101,7 +103,7 @@ class TestReporter : public ITestReporter
 
     void outputSummary();
 
-    SlangResult init(TestOutputMode outputMode, bool isSubReporter = false);
+    SlangResult init(TestOutputMode outputMode, const Slang::HashSet<Slang::String>& expectedFailureList, bool isSubReporter = false);
 
         /// Ctor
     TestReporter();
@@ -121,6 +123,7 @@ class TestReporter : public ITestReporter
     int m_passedTestCount;
     int m_failedTestCount;
     int m_ignoredTestCount;
+    int m_expectedFailedTestCount;
 
     int m_maxFailTestResults;                   ///< Maximum amount of results per test. If 0 it's infinite.
 
@@ -129,10 +132,10 @@ class TestReporter : public ITestReporter
     bool m_isVerbose = false;
     bool m_hideIgnored = false;
     bool m_isSubReporter = false;
-
+    Slang::HashSet<Slang::String> m_expectedFailureList;
 protected:
     
-    void _addResult(const TestInfo& info);
+    void _addResult(TestInfo info);
     
     Slang::StringBuilder m_currentMessage;
     TestInfo m_currentInfo;
diff --git a/tools/unit-test/slang-unit-test.h b/tools/unit-test/slang-unit-test.h
index 3c8bcaea6..8f96f6805 100644
--- a/tools/unit-test/slang-unit-test.h
+++ b/tools/unit-test/slang-unit-test.h
@@ -9,6 +9,7 @@ enum class TestResult
     // and a fail, is still a fail overall. 
     Ignored,
     Pass,
+    ExpectedFail,
     Fail,
 };