Run vk tests on spirv backend with expected failure list. (#3128)

7 files changed, 87 insertions, 19 deletions

diff --git a/tools/slang-test/options.cpp b/tools/slang-test/options.cpp
index 8ab9e9ae2..3fb71dd4a 100644
--- a/tools/slang-test/options.cpp
+++ b/tools/slang-test/options.cpp
@@ -3,7 +3,6 @@
 
 #include "../../source/core/slang-string-util.h"
 #include "../../source/core/slang-io.h"
-
 #include <stdio.h>
 #include <stdlib.h>
 
@@ -291,6 +290,27 @@ static bool _isSubCommand(const char* arg)
         {
             optionsOut->skipApiDetection = true;
         }
+        else if (strcmp(arg, "-emit-spirv-directly") == 0)
+        {
+            optionsOut->emitSPIRVDirectly = true;
+        }
+        else if (strcmp(arg, "-expected-failure-list") == 0)
+        {
+            if (argCursor == argEnd)
+            {
+                stdError.print("error: expected operand for '%s'\n", arg);
+                return SLANG_FAIL;
+            }
+            auto fileName = *argCursor++;
+            String text;
+            File::readAllText(fileName, text);
+            List<UnownedStringSlice> lines;
+            StringUtil::split(text.getUnownedSlice(), '\n', lines);
+            for (auto line : lines)
+            {
+                optionsOut->expectedFailureList.add(line);
+            }
+        }
         else
         {
             stdError.print("unknown option '%s'\n", arg);
diff --git a/tools/slang-test/options.h b/tools/slang-test/options.h
index e786a0a74..398334e46 100644
--- a/tools/slang-test/options.h
+++ b/tools/slang-test/options.h
@@ -113,7 +113,11 @@ struct Options
     Slang::String adapter;
 
     // Maximum number of test servers to run.
-    int serverCount = 4;
+    int serverCount = 1;
+
+    bool emitSPIRVDirectly = false;
+
+    Slang::HashSet<Slang::String> expectedFailureList;
 
         /// Parse the args, report any errors into stdError, and write the results into optionsOut
     static SlangResult parse(int argc, char** argv, TestCategorySet* categorySet, Slang::WriterHelper stdError, Options* optionsOut);
diff --git a/tools/slang-test/slang-test-main.cpp b/tools/slang-test/slang-test-main.cpp
index 95ef4d55d..cb2db064d 100644
--- a/tools/slang-test/slang-test-main.cpp
+++ b/tools/slang-test/slang-test-main.cpp
@@ -2992,6 +2992,10 @@ static void _addRenderTestOptions(const Options& options, CommandLine& ioCmdLine
         ioCmdLine.addArg("-adapter");
         ioCmdLine.addArg(options.adapter);
     }
+    if (options.emitSPIRVDirectly)
+    {
+        ioCmdLine.addArg("-emit-spirv-directly");
+    }
 }
 
 static SlangResult _extractProfileTime(const UnownedStringSlice& text, double& timeOut)
@@ -4117,7 +4121,7 @@ void runTestsInDirectory(
         auto threadFunc = [&](int threadId)
         {
             TestReporter reporter;
-            reporter.init(context->options.outputMode, true);
+            reporter.init(context->options.outputMode, context->options.expectedFailureList, true);
             TestReporter::SuiteScope suiteScope(&reporter, "tests");
             context->setThreadIndex(threadId);
             context->setTestReporter(&reporter);
@@ -4275,10 +4279,6 @@ static SlangResult runUnitTestModule(TestContext* context, TestOptions& testOpti
 
 SlangResult innerMain(int argc, char** argv)
 {
-    // Disable buffering for out and std out
-    StreamUtil::setStreamBufferStyle(StdStreamType::Out, StreamBufferStyle::None);
-    StreamUtil::setStreamBufferStyle(StdStreamType::ErrorOut, StreamBufferStyle::None);
-
     auto stdWriters = StdWriters::initDefaultSingleton();
 
     // The context holds useful things used during testing
@@ -4457,7 +4457,7 @@ SlangResult innerMain(int argc, char** argv)
     {
         // Setup the reporter
         TestReporter reporter;
-        SLANG_RETURN_ON_FAIL(reporter.init(options.outputMode));
+        SLANG_RETURN_ON_FAIL(reporter.init(options.outputMode, options.expectedFailureList));
 
         context.setTestReporter(&reporter);
 
diff --git a/tools/slang-test/test-context.cpp b/tools/slang-test/test-context.cpp
index e0f2749b9..39f5c92ed 100644
--- a/tools/slang-test/test-context.cpp
+++ b/tools/slang-test/test-context.cpp
@@ -186,11 +186,12 @@ SlangResult TestContext::_createJSONRPCConnection(RefPtr<JSONRPCConnection>& out
     {
         CommandLine cmdLine;
         cmdLine.setExecutableLocation(ExecutableLocation(exeDirectoryPath, "test-server"));
-        SLANG_RETURN_ON_FAIL(Process::create(cmdLine, Process::Flag::AttachDebugger, process));
+        SLANG_RETURN_ON_FAIL(Process::create(cmdLine, Process::Flag::AttachDebugger | Process::Flag::DisableStdErrRedirection, process));
     }
 
     Stream* writeStream = process->getStream(StdStreamType::In);
     RefPtr<BufferedReadStream> readStream(new BufferedReadStream(process->getStream(StdStreamType::Out)));
+    RefPtr<BufferedReadStream> readErrStream(new BufferedReadStream(process->getStream(StdStreamType::ErrorOut)));
 
     RefPtr<HTTPPacketConnection> connection = new HTTPPacketConnection(readStream, writeStream);
     RefPtr<JSONRPCConnection> rpcConnection = new JSONRPCConnection;
diff --git a/tools/slang-test/test-reporter.cpp b/tools/slang-test/test-reporter.cpp
index ede5823c7..3ac8d90ea 100644
--- a/tools/slang-test/test-reporter.cpp
+++ b/tools/slang-test/test-reporter.cpp
@@ -76,7 +76,7 @@ TestReporter::TestReporter() :
     m_passedTestCount = 0;
     m_failedTestCount = 0;
     m_ignoredTestCount = 0;
-
+    m_expectedFailedTestCount = 0;
     m_maxFailTestResults = 10;
 
     m_inTest = false;
@@ -84,10 +84,11 @@ TestReporter::TestReporter() :
     m_isVerbose = false;
 }
 
-Result TestReporter::init(TestOutputMode outputMode, bool isSubReporter)
+Result TestReporter::init(TestOutputMode outputMode, const HashSet<String>& expectedFailureList, bool isSubReporter)
 {
     m_outputMode = outputMode;
     m_isSubReporter = isSubReporter;
+    m_expectedFailureList = expectedFailureList;
     return SLANG_OK;
 }
 
@@ -141,7 +142,8 @@ void TestReporter::addResult(TestResult result)
     assert(m_inTest);
 
     std::lock_guard<std::recursive_mutex> lock(m_mutex);
-
+    if (result == TestResult::Fail && m_expectedFailureList.contains(m_currentInfo.name))
+        result = TestResult::ExpectedFail;
     m_currentInfo.testResult = combine(m_currentInfo.testResult, result);
     m_numCurrentResults++;
 }
@@ -158,6 +160,7 @@ void TestReporter::addResultWithLocation(TestResult result, const char* testText
     assert(m_inTest);
 
     std::lock_guard<std::recursive_mutex> lock(m_mutex);
+    result = adjustResult(m_currentInfo.name.getUnownedSlice(), result);
 
     m_numCurrentResults++;
 
@@ -207,6 +210,7 @@ void TestReporter::consolidateWith(TestReporter* other)
     m_failedTestCount += other->m_failedTestCount;
     m_ignoredTestCount += other->m_ignoredTestCount;
     m_passedTestCount += other->m_passedTestCount;
+    m_expectedFailedTestCount += other->m_expectedFailedTestCount;
     m_totalTestCount += other->m_totalTestCount;
 }
 
@@ -295,12 +299,13 @@ static void _appendTime(double timeInSec, StringBuilder& out)
     out << timeInSec << "ns";
 }
 
-void TestReporter::_addResult(const TestInfo& info)
+void TestReporter::_addResult(TestInfo info)
 {
     if (info.testResult == TestResult::Ignored && m_hideIgnored)
     {
         return;
     }
+    info.testResult = adjustResult(info.name.getUnownedSlice(), info.testResult);
 
     m_totalTestCount++;
 
@@ -313,6 +318,9 @@ void TestReporter::_addResult(const TestInfo& info)
         case TestResult::Pass:
             m_passedTestCount++;
             break;
+        case TestResult::ExpectedFail:
+            m_expectedFailedTestCount++;
+            break;
 
         case TestResult::Ignored:
             m_ignoredTestCount++;
@@ -333,6 +341,9 @@ void TestReporter::_addResult(const TestInfo& info)
         case TestResult::Fail:
             resultString = "FAILED";
             break;
+        case TestResult::ExpectedFail:
+            resultString = "failed(expected)";
+            break;
         case TestResult::Pass:
             resultString = "passed";
             break;
@@ -383,7 +394,8 @@ void TestReporter::_addResult(const TestInfo& info)
                     }
                     break;
                 }
-                case TestResult::Pass:     
+                case TestResult::Pass:
+                case TestResult::ExpectedFail:
                 {
                     StringBuilder message;
                     message << info.message;
@@ -443,6 +455,8 @@ void TestReporter::_addResult(const TestInfo& info)
                 case TestResult::Fail:      resultString = "Failed";  break;
                 case TestResult::Pass:      resultString = "Passed";  break;
                 case TestResult::Ignored:   resultString = "Ignored"; break;
+                case TestResult::ExpectedFail:   resultString = "ExpectedFail"; break;
+
                 default:
                     assert(!"unexpected");
                     break;
@@ -560,7 +574,7 @@ void TestReporter::message(TestMessageType type, const char* messageContent)
 
 bool TestReporter::didAllSucceed() const
 {
-    return m_passedTestCount == (m_totalTestCount - m_ignoredTestCount);
+    return m_failedTestCount == 0;
 }
 
 void TestReporter::outputSummary()
@@ -592,8 +606,25 @@ void TestReporter::outputSummary()
             {
                 printf(", %d tests ignored", ignoredCount);
             }
+            if (m_expectedFailedTestCount)
+            {
+                printf(", %d tests failed expectedly", m_expectedFailedTestCount);
+                printf("\n===\n\n");
+                printf("\npassing tests that are expected to fail:\n");
+                printf("---\n");
+                for (const auto& testInfo : m_testInfos)
+                {
+                    if (testInfo.testResult == TestResult::Pass)
+                    {
+                        if (m_expectedFailureList.contains(testInfo.name))
+                        {
+                            printf("%s\n", testInfo.name.getBuffer());
+                        }
+                    }
+                }
+                printf("---\n");
+            }
             printf("\n===\n\n");
-
             if (m_failedTestCount)
             {
                 printf("failing tests:\n");
@@ -607,6 +638,7 @@ void TestReporter::outputSummary()
                 }
                 printf("---\n");
             }
+            
             break;
         }
         
@@ -714,3 +746,10 @@ void TestReporter::endSuite()
     
     m_suiteStack.removeLast();
 }
+
+TestResult TestReporter::adjustResult(UnownedStringSlice testName, TestResult result)
+{
+    if (result == TestResult::Fail && m_expectedFailureList.contains(testName))
+        result = TestResult::ExpectedFail;
+    return result;
+}
diff --git a/tools/slang-test/test-reporter.h b/tools/slang-test/test-reporter.h
index b92865f8a..1d2857463 100644
--- a/tools/slang-test/test-reporter.h
+++ b/tools/slang-test/test-reporter.h
@@ -70,6 +70,8 @@ class TestReporter : public ITestReporter
     void startSuite(const Slang::String& name);
     void endSuite();
 
+    TestResult adjustResult(Slang::UnownedStringSlice testName, TestResult result);
+
     virtual SLANG_NO_THROW void SLANG_MCALL startTest(const char* testName) override;
     virtual SLANG_NO_THROW void SLANG_MCALL addResult(TestResult result) override;
     virtual SLANG_NO_THROW void SLANG_MCALL addResultWithLocation(TestResult result, const char* testText, const char* file, int line) override;
@@ -101,7 +103,7 @@ class TestReporter : public ITestReporter
 
     void outputSummary();
 
-    SlangResult init(TestOutputMode outputMode, bool isSubReporter = false);
+    SlangResult init(TestOutputMode outputMode, const Slang::HashSet<Slang::String>& expectedFailureList, bool isSubReporter = false);
 
         /// Ctor
     TestReporter();
@@ -121,6 +123,7 @@ class TestReporter : public ITestReporter
     int m_passedTestCount;
     int m_failedTestCount;
     int m_ignoredTestCount;
+    int m_expectedFailedTestCount;
 
     int m_maxFailTestResults;                   ///< Maximum amount of results per test. If 0 it's infinite.
 
@@ -129,10 +132,10 @@ class TestReporter : public ITestReporter
     bool m_isVerbose = false;
     bool m_hideIgnored = false;
     bool m_isSubReporter = false;
-
+    Slang::HashSet<Slang::String> m_expectedFailureList;
 protected:
     
-    void _addResult(const TestInfo& info);
+    void _addResult(TestInfo info);
     
     Slang::StringBuilder m_currentMessage;
     TestInfo m_currentInfo;
diff --git a/tools/unit-test/slang-unit-test.h b/tools/unit-test/slang-unit-test.h
index 3c8bcaea6..8f96f6805 100644
--- a/tools/unit-test/slang-unit-test.h
+++ b/tools/unit-test/slang-unit-test.h
@@ -9,6 +9,7 @@ enum class TestResult
     // and a fail, is still a fail overall. 
     Ignored,
     Pass,
+    ExpectedFail,
     Fail,
 };