Runs all gfx unit tests through a 'test proxy' (#1981)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Support for test proxy.

* Turn on testing using proxy.

* Don't pass sink into check of downstream compiler.

* Small change to kick off build.

* Remove register specification on transcendental.

* Increase poll timeout.
Small improvements to proxy.

* Disable gfx unit tests.

* Put test runner in shared library mode by default.

* Change comment. Kick off another CI test.

* Small edit to kick off builds.

* Run unit tests on proxy.

* Turn on using proxy for now.

* Enable swift shader.

* Fix typo.
Add exception support.

* Make the default spwan type SharedLibrary
Use isolation for gfx unit tests.

* Update slang-binaries.

* Fix typo.

* Report unit test output information.

4 files changed, 372 insertions, 19 deletions

diff --git a/tools/slang-test/options.cpp b/tools/slang-test/options.cpp
index fa7f332b0..d24ea83e2 100644
--- a/tools/slang-test/options.cpp
+++ b/tools/slang-test/options.cpp
@@ -125,9 +125,9 @@ static bool _isSubCommand(const char* arg)
             }
             optionsOut->binDir = *argCursor++;
         }
-        else if (strcmp(arg, "-useexes") == 0)
+        else if (strcmp(arg, "-use-test-proxy") == 0)
         {
-            optionsOut->useExes = true;
+            optionsOut->defaultSpawnType = SpawnType::UseProxy;
         }
         else if (strcmp(arg, "-v") == 0)
         {
diff --git a/tools/slang-test/options.h b/tools/slang-test/options.h
index eacbc7acb..4a40cba9a 100644
--- a/tools/slang-test/options.h
+++ b/tools/slang-test/options.h
@@ -36,6 +36,13 @@ protected:
     Slang::Dictionary<Slang::String, Slang::RefPtr<TestCategory> > m_categoryMap;
 };
 
+enum class SpawnType
+{
+    UseExe,
+    UseSharedLibrary,
+    UseProxy,
+};
+
 struct Options
 {
     char const* appName = "slang-test";
@@ -69,9 +76,13 @@ struct Options
     // integration builds.
     bool dumpOutputOnFailure = false;
 
-    // If set, will force using of executables (not shared library) for tests
-    bool useExes = false;
-
+    // Set the default spawn type to use
+    // Set to SpawnType::UseProxy, if isolation of test execution is desired.
+    // Having tests isolated, slows down testing considerably, so using UseSharedLibrary is the most
+    // desirable default usually.
+    SpawnType defaultSpawnType = SpawnType::UseSharedLibrary;
+    //SpawnType defaultSpawnType = SpawnType::UseProxy;
+    
     // kind of output to generate
     TestOutputMode outputMode = TestOutputMode::Default;
 
diff --git a/tools/slang-test/slang-test-main.cpp b/tools/slang-test/slang-test-main.cpp
index 8f7c7e35b..49ee7d688 100644
--- a/tools/slang-test/slang-test-main.cpp
+++ b/tools/slang-test/slang-test-main.cpp
@@ -97,11 +97,6 @@ struct FileTestList
     List<TestDetails> tests;
 };
 
-enum class SpawnType
-{
-    UseExe,
-    UseSharedLibrary,
-};
 
 struct TestInput
 {
@@ -531,6 +526,8 @@ static SlangResult _gatherTestsForFile(
     return SLANG_OK;
 }
 
+
+
 Result spawnAndWaitExe(TestContext* context, const String& testPath, const CommandLine& cmdLine, ExecuteResult& outRes)
 {
     const auto& options = context->options;
@@ -550,6 +547,7 @@ Result spawnAndWaitExe(TestContext* context, const String& testPath, const Comma
     return res;
 }
 
+
 Result spawnAndWaitSharedLibrary(TestContext* context, const String& testPath, const CommandLine& cmdLine, ExecuteResult& outRes)
 {
     const auto& options = context->options;
@@ -620,6 +618,44 @@ Result spawnAndWaitSharedLibrary(TestContext* context, const String& testPath, c
 }
 
 
+Result spawnAndWaitProxy(TestContext* context, const String& testPath, const CommandLine& inCmdLine, ExecuteResult& outRes)
+{
+    // Get the name of the thing to execute
+    String exeName = Path::getFileNameWithoutExt(inCmdLine.m_executable);
+
+    if (exeName == "slangc")
+    {
+        // If the test is slangc there is a command line version we can just directly use
+        //return spawnAndWaitExe(context, testPath, inCmdLine, outRes);
+        return spawnAndWaitSharedLibrary(context, testPath, inCmdLine, outRes);
+    }
+
+    CommandLine cmdLine(inCmdLine);
+
+    // Make the first arg the name of the tool to invoke
+    cmdLine.m_args.insert(0, exeName);
+
+    auto exePath = Path::combine(Path::getParentDirectory(inCmdLine.m_executable), String("test-proxy") + ProcessUtil::getExecutableSuffix());
+    cmdLine.setExecutablePath(exePath);
+
+    const auto& options = context->options;
+    if (options.shouldBeVerbose)
+    {
+        String commandLine = ProcessUtil::getCommandLineString(cmdLine);
+        context->reporter->messageFormat(TestMessageType::Info, "%s\n", commandLine.begin());
+    }
+
+    // Execute
+    Result res = ProcessUtil::execute(cmdLine, outRes);
+    if (SLANG_FAILED(res))
+    {
+        //        fprintf(stderr, "failed to run test '%S'\n", testPath.ToWString());
+        context->reporter->messageFormat(TestMessageType::RunError, "failed to run test '%S'", testPath.toWString().begin());
+    }
+
+    return res;
+}
+
 static SlangResult _extractArg(const CommandLine& cmdLine, const String& argName, String& outValue)
 {
     SLANG_ASSERT(argName.getLength() > 0 && argName[0] == '-');
@@ -972,6 +1008,11 @@ ToolReturnCode spawnAndWait(TestContext* context, const String& testPath, SpawnT
             spawnResult = spawnAndWaitSharedLibrary(context, testPath, cmdLine, outExeRes);
             break;
         }
+        case SpawnType::UseProxy:
+        {
+            spawnResult = spawnAndWaitProxy(context, testPath, cmdLine, outExeRes);
+            break;
+        }
         default: break;
     }
 
@@ -2880,8 +2921,6 @@ TestResult runTest(
         return TestResult::Pass;
     }
 
-    const SpawnType defaultSpawnType = context->options.useExes ? SpawnType::UseExe : SpawnType::UseSharedLibrary;
-
     auto testInfo = _findTestCommandInfoByCommand(testOptions.command.getUnownedSlice());
 
     if (testInfo)
@@ -2890,7 +2929,7 @@ TestResult runTest(
         testInput.filePath = filePath;
         testInput.outputStem = outputStem;
         testInput.testOptions = &testOptions;
-        testInput.spawnType = defaultSpawnType;
+        testInput.spawnType = context->options.defaultSpawnType;
 
         return testInfo->callback(context, testInput);
     }
@@ -3346,8 +3385,18 @@ static void _disableCPPBackends(TestContext* context)
     }
 }
 
+static TestResult _asTestResult(ToolReturnCode retCode)
+{
+    switch (retCode)
+    {
+        default:                        return TestResult::Fail;
+        case ToolReturnCode::Success:   return TestResult::Pass;
+        case ToolReturnCode::Ignored:   return TestResult::Ignored;
+    }
+}
+
     /// Loads a DLL containing unit test functions and run them one by one.
-static SlangResult runUnitTestModule(TestContext* context, TestOptions& testOptions, const char* moduleName)
+static SlangResult runUnitTestModule(TestContext* context, TestOptions& testOptions, SpawnType spawnType, const char* moduleName)
 {
     SharedLibrary::Handle moduleHandle;
     SLANG_RETURN_ON_FAIL(SharedLibrary::load(
@@ -3368,6 +3417,9 @@ static SlangResult runUnitTestModule(TestContext* context, TestOptions& testOpti
     unitTestContext.workDirectory = "";
     unitTestContext.enabledApis = context->options.enabledApis;
     auto testCount = testModule->getTestCount();
+
+    TestReporter* reporter = TestReporter::get();
+
     for (SlangInt i = 0; i < testCount; i++)
     {
         auto testFunc = testModule->getTestFunc(i);
@@ -3382,9 +3434,50 @@ static SlangResult runUnitTestModule(TestContext* context, TestOptions& testOpti
         {
             if (testPassesCategoryMask(context, testOptions))
             {
-                TestReporter::get()->startTest(testOptions.command.getBuffer());
-                testFunc(&unitTestContext);
-                TestReporter::get()->endTest();
+                if (spawnType == SpawnType::UseProxy)
+                {
+                    CommandLine cmdLine;
+
+                    // The 'command' is the module 
+                    cmdLine.setExecutablePath(Path::combine(context->exeDirectoryPath, moduleName));
+
+                    // Pass the test name / index
+                    cmdLine.addArg(testName);
+
+                    {
+                        StringBuilder buf;
+                        buf << i;
+                        cmdLine.addArg(buf.ProduceString());
+                    }
+
+                    // Pass the enabled apis
+                    {
+                        StringBuilder buf;
+                        buf << context->options.enabledApis;
+                        cmdLine.addArg(buf.ProduceString());
+                    }
+
+                    {
+                        TestReporter::TestScope scopeTest(reporter, testOptions.command);
+                        ExecuteResult exeRes;
+
+                        const auto testResult = _asTestResult(spawnAndWait(context, filePath, spawnType, cmdLine, exeRes));
+
+                        // If the test fails, output any output - which might give information about individual tests that have failed.
+                        if (testResult == TestResult::Fail)
+                        {
+                            String output = getOutput(exeRes);
+                            reporter->message(TestMessageType::TestFailure, output.getBuffer());
+                        }
+
+                        reporter->addResult(testResult);
+                    }
+                }
+                else
+                {
+                    TestReporter::TestScope scopeTest(reporter, testOptions.command);
+                    testFunc(&unitTestContext);
+                }
             }
         }
     }
@@ -3570,13 +3663,15 @@ SlangResult innerMain(int argc, char** argv)
                 TestOptions testOptions;
                 testOptions.categories.add(unitTestCategory);
                 testOptions.categories.add(smokeTestCategory);
-                runUnitTestModule(&context, testOptions, "slang-unit-test-tool");
+                runUnitTestModule(&context, testOptions, context.options.defaultSpawnType, "slang-unit-test-tool");
             }
+            
             {
                 TestOptions testOptions;
                 testOptions.categories.add(unitTestCategory);
-                runUnitTestModule(&context, testOptions, "gfx-unit-test-tool");
+                runUnitTestModule(&context, testOptions, SpawnType::UseProxy, "gfx-unit-test-tool");
             }
+
             TestReporter::set(nullptr);
         }
 
diff --git a/tools/test-proxy/test-proxy-main.cpp b/tools/test-proxy/test-proxy-main.cpp
new file mode 100644
index 000000000..2b6c59047
--- /dev/null
+++ b/tools/test-proxy/test-proxy-main.cpp
@@ -0,0 +1,247 @@
+// main.cpp
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "../../source/core/slang-secure-crt.h"
+
+#include "../../slang-com-helper.h"
+
+#include "../../source/core/slang-string.h"
+#include "../../source/core/slang-io.h"
+#include "../../source/core/slang-writer.h"
+#include "../../source/core/slang-string-util.h"
+
+#include "../../source/core/slang-shared-library.h"
+
+#include "../../source/core/slang-test-tool-util.h"
+
+#include "tools/unit-test/slang-unit-test.h"
+
+namespace TestProxy
+{
+using namespace Slang;
+
+class TestReporter : public ITestReporter
+{
+public:
+    // ITestReporter
+    virtual SLANG_NO_THROW void SLANG_MCALL startTest(const char* testName) SLANG_OVERRIDE { }
+    virtual SLANG_NO_THROW void SLANG_MCALL addResult(TestResult result)SLANG_OVERRIDE;
+    virtual SLANG_NO_THROW void SLANG_MCALL addResultWithLocation(TestResult result, const char* testText, const char* file, int line) SLANG_OVERRIDE;
+    virtual SLANG_NO_THROW void SLANG_MCALL addResultWithLocation(bool testSucceeded, const char* testText, const char* file, int line) SLANG_OVERRIDE;
+    virtual SLANG_NO_THROW void SLANG_MCALL addExecutionTime(double time) SLANG_OVERRIDE { }
+    virtual SLANG_NO_THROW void SLANG_MCALL message(TestMessageType type, const char* message) SLANG_OVERRIDE;
+    virtual SLANG_NO_THROW void SLANG_MCALL endTest() SLANG_OVERRIDE { }
+
+    StringBuilder m_buf;
+    Index m_failCount = 0;
+    Index m_testCount = 0;
+};
+
+
+void TestReporter::message(TestMessageType type, const char* message)
+{
+    if (type == TestMessageType::RunError ||
+        type == TestMessageType::TestFailure)
+    {
+        m_failCount++;
+    }
+
+    m_buf << message << "\n";
+}
+
+void TestReporter::addResultWithLocation(TestResult result, const char* testText, const char* file, int line)
+{
+    if (result == TestResult::Fail)
+    {
+        addResultWithLocation(false, testText, file, line);
+    }
+    else
+    {
+        m_testCount++;
+    }
+}
+
+void TestReporter::addResultWithLocation(bool testSucceeded, const char* testText, const char* file, int line)
+{
+    m_testCount++;
+
+    if (testSucceeded)
+    {
+        return;
+    }
+
+    m_buf << "[Failed]: " << testText << "\n";
+    m_buf << file << ":" << line << "\n";
+
+    m_failCount++;
+}
+
+void TestReporter::addResult(TestResult result)
+{
+    if (result == TestResult::Fail)
+    {
+        m_failCount++;
+    }
+}
+
+static SlangResult _createSlangSession(const char* exePath, int argc, const char* const* argv, ComPtr<slang::IGlobalSession>& outSession)
+{
+    
+    ComPtr<slang::IGlobalSession> session;
+
+    // The sharedSession always has a pre-loaded stdlib, is sharedSession is not nullptr.
+    // This differed test checks if the command line has an option to setup the stdlib.
+    // If so we *don't* use the sharedSession, and create a new stdlib-less session just for this compilation. 
+    if (TestToolUtil::hasDeferredStdLib(argc, argv))
+    {
+        SLANG_RETURN_ON_FAIL(slang_createGlobalSessionWithoutStdLib(SLANG_API_VERSION, session.writeRef()));
+        TestToolUtil::setSessionDefaultPreludeFromExePath(exePath, session);
+    }
+    else
+    {
+        // Just create the global session in the regular way if there isn't one set
+        SLANG_RETURN_ON_FAIL(slang_createGlobalSession(SLANG_API_VERSION, session.writeRef()));
+        TestToolUtil::setSessionDefaultPreludeFromExePath(exePath, session);
+    }
+
+    outSession.swap(session);
+    return SLANG_OK;
+}
+
+static SlangResult execute(int argc, const char* const* argv)
+{
+    typedef Slang::TestToolUtil::InnerMainFunc InnerMainFunc;
+
+    if (argc < 2)
+    {
+        return SLANG_FAIL;
+    }
+
+    String exePath = Path::getParentDirectory(argv[0]);
+
+    // The 'exeName' of the tool. 
+    const String toolName = argv[1];
+
+    {
+        StringBuilder sharedLibToolBuilder;
+        sharedLibToolBuilder.append(toolName);
+        sharedLibToolBuilder.append("-tool");
+
+        auto toolPath = Path::combine(exePath, sharedLibToolBuilder);
+
+        ComPtr<ISlangSharedLibrary> sharedLibrary;
+        if (SLANG_SUCCEEDED(DefaultSharedLibraryLoader::getSingleton()->loadSharedLibrary(toolPath.getBuffer(), sharedLibrary.writeRef())))
+        {
+            // Assume we will used the shared session
+            ComPtr<slang::IGlobalSession> session;
+            SLANG_RETURN_ON_FAIL(_createSlangSession(argv[0], argc - 2, argv + 2, session));
+
+            auto func = (InnerMainFunc)sharedLibrary->findFuncByName("innerMain");
+            if (!func)
+            {
+                return SLANG_FAIL;
+            }
+
+            // Work out the args sent to the shared library
+            List<const char*> args;
+            args.add(argv[0]);
+            args.addRange(argv + 2, Index(argc - 2));
+
+            RefPtr<StdWriters> stdWriters = StdWriters::createDefault();
+
+            const SlangResult res = func(stdWriters, session, int(args.getCount()), args.begin());
+            return res;
+        }
+    }
+
+    // Okay lets assume it's a unit test
+    {
+        auto toolPath = Path::combine(exePath, toolName);
+
+        ComPtr<ISlangSharedLibrary> sharedLibrary;
+        SLANG_RETURN_ON_FAIL(SLANG_SUCCEEDED(DefaultSharedLibraryLoader::getSingleton()->loadSharedLibrary(toolPath.getBuffer(), sharedLibrary.writeRef())));
+
+        // get the unit test export name
+
+        UnitTestGetModuleFunc getModuleFunc = (UnitTestGetModuleFunc)sharedLibrary->findFuncByName("slangUnitTestGetModule");
+        if (!getModuleFunc)
+            return SLANG_FAIL;
+
+        IUnitTestModule* testModule = getModuleFunc();
+        if (!testModule)
+            return SLANG_FAIL;
+
+        // argv[0] exe,
+        // argv[1] tool
+        // argv[2] test name
+        // argv[3] test index
+        // argv[4] the enabled apis
+
+        if (argc < 5)
+        {
+            return SLANG_FAIL;
+        }
+
+        TestReporter testReporter;
+
+        testModule->setTestReporter(&testReporter);
+
+        Int enabledApis;
+        SLANG_RETURN_ON_FAIL(StringUtil::parseInt(UnownedStringSlice(argv[4]), enabledApis));
+
+        Int testIndex;
+        SLANG_RETURN_ON_FAIL(StringUtil::parseInt(UnownedStringSlice(argv[3]), testIndex));
+
+        // Assume we will used the shared session
+        ComPtr<slang::IGlobalSession> session;
+        SLANG_RETURN_ON_FAIL(_createSlangSession(argv[0], argc - 5, argv + 5, session));
+
+        UnitTestContext unitTestContext;
+        unitTestContext.slangGlobalSession = session;
+        unitTestContext.workDirectory = "";
+        unitTestContext.enabledApis = RenderApiFlags(enabledApis);
+
+        auto testCount = testModule->getTestCount();
+        SLANG_ASSERT(testIndex >= 0 && testIndex < testCount);
+
+        UnitTestFunc testFunc = testModule->getTestFunc(testIndex);
+
+        try
+        {
+            testFunc(&unitTestContext);
+        }
+        catch (...)
+        {
+            testReporter.m_failCount++;
+        }
+
+        if (testReporter.m_failCount > 0)
+        {
+            // Write out to stderr...
+            auto writers = StdWriters::createDefault();
+            writers->getError().put(testReporter.m_buf.getUnownedSlice());
+            return SLANG_FAIL;
+        }
+
+        if (testReporter.m_testCount == 0)
+        {
+            return SLANG_E_NOT_AVAILABLE;
+        }
+
+        return SLANG_OK;
+    }
+
+    return SLANG_E_NOT_AVAILABLE;
+}
+
+} // namespace TestProxy
+
+int main(int argc, const char* const* argv)
+{
+    using namespace TestProxy;
+    SlangResult res = execute(argc, argv);
+    return (int)TestToolUtil::getReturnCode(res);
+}