Enable running tests in parallel. (#2078)

* Enable running tests in parallel.

* Fix linux build.

* Add pthread dependency for slang-test.

* Fix teamcity output.

* Fix race condition.

* Make testReporter thread safe.

* Clean up.

* Fix.

* trigger build

* Fix.

Co-authored-by: Yong He <yhe@nvidia.com>
Co-authored-by: Theresa Foley <tfoleyNV@users.noreply.github.com>

11 files changed, 263 insertions, 87 deletions

diff --git a/premake5.lua b/premake5.lua
index a7f213b95..69f18a5b4 100644
--- a/premake5.lua
+++ b/premake5.lua
@@ -793,12 +793,13 @@ end
      uuid "0C768A18-1D25-4000-9F37-DA5FE99E3B64"
      includedirs { "." }
      links { "compiler-core", "slang", "core", "miniz", "lz4" }
- 
      -- We want to set to the root of the project, but that doesn't seem to work with '.'.
      -- So set a path that resolves to the same place.
- 
      debugdir("source/..")
- 
+    if not targetInfo.isWindows then
+        links { "pthread" }
+    end
+     
  --
  -- The reflection test harness `slang-reflection-test` is pretty
  -- simple, in that it only needs to link against the slang library
diff --git a/source/compiler-core/slang-visual-studio-compiler-util.cpp b/source/compiler-core/slang-visual-studio-compiler-util.cpp
index e2c488ee9..9bdfd406a 100644
--- a/source/compiler-core/slang-visual-studio-compiler-util.cpp
+++ b/source/compiler-core/slang-visual-studio-compiler-util.cpp
@@ -87,8 +87,7 @@ namespace Slang
     // https://docs.microsoft.com/en-us/cpp/build/reference/compiler-options-listed-alphabetically?view=vs-2019
 
     cmdLine.addArg("/nologo");
-    // Generate complete debugging information
-    cmdLine.addArg("/Zi");
+
     // Display full path of source files in diagnostics
     cmdLine.addArg("/FC");
 
@@ -139,6 +138,8 @@ namespace Slang
     // /Fd - followed by name of the pdb file
     if (options.debugInfoType != DebugInfoType::None)
     {
+        // Generate complete debugging information
+        cmdLine.addArg("/Zi");
         cmdLine.addPrefixPathArg("/Fd", options.modulePath, ".pdb");
     }
 
diff --git a/source/core/slang-writer.cpp b/source/core/slang-writer.cpp
index 3b1a5756a..849cb2da0 100644
--- a/source/core/slang-writer.cpp
+++ b/source/core/slang-writer.cpp
@@ -71,6 +71,7 @@ ISlangUnknown* BaseWriter::getInterface(const Guid& guid)
 
 SLANG_NO_THROW char* SLANG_MCALL AppendBufferWriter::beginAppendBuffer(size_t maxNumChars)
 {
+    mutex.lock();
     m_appendBuffer.setCount(maxNumChars);
     return m_appendBuffer.getBuffer();
 }
@@ -82,6 +83,7 @@ SLANG_NO_THROW SlangResult SLANG_MCALL AppendBufferWriter::endAppendBuffer(char*
     SlangResult res = write(buffer, numChars);
     // Clear so that buffer can't be written from again without assert
     m_appendBuffer.clear();
+    mutex.unlock();
     return res;
 }
 
diff --git a/source/core/slang-writer.h b/source/core/slang-writer.h
index 7c768a22f..ccdcb3747 100644
--- a/source/core/slang-writer.h
+++ b/source/core/slang-writer.h
@@ -8,6 +8,8 @@
 
 #include "slang-list.h"
 
+#include <mutex>
+
 namespace Slang
 {
 
@@ -81,6 +83,7 @@ public:
 
 protected:
     List<char> m_appendBuffer;
+    std::mutex mutex;
 };
 
 class CallbackWriter : public AppendBufferWriter
diff --git a/tools/slang-test/options.cpp b/tools/slang-test/options.cpp
index 48a43d621..38ed902a8 100644
--- a/tools/slang-test/options.cpp
+++ b/tools/slang-test/options.cpp
@@ -23,12 +23,11 @@ TestCategory* TestCategorySet::add(String const& name, TestCategory* parent)
 
 TestCategory* TestCategorySet::find(String const& name)
 {
-    RefPtr<TestCategory> category;
-    if (!m_categoryMap.TryGetValue(name, category))
+    if (auto category = m_categoryMap.TryGetValue(name))
     {
-        return nullptr;
+        return category->Ptr();
     }
-    return category;
+    return nullptr;
 }
 
 TestCategory* TestCategorySet::findOrError(String const& name)
@@ -191,6 +190,19 @@ static bool _isSubCommand(const char* arg)
             }
             optionsOut->adapter = *argCursor++;
         }
+        else if (strcmp(arg, "-server-count") == 0)
+        {
+            if (argCursor == argEnd)
+            {
+                stdError.print("error: expected operand for '%s'\n", arg);
+                return SLANG_FAIL;
+            }
+            optionsOut->serverCount = StringToInt(* argCursor++);
+            if (optionsOut->serverCount <= 0)
+            {
+                optionsOut->serverCount = 1;
+            }
+        }
         else if (strcmp(arg, "-appveyor") == 0)
         {
             optionsOut->outputMode = TestOutputMode::AppVeyor;
diff --git a/tools/slang-test/options.h b/tools/slang-test/options.h
index cf86bc901..26b8e7500 100644
--- a/tools/slang-test/options.h
+++ b/tools/slang-test/options.h
@@ -112,6 +112,9 @@ struct Options
     // The adapter to use. If empty will match first found adapter.
     Slang::String adapter;
 
+    // Maximum number of test servers to run.
+    int serverCount = 4;
+
         /// Parse the args, report any errors into stdError, and write the results into optionsOut
     static SlangResult parse(int argc, char** argv, TestCategorySet* categorySet, Slang::WriterHelper stdError, Options* optionsOut);
 };
diff --git a/tools/slang-test/slang-test-main.cpp b/tools/slang-test/slang-test-main.cpp
index 685e9dd7a..05be9e0de 100644
--- a/tools/slang-test/slang-test-main.cpp
+++ b/tools/slang-test/slang-test-main.cpp
@@ -43,6 +43,8 @@
 #define SLANG_PRELUDE_NAMESPACE CPPPrelude
 #include "../../prelude/slang-cpp-types.h"
 
+#include <atomic>
+#include <thread>
 
 using namespace Slang;
 
@@ -532,19 +534,23 @@ static SlangResult _gatherTestsForFile(
 
 Result spawnAndWaitExe(TestContext* context, const String& testPath, const CommandLine& cmdLine, ExecuteResult& outRes)
 {
+    std::lock_guard<std::mutex> lock(context->mutex);
+
     const auto& options = context->options;
 
     if (options.shouldBeVerbose)
     {
         String commandLine = cmdLine.toString();
-        context->reporter->messageFormat(TestMessageType::Info, "%s\n", commandLine.begin());
+        context->getTestReporter()->messageFormat(
+            TestMessageType::Info, "%s\n", commandLine.begin());
     }
 
     Result res = ProcessUtil::execute(cmdLine, outRes);
     if (SLANG_FAILED(res))
     {
         //        fprintf(stderr, "failed to run test '%S'\n", testPath.ToWString());
-        context->reporter->messageFormat(TestMessageType::RunError, "failed to run test '%S'", testPath.toWString().begin());
+        context->getTestReporter()->messageFormat(
+            TestMessageType::RunError, "failed to run test '%S'", testPath.toWString().begin());
     }
     return res;
 }
@@ -552,6 +558,8 @@ Result spawnAndWaitExe(TestContext* context, const String& testPath, const Comma
 
 Result spawnAndWaitSharedLibrary(TestContext* context, const String& testPath, const CommandLine& cmdLine, ExecuteResult& outRes)
 {
+    std::lock_guard<std::mutex> lock(context->mutex);
+
     const auto& options = context->options;
     String exeName = Path::getFileNameWithoutExt(cmdLine.m_executableLocation.m_pathOrName);
 
@@ -570,7 +578,8 @@ Result spawnAndWaitSharedLibrary(TestContext* context, const String& testPath, c
         testCmdLine.addArg(exeName);
         testCmdLine.m_args.addRange(cmdLine.m_args);
 
-        context->reporter->messageFormat(TestMessageType::Info, "%s\n", testCmdLine.toString().getBuffer());
+        context->getTestReporter()->messageFormat(
+            TestMessageType::Info, "%s\n", testCmdLine.toString().getBuffer());
     }
 
     auto func = context->getInnerMainFunc(context->options.binDir, exeName);
@@ -621,6 +630,8 @@ Result spawnAndWaitSharedLibrary(TestContext* context, const String& testPath, c
 
 Result spawnAndWaitProxy(TestContext* context, const String& testPath, const CommandLine& inCmdLine, ExecuteResult& outRes)
 {
+    std::lock_guard<std::mutex> lock(context->mutex);
+
     // Get the name of the thing to execute
     String exeName = Path::getFileNameWithoutExt(inCmdLine.m_executableLocation.m_pathOrName);
 
@@ -641,7 +652,8 @@ Result spawnAndWaitProxy(TestContext* context, const String& testPath, const Com
     if (options.shouldBeVerbose)
     {
         String commandLine = cmdLine.toString();
-        context->reporter->messageFormat(TestMessageType::Info, "%s\n", commandLine.begin());
+        context->getTestReporter()->messageFormat(
+            TestMessageType::Info, "%s\n", commandLine.begin());
     }
 
     // Execute
@@ -649,7 +661,8 @@ Result spawnAndWaitProxy(TestContext* context, const String& testPath, const Com
     if (SLANG_FAILED(res))
     {
         //        fprintf(stderr, "failed to run test '%S'\n", testPath.ToWString());
-        context->reporter->messageFormat(TestMessageType::RunError, "failed to run test '%S'", testPath.toWString().begin());
+        context->getTestReporter()->messageFormat(
+            TestMessageType::RunError, "failed to run test '%S'", testPath.toWString().begin());
     }
 
     return res;
@@ -966,6 +979,8 @@ static SlangResult _extractTestRequirements(const CommandLine& cmdLine, TestRequ
 
 static RenderApiFlags _getAvailableRenderApiFlags(TestContext* context)
 {
+    static std::mutex mutex;
+    std::lock_guard<std::mutex> lock(mutex);
     // Only evaluate if it hasn't already been evaluated (the actual evaluation is slow...)
     if (!context->isAvailableRenderApiFlagsValid)
     {
@@ -1046,8 +1061,9 @@ ToolReturnCode spawnAndWait(TestContext* context, const String& testPath, SpawnT
 {
     if (context->isCollectingRequirements())
     {
+        std::lock_guard<std::mutex> lock(context->mutex);
         // If we just want info... don't bother running anything
-        const SlangResult res = _extractTestRequirements(cmdLine, context->testRequirements);
+        const SlangResult res = _extractTestRequirements(cmdLine, context->getTestRequirements());
         // Keep compiler happy on release
         SLANG_UNUSED(res);
         SLANG_ASSERT(SLANG_SUCCEEDED(res));
@@ -1292,7 +1308,7 @@ TestResult runDocTest(TestContext* context, TestInput& input)
     // Otherwise we compare to the expected output
     if (!_areResultsEqual(input.testOptions->type, expectedOutput, actualOutput))
     {
-        context->reporter->dumpOutputDifference(expectedOutput, actualOutput);
+        context->getTestReporter()->dumpOutputDifference(expectedOutput, actualOutput);
         result = TestResult::Fail;
     }
 
@@ -1304,7 +1320,7 @@ TestResult runDocTest(TestContext* context, TestInput& input)
         String actualOutputPath = outputStem + ".actual";
         Slang::File::writeAllText(actualOutputPath, actualOutput);
 
-        context->reporter->dumpOutputDifference(expectedOutput, actualOutput);
+        context->getTestReporter()->dumpOutputDifference(expectedOutput, actualOutput);
     }
 
     return result;
@@ -1377,7 +1393,7 @@ TestResult runSimpleTest(TestContext* context, TestInput& input)
     // Otherwise we compare to the expected output
     if (!_areResultsEqual(input.testOptions->type, expectedOutput, actualOutput))
     {
-        context->reporter->dumpOutputDifference(expectedOutput, actualOutput);
+        context->getTestReporter()->dumpOutputDifference(expectedOutput, actualOutput);
         result = TestResult::Fail;
     }
 
@@ -1389,7 +1405,7 @@ TestResult runSimpleTest(TestContext* context, TestInput& input)
         String actualOutputPath = outputStem + ".actual";
         Slang::File::writeAllText(actualOutputPath, actualOutput);
 
-        context->reporter->dumpOutputDifference(expectedOutput, actualOutput);
+        context->getTestReporter()->dumpOutputDifference(expectedOutput, actualOutput);
     }
 
     return result;
@@ -1486,14 +1502,14 @@ TestResult runSimpleLineTest(TestContext* context, TestInput& input)
         }
         else
         {
-            context->reporter->dumpOutputDifference(expectedOutput, actualOutput);
+            context->getTestReporter()->dumpOutputDifference(expectedOutput, actualOutput);
         }
     }
     else
     {
         StringBuilder buf;
         buf << "Unable to find expected output for '" << outputStem << "'";
-        context->reporter->message(TestMessageType::TestFailure, buf);
+        context->getTestReporter()->message(TestMessageType::TestFailure, buf);
     }
 
     // If the test failed, then we write the actual output to a file
@@ -1541,7 +1557,7 @@ TestResult runCompile(TestContext* context, TestInput& input)
 
     if (exeRes.resultCode != 0)
     {
-        auto reporter = context->reporter;
+        auto reporter = context->getTestReporter();
         if (reporter)
         {
             auto output = getOutput(exeRes);
@@ -1657,7 +1673,7 @@ TestResult runReflectionTest(TestContext* context, TestInput& input)
         String actualOutputPath = outputStem + ".actual";
         Slang::File::writeAllText(actualOutputPath, actualOutput);
 
-        context->reporter->dumpOutputDifference(expectedOutput, actualOutput);
+        context->getTestReporter()->dumpOutputDifference(expectedOutput, actualOutput);
     }
 
     return result;
@@ -1750,13 +1766,14 @@ static TestResult runCPPCompilerSharedLibrary(TestContext* context, TestInput& i
     DownstreamCompiler* compiler = context->getDefaultCompiler(SLANG_SOURCE_LANGUAGE_CPP);
     if (!compiler)
     {
+        std::lock_guard<std::mutex> lock(context->mutex);
         return TestResult::Ignored;
     }
 
     // If we are just collecting requirements, say it passed
     if (context->isCollectingRequirements())
     {
-        context->testRequirements->addUsedBackEnd(SLANG_PASS_THROUGH_GENERIC_C_CPP);
+        context->getTestRequirements()->addUsedBackEnd(SLANG_PASS_THROUGH_GENERIC_C_CPP);
         return TestResult::Pass;
     }
 
@@ -1815,7 +1832,7 @@ static TestResult runCPPCompilerSharedLibrary(TestContext* context, TestInput& i
             // Compare if they are the same 
             if (!StringUtil::areLinesEqual(actualOutput.getUnownedSlice(), expectedOutput.getUnownedSlice()))
             {
-                context->reporter->dumpOutputDifference(expectedOutput, actualOutput);
+                context->getTestReporter()->dumpOutputDifference(expectedOutput, actualOutput);
                 return TestResult::Fail;
             }
         }
@@ -1869,7 +1886,8 @@ static TestResult runCPPCompilerExecute(TestContext* context, TestInput& input)
     // If we are just collecting requirements, say it passed
     if (context->isCollectingRequirements())
     {
-        context->testRequirements->addUsedBackEnd(SLANG_PASS_THROUGH_GENERIC_C_CPP);
+        std::lock_guard<std::mutex> lock(context->mutex);
+        context->getTestRequirements()->addUsedBackEnd(SLANG_PASS_THROUGH_GENERIC_C_CPP);
         return TestResult::Pass;
     }
 
@@ -1953,7 +1971,7 @@ static TestResult runCPPCompilerExecute(TestContext* context, TestInput& input)
         // Compare if they are the same 
         if (!StringUtil::areLinesEqual(actualOutput.getUnownedSlice(), expectedOutput.getUnownedSlice()))
         {
-            context->reporter->dumpOutputDifference(expectedOutput, actualOutput);
+            context->getTestReporter()->dumpOutputDifference(expectedOutput, actualOutput);
             return TestResult::Fail;
         }
     }
@@ -2076,7 +2094,7 @@ TestResult runCrossCompilerTest(TestContext* context, TestInput& input)
         String actualOutputPath = outputStem + ".actual";
         Slang::File::writeAllText(actualOutputPath, actualOutput);
 
-        context->reporter->dumpOutputDifference(expectedOutput, actualOutput);
+        context->getTestReporter()->dumpOutputDifference(expectedOutput, actualOutput);
     }
 
     return result;
@@ -2230,7 +2248,7 @@ static TestResult _runHLSLComparisonTest(
         String actualOutputPath = outputStem + ".actual";
         Slang::File::writeAllText(actualOutputPath, actualOutput);
 
-        context->reporter->dumpOutputDifference(expectedOutput, actualOutput);
+        context->getTestReporter()->dumpOutputDifference(expectedOutput, actualOutput);
     }
 
     return result;
@@ -2347,7 +2365,7 @@ TestResult runGLSLComparisonTest(TestContext* context, TestInput& input)
 
     if (!StringUtil::areLinesEqual(actualOutput.getUnownedSlice(), expectedOutput.getUnownedSlice()))
     {
-        context->reporter->dumpOutputDifference(expectedOutput, actualOutput);
+        context->getTestReporter()->dumpOutputDifference(expectedOutput, actualOutput);
 
         return TestResult::Fail;
     }
@@ -2418,7 +2436,7 @@ TestResult runPerformanceProfile(TestContext* context, TestInput& input)
         return TestResult::Fail;
     }
 
-    context->reporter->addExecutionTime(time);
+    context->getTestReporter()->addExecutionTime(time);
 
     return TestResult::Pass;
 }
@@ -2587,7 +2605,7 @@ TestResult runComputeComparisonImpl(TestContext* context, TestInput& input, cons
     
     if (!StringUtil::areLinesEqual(actualOutput.getUnownedSlice(), expectedOutput.getUnownedSlice()))
     {
-        context->reporter->dumpOutputDifference(expectedOutput, actualOutput);
+        context->getTestReporter()->dumpOutputDifference(expectedOutput, actualOutput);
 
         String actualOutputPath = outputStem + ".actual";
         Slang::File::writeAllText(actualOutputPath, actualOutput);
@@ -2614,7 +2632,11 @@ TestResult runComputeComparisonImpl(TestContext* context, TestInput& input, cons
 
     if (SLANG_FAILED(_compareWithType(actualOutputContent.getUnownedSlice(), referenceOutputContent.getUnownedSlice())))
     {
-        context->reporter->messageFormat(TestMessageType::TestFailure, "output mismatch! actual output: {\n%s\n}, \n%s\n", actualOutputContent.getBuffer(), referenceOutputContent.getBuffer());
+        context->getTestReporter()->messageFormat(
+            TestMessageType::TestFailure,
+            "output mismatch! actual output: {\n%s\n}, \n%s\n",
+            actualOutputContent.getBuffer(),
+            referenceOutputContent.getBuffer());
         return TestResult::Fail;
     }
 
@@ -2763,7 +2785,7 @@ bool STBImage::isComparable(const ThisType& rhs) const
 
 TestResult doImageComparison(TestContext* context, String const& filePath)
 {
-    auto reporter = context->reporter;
+    auto reporter = context->getTestReporter();
 
     // Allow a difference in the low bits of the 8-bit result, just to play it safe
     static const int kAbsoluteDiffCutoff = 2;
@@ -2893,7 +2915,7 @@ TestResult runHLSLRenderComparisonTestImpl(
 
     if (!StringUtil::areLinesEqual(actualOutput.getUnownedSlice(), expectedOutput.getUnownedSlice()))
     {
-        context->reporter->dumpOutputDifference(expectedOutput, actualOutput);
+        context->getTestReporter()->dumpOutputDifference(expectedOutput, actualOutput);
 
         return TestResult::Fail;
     }
@@ -3028,7 +3050,7 @@ bool testCategoryMatches(
 
 bool testCategoryMatches(
     TestCategory*                           categoryToMatch,
-    Dictionary<TestCategory*, TestCategory*> categorySet)
+    const Dictionary<TestCategory*, TestCategory*>& categorySet)
 {
     for( auto item : categorySet )
     {
@@ -3150,9 +3172,9 @@ static void _calcSynthesizedTests(TestContext* context, RenderApiType synthRende
         }
 
         // Work out the info about this tests
-        context->testRequirements = &synthTestDetails.requirements;
+        context->setTestRequirements(& synthTestDetails.requirements);
         runTest(context, "", "", "", synthOptions);
-        context->testRequirements = nullptr;
+        context->setTestRequirements(nullptr);
 
         // It does set the explicit render target
         SLANG_ASSERT(synthTestDetails.requirements.explicitRenderApi == synthRenderApiType);
@@ -3212,7 +3234,7 @@ static SlangResult _runTestsOnFile(
     // Note cases where a test file exists, but we found nothing to run
     if( testList.tests.getCount() == 0 )
     {
-        context->reporter->addTest(filePath, TestResult::Ignored);
+        context->getTestReporter()->addTest(filePath, TestResult::Ignored);
         return SLANG_OK;
     }
 
@@ -3226,14 +3248,14 @@ static SlangResult _runTestsOnFile(
             auto& requirements = testDetails.requirements;
 
             // Collect what the test needs (by setting restRequirements the test isn't actually run)
-            context->testRequirements = &requirements;
+            context->setTestRequirements(&requirements);
             runTest(context, filePath, filePath, filePath, testDetails.options);
 
             // 
             apiUsedFlags |= requirements.usedRenderApiFlags;
             explictUsedApiFlags |= (requirements.explicitRenderApi != RenderApiType::Unknown) ? (RenderApiFlags(1) << int(requirements.explicitRenderApi)) : 0;
         }
-        context->testRequirements = nullptr;
+        context->setTestRequirements(nullptr);
     }
 
     SLANG_ASSERT((apiUsedFlags & explictUsedApiFlags) == explictUsedApiFlags);
@@ -3323,8 +3345,7 @@ static SlangResult _runTestsOnFile(
 
         // Report the test and run/ignore
         {
-            auto reporter = context->reporter;
-            TestReporter::TestScope scope(reporter, testName);
+            TestReporter::TestScope scope(context->getTestReporter(), testName);
 
             TestResult testResult = TestResult::Fail;
 
@@ -3338,7 +3359,7 @@ static SlangResult _runTestsOnFile(
                 testResult = runTest(context, filePath, outputStem, testName, testDetails.options);
             }
 
-            reporter->addResult(testResult);
+            context->getTestReporter()->addResult(testResult);
 
             // Could determine if to continue or not here... based on result
         }        
@@ -3400,43 +3421,101 @@ static bool shouldRunTest(
     return true;
 }
 
+void getFilesInDirectory(String directoryPath, List<String>& files)
+{
+    {
+        List<String> localFiles;
+        DirectoryUtil::findFiles(directoryPath, localFiles);
+        files.addRange(localFiles);
+    }
+    {
+        List<String> subDirs;
+        DirectoryUtil::findDirectories(directoryPath, subDirs);
+        for (auto subDir : subDirs)
+        {
+            getFilesInDirectory(subDir, files);
+        }
+    }
+}
+
 void runTestsInDirectory(
     TestContext*		context,
     String				directoryPath)
 {
+    List<String> files;
+    getFilesInDirectory(directoryPath, files);
+    auto processFile = [&](String file)
     {
-        List<String> files;
-        DirectoryUtil::findFiles(directoryPath, files);
-        for (auto file : files)
+        if (shouldRunTest(context, file))
         {
-            if( shouldRunTest(context, file) )
+            //            fprintf(stderr, "slang-test: found '%s'\n", file.getBuffer());
+            if (SLANG_FAILED(_runTestsOnFile(context, file)))
             {
-    //            fprintf(stderr, "slang-test: found '%s'\n", file.getBuffer());
-                if (SLANG_FAILED(_runTestsOnFile(context, file)))
                 {
-                    auto reporter = context->reporter;
-
-                    {
-                        TestReporter::TestScope scope(reporter, file);
-                        reporter->message(TestMessageType::RunError, "slang-test: unable to parse test");
+                    TestReporter::TestScope scope(context->getTestReporter(), file);
+                    context->getTestReporter()->message(
+                        TestMessageType::RunError, "slang-test: unable to parse test");
 
-                        reporter->addResult(TestResult::Fail);
-                    }
-
-                    // Output there was some kind of error trying to run the tests on this file
-                    // fprintf(stderr, "slang-test: unable to parse test '%s'\n", file.getBuffer());
+                    context->getTestReporter()->addResult(TestResult::Fail);
                 }
+
+                // Output there was some kind of error trying to run the tests on this file
+                // fprintf(stderr, "slang-test: unable to parse test '%s'\n", file.getBuffer());
             }
         }
+    };
+    bool useMultiThread = false;
+    switch (context->options.defaultSpawnType)
+    {
+    case SpawnType::UseFullyIsolatedTestServer:
+    case SpawnType::UseTestServer:
+        useMultiThread = true;
+        break;
+    }
+    if (context->options.serverCount == 1)
+    {
+        useMultiThread = false;
+    }
+    if (!useMultiThread)
+    {
+        for (auto file : files)
+        {
+            processFile(file);
+        }
     }
-
+    else
     {
-        List<String> subDirs;
-        DirectoryUtil::findDirectories(directoryPath, subDirs);
-        for (auto subDir : subDirs)
+        auto originalReporter = context->getTestReporter();
+        std::atomic<int> consumePtr;
+        consumePtr = 0;
+        auto threadFunc = [&](int threadId)
         {
-            runTestsInDirectory(context, subDir);
+            TestReporter reporter;
+            reporter.init(context->options.outputMode, true);
+            TestReporter::SuiteScope suiteScope(&reporter, "tests");
+            context->setThreadIndex(threadId);
+            context->setTestReporter(&reporter);
+            do
+            {
+                int index = consumePtr.fetch_add(1);
+                if (index >= (int)files.getCount())
+                    break;
+                processFile(files[index]);
+            } while (true);
+            {
+                std::lock_guard<std::mutex> lock(context->mutex);
+                originalReporter->consolidateWith(&reporter);
+            }
+            context->setTestReporter(nullptr);
+        };
+        List<std::thread> threads;
+        for (int threadId = 0; threadId < context->options.serverCount; threadId++)
+        {
+            threads.add(std::thread(threadFunc, threadId));
         }
+        for (auto& t : threads)
+            t.join();
+        context->setTestReporter(originalReporter);
     }
 }
 
@@ -3654,9 +3733,11 @@ SlangResult innerMain(int argc, char** argv)
     }
 
     SLANG_RETURN_ON_FAIL(Options::parse(argc, argv, &categorySet, StdWriters::getError(), &context.options));
-    
+
     Options& options = context.options;
 
+    context.setMaxTestRunnerThreadCount(options.serverCount);
+
     // Set up the prelude/s
     TestToolUtil::setSessionDefaultPreludeFromExePath(argv[0], context.getSession());
     
@@ -3718,7 +3799,7 @@ SlangResult innerMain(int argc, char** argv)
         TestReporter reporter;
         SLANG_RETURN_ON_FAIL(reporter.init(options.outputMode));
 
-        context.reporter = &reporter;
+        context.setTestReporter(&reporter);
 
         reporter.m_dumpOutputOnFailure = options.dumpOutputOnFailure;
         reporter.m_isVerbose = options.shouldBeVerbose;
diff --git a/tools/slang-test/test-context.cpp b/tools/slang-test/test-context.cpp
index 3e61287cf..cdafbc30d 100644
--- a/tools/slang-test/test-context.cpp
+++ b/tools/slang-test/test-context.cpp
@@ -12,6 +12,8 @@
 
 using namespace Slang;
 
+thread_local int slangTestThreadIndex = 0;
+
 TestContext::TestContext() 
 {
     m_session = nullptr;
@@ -23,6 +25,39 @@ TestContext::TestContext()
 #endif
 }
 
+void TestContext::setThreadIndex(int index) { slangTestThreadIndex = index; }
+
+void TestContext::setMaxTestRunnerThreadCount(int count)
+{
+    m_jsonRpcConnections.setCount(count);
+    m_testRequirements.setCount(count);
+    m_reporters.setCount(count);
+    for (auto& reporter : m_reporters)
+    {
+        reporter = nullptr;
+    }
+}
+
+void TestContext::setTestRequirements(TestRequirements* req)
+{
+    m_testRequirements[slangTestThreadIndex] = req;
+}
+
+TestRequirements* TestContext::getTestRequirements() const
+{
+    return m_testRequirements[slangTestThreadIndex];
+}
+
+void TestContext::setTestReporter(TestReporter* reporter)
+{
+    m_reporters[slangTestThreadIndex] = reporter;
+}
+
+TestReporter* TestContext::getTestReporter()
+{
+    return m_reporters[slangTestThreadIndex];
+}
+
 Result TestContext::init(const char* exePath)
 {
     m_session = spCreateSession(nullptr);
@@ -91,6 +126,7 @@ void TestContext::setInnerMainFunc(const String& name, InnerMainFunc func)
 
 DownstreamCompilerSet* TestContext::getCompilerSet()
 {
+    std::lock_guard<std::mutex> lock(mutex);
     if (!compilerSet)
     {
         compilerSet = new DownstreamCompilerSet;
@@ -138,24 +174,24 @@ SlangResult TestContext::_createJSONRPCConnection(RefPtr<JSONRPCConnection>& out
 
 void TestContext::destroyRPCConnection()
 {
-    if (m_jsonRpcConnection)
+    if (m_jsonRpcConnections[slangTestThreadIndex])
     {
-        m_jsonRpcConnection->disconnect();
-        m_jsonRpcConnection.setNull();
+        m_jsonRpcConnections[slangTestThreadIndex]->disconnect();
+        m_jsonRpcConnections[slangTestThreadIndex].setNull();
     }
 }
 
 Slang::JSONRPCConnection* TestContext::getOrCreateJSONRPCConnection()
 {
-    if (!m_jsonRpcConnection)
+    if (!m_jsonRpcConnections[slangTestThreadIndex])
     {
-        if (SLANG_FAILED(_createJSONRPCConnection(m_jsonRpcConnection)))
+        if (SLANG_FAILED(_createJSONRPCConnection(m_jsonRpcConnections[slangTestThreadIndex])))
         {
             return nullptr;
         }
     }
 
-    return m_jsonRpcConnection;
+    return m_jsonRpcConnections[slangTestThreadIndex];
 }
 
 
diff --git a/tools/slang-test/test-context.h b/tools/slang-test/test-context.h
index 03b74217f..adb3e5ad2 100644
--- a/tools/slang-test/test-context.h
+++ b/tools/slang-test/test-context.h
@@ -17,6 +17,8 @@
 
 #include "options.h"
 
+#include <mutex>
+
 typedef uint32_t PassThroughFlags;
 struct PassThroughFlag
 {
@@ -93,10 +95,14 @@ class TestContext
         /// Set the function for the shared library
     void setInnerMainFunc(const Slang::String& name, InnerMainFunc func);
 
+    void setTestRequirements(TestRequirements* req);
+
+    TestRequirements* getTestRequirements() const;
+
         /// If true tests aren't being run just the information on testing is being accumulated
-    bool isCollectingRequirements() const { return testRequirements != nullptr; }
+    bool isCollectingRequirements() const { return getTestRequirements() != nullptr; }
         /// If set, then tests are executed
-    bool isExecuting() const { return testRequirements == nullptr; }
+    bool isExecuting() const { return getTestRequirements() == nullptr; }
 
         /// True if a render API filter is enabled
     bool isRenderApiFilterEnabled() const { return options.enabledApis != Slang::RenderApiFlag::AllOf && options.enabledApis != 0; }
@@ -126,11 +132,9 @@ class TestContext
     ~TestContext();
 
     Options options;
-    TestReporter* reporter = nullptr;
     TestCategorySet categorySet;
 
         /// If set then tests are not run, but their requirements are set 
-    TestRequirements* testRequirements = nullptr;
 
     PassThroughFlags availableBackendFlags = 0;
     Slang::RenderApiFlags availableRenderApiFlags = 0;
@@ -153,6 +157,14 @@ class TestContext
         /// Current default is 2 mins.
     Slang::Int connectionTimeOutInMs = 2 * 60 * 1000;
 
+    void setThreadIndex(int index);
+    void setMaxTestRunnerThreadCount(int count);
+
+    void setTestReporter(TestReporter* reporter);
+    TestReporter* getTestReporter();
+
+    std::mutex mutex;
+
 protected:
     SlangResult _createJSONRPCConnection(Slang::RefPtr<Slang::JSONRPCConnection>& out);
 
@@ -162,7 +174,9 @@ protected:
         InnerMainFunc m_func;
     };
 
-    Slang::RefPtr<Slang::JSONRPCConnection> m_jsonRpcConnection;
+    Slang::List<Slang::RefPtr<Slang::JSONRPCConnection>> m_jsonRpcConnections;
+    Slang::List<TestReporter*> m_reporters;
+    Slang::List<TestRequirements*> m_testRequirements = nullptr;
     
     SlangSession* m_session;
 
diff --git a/tools/slang-test/test-reporter.cpp b/tools/slang-test/test-reporter.cpp
index 83df0fb0b..ee8421e81 100644
--- a/tools/slang-test/test-reporter.cpp
+++ b/tools/slang-test/test-reporter.cpp
@@ -7,6 +7,8 @@
 #include <stdio.h>
 #include <stdlib.h>
 
+#include <mutex>
+
 using namespace Slang;
 
 /* static */TestReporter* TestReporter::s_reporter = nullptr;
@@ -82,9 +84,10 @@ TestReporter::TestReporter() :
     m_isVerbose = false;
 }
 
-Result TestReporter::init(TestOutputMode outputMode)
+Result TestReporter::init(TestOutputMode outputMode, bool isSubReporter)
 {
     m_outputMode = outputMode;
+    m_isSubReporter = isSubReporter;
     return SLANG_OK;
 }
 
@@ -191,6 +194,15 @@ TestResult TestReporter::addTest(const String& testName, bool isPass)
     return res;
 }
 
+void TestReporter::consolidateWith(TestReporter* other)
+{
+    m_testInfos.addRange(other->m_testInfos);
+    m_failedTestCount += other->m_failedTestCount;
+    m_ignoredTestCount += other->m_ignoredTestCount;
+    m_passedTestCount += other->m_passedTestCount;
+    m_totalTestCount += other->m_totalTestCount;
+}
+
 void TestReporter::dumpOutputDifference(const String& expectedOutput, const String& actualOutput)
 {
     StringBuilder builder;
@@ -493,6 +505,8 @@ void TestReporter::addTest(const String& testName, TestResult testResult)
 
 void TestReporter::message(TestMessageType type, const String& message)
 {
+    static std::mutex mutex;
+    std::lock_guard<std::mutex> lock(mutex);
     if (type == TestMessageType::Info)
     {
         if (m_isVerbose && canWriteStdError())
@@ -666,9 +680,12 @@ void TestReporter::startSuite(const String& name)
     {
         case TestOutputMode::TeamCity:
         {
-            StringBuilder escapedSuiteName;
-            _appendEncodedTeamCityString(name.getUnownedSlice(), escapedSuiteName);
-            printf("##teamcity[testSuiteStarted name='%s']\n", escapedSuiteName.begin());
+            if (!m_isSubReporter)
+            {
+                StringBuilder escapedSuiteName;
+                _appendEncodedTeamCityString(name.getUnownedSlice(), escapedSuiteName);
+                printf("##teamcity[testSuiteStarted name='%s']\n", escapedSuiteName.begin());
+            }
             break;
         }
         default: break;
@@ -683,10 +700,13 @@ void TestReporter::endSuite()
     {
         case TestOutputMode::TeamCity:
         {
-            const String& name = m_suiteStack.getLast();
-            StringBuilder escapedSuiteName;
-            _appendEncodedTeamCityString(name.getUnownedSlice(), escapedSuiteName);
-            printf("##teamcity[testSuiteFinished name='%s']\n", escapedSuiteName.begin());
+            if (!m_isSubReporter)
+            {
+                const String& name = m_suiteStack.getLast();
+                StringBuilder escapedSuiteName;
+                _appendEncodedTeamCityString(name.getUnownedSlice(), escapedSuiteName);
+                printf("##teamcity[testSuiteFinished name='%s']\n", escapedSuiteName.begin());
+            }
             break;
         }
         default: break;
diff --git a/tools/slang-test/test-reporter.h b/tools/slang-test/test-reporter.h
index 94e5de6ed..b87c5368f 100644
--- a/tools/slang-test/test-reporter.h
+++ b/tools/slang-test/test-reporter.h
@@ -87,6 +87,8 @@ class TestReporter : public ITestReporter
 
     void dumpOutputDifference(const Slang::String& expectedOutput, const Slang::String& actualOutput);
 
+    void consolidateWith(TestReporter* other);
+
         /// True if can write output directly to stderr
     bool canWriteStdError() const;
 
@@ -96,7 +98,7 @@ class TestReporter : public ITestReporter
 
     void outputSummary();
 
-    SlangResult init(TestOutputMode outputMode);
+    SlangResult init(TestOutputMode outputMode, bool isSubReporter = false);
 
         /// Ctor
     TestReporter();
@@ -123,6 +125,7 @@ class TestReporter : public ITestReporter
     bool m_dumpOutputOnFailure;
     bool m_isVerbose = false;
     bool m_hideIgnored = false;
+    bool m_isSubReporter = false;
 
 protected: