Fix memory leaks around slang-test (#757)

* Remove circular reference to renderer on Vk & D3D12 DescriptorSetImpl

* Refactor Stbi image loading such that memory is correctly freed when goes out of scope.
Added Crt memory dump at termination.
Reduced erroneous reporting by scoping TestContext.

* Used capitalized acronym for STBImage to keep Tim happy.

* Split out TestReporter - to just handle reporting test results
Split out Options
Made TestContext hold options, and the reporter
Removed remaining memory leaks.

* Small optimization for rawWrite, such that it directly writes over print..

* Improve comments on TestCategorySet

* Fix typos in TestCategorySet

1 files changed, 96 insertions, 399 deletions

diff --git a/tools/slang-test/main.cpp b/tools/slang-test/main.cpp
index 8a03a6100..774fbd6e2 100644
--- a/tools/slang-test/main.cpp
+++ b/tools/slang-test/main.cpp
@@ -13,6 +13,8 @@ using namespace Slang;
 #include "os.h"
 #include "render-api-util.h"
 #include "test-context.h"
+#include "test-reporter.h"
+#include "options.h"
 
 #define STB_IMAGE_IMPLEMENTATION
 #include "external/stb/stb_image.h"
@@ -29,16 +31,6 @@ using namespace Slang;
 #include <stdlib.h>
 #include <stdarg.h>
 
-// A category that a test can be tagged with
-struct TestCategory
-{
-    // The name of the category, from the user perspective
-    String name;
-
-    // The logical "super-category" of this category
-    TestCategory* parent;
-};
-
 // Options for a particular test
 struct TestOptions
 {
@@ -75,275 +67,10 @@ struct TestInput
 
 typedef TestResult(*TestCallback)(TestContext* context, TestInput& input);
 
-struct Options
-{
-    char const* appName = "slang-test";
-
-    // Directory to use when looking for binaries to run
-    char const* binDir = "";
-
-    // only run test cases with names that have this prefix
-    char const* testPrefix = nullptr;
-
-    // generate extra output (notably: command lines we run)
-    bool shouldBeVerbose = false;
-
-    // force generation of baselines for HLSL tests
-    bool generateHLSLBaselines = false;
-
-    // Dump expected/actual output on failures, for debugging.
-    // This is especially intended for use in continuous
-    // integration builds.
-    bool dumpOutputOnFailure = false;
-
-    // If set, will force using of executables (not shared library) for tests
-    bool useExes = false;
-
-    // kind of output to generate
-    TestOutputMode outputMode = TestOutputMode::Default;
-
-    // Only run tests that match one of the given categories
-    Dictionary<TestCategory*, TestCategory*> includeCategories;
-
-    // Exclude test that match one these categories
-    Dictionary<TestCategory*, TestCategory*> excludeCategories;
-
-    // By default we can test against all apis
-    RenderApiFlags enabledApis = RenderApiFlag::AllOf;
-
-    // By default we potentially synthesize test for all 
-    // TODO: Vulkan is disabled by default for now as the majority as vulkan synthesized tests fail  
-    RenderApiFlags synthesizedTestApis = RenderApiFlag::AllOf & ~RenderApiFlag::Vulkan;
-};
-
 // Globals
 
-static const Options g_defaultOptions;
-
-Options g_options;
-Dictionary<String, TestCategory*> g_testCategories;
-TestCategory* g_defaultTestCategory;
-
-// pre declare
-
-TestCategory* findTestCategory(String const& name);
-
 /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!! Functions !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */
 
-
-Result parseOptions(int argc, char** argv, Slang::WriterHelper stdError)
-{
-    g_options = g_defaultOptions;
-
-    List<const char*> positionalArgs;
-    
-    int argCount = argc;
-    char const* const* argCursor = argv;
-    char const* const* argEnd = argCursor + argCount;
-
-    // first argument is the application name
-    if( argCursor != argEnd )
-    {
-        g_options.appName = *argCursor++;
-    }
-
-    // now iterate over arguments to collect options
-    while(argCursor != argEnd)
-    {
-        char const* arg = *argCursor++;
-        if( arg[0] != '-' )
-        {
-            positionalArgs.Add(arg);
-            continue;
-        }
-
-        if( strcmp(arg, "--") == 0 )
-        {
-            while(argCursor != argEnd)
-            {
-                positionalArgs.Add(*argCursor++);
-            }
-            break;
-        }
-
-        if( strcmp(arg, "-bindir") == 0 )
-        {
-            if( argCursor == argEnd )
-            {
-                stdError.print("error: expected operand for '%s'\n", arg);
-                return SLANG_FAIL;
-            }
-            g_options.binDir = *argCursor++;
-        }
-        else if (strcmp(arg, "-useexes") == 0)
-        {
-            g_options.useExes = true;
-        }
-        else if( strcmp(arg, "-v") == 0 )
-        {
-            g_options.shouldBeVerbose = true;
-        }
-        else if( strcmp(arg, "-generate-hlsl-baselines") == 0 )
-        {
-            g_options.generateHLSLBaselines = true;
-        }
-        else if( strcmp(arg, "-release") == 0 )
-        {
-            // Assumed to be handle by .bat file that called us
-        }
-        else if( strcmp(arg, "-debug") == 0 )
-        {
-            // Assumed to be handle by .bat file that called us
-        }
-        else if( strcmp(arg, "-configuration") == 0 )
-        {
-            if( argCursor == argEnd )
-            {
-                stdError.print("error: expected operand for '%s'\n", arg);
-				return SLANG_FAIL;
-            }
-            argCursor++;
-            // Assumed to be handle by .bat file that called us
-        }
-        else if( strcmp(arg, "-platform") == 0 )
-        {
-            if( argCursor == argEnd )
-            {
-                stdError.print("error: expected operand for '%s'\n", arg);
-				return SLANG_FAIL;
-            }
-            argCursor++;
-            // Assumed to be handle by .bat file that called us
-        }
-        else if( strcmp(arg, "-appveyor") == 0 )
-        {
-            g_options.outputMode = TestOutputMode::AppVeyor;
-            g_options.dumpOutputOnFailure = true;
-        }
-        else if( strcmp(arg, "-travis") == 0 )
-        {
-            g_options.outputMode = TestOutputMode::Travis;
-            g_options.dumpOutputOnFailure = true;
-        }
-        else if (strcmp(arg, "-xunit") == 0)
-        {
-            g_options.outputMode = TestOutputMode::XUnit;
-        }
-        else if (strcmp(arg, "-xunit2") == 0)
-        {
-            g_options.outputMode = TestOutputMode::XUnit2;
-        }
-        else if (strcmp(arg, "-teamcity") == 0)
-        {
-            g_options.outputMode = TestOutputMode::TeamCity;
-        }
-        else if( strcmp(arg, "-category") == 0 )
-        {
-            if( argCursor == argEnd )
-            {
-                stdError.print("error: expected operand for '%s'\n", arg);
-                return SLANG_FAIL;
-            }
-            auto category = findTestCategory(*argCursor++);
-            if(category)
-            {
-                g_options.includeCategories.Add(category, category);
-            }
-        }
-        else if( strcmp(arg, "-exclude") == 0 )
-        {
-            if( argCursor == argEnd )
-            {
-                stdError.print("error: expected operand for '%s'\n", arg);
-                return SLANG_FAIL;
-            }
-            auto category = findTestCategory(*argCursor++);
-            if(category)
-            {
-                g_options.excludeCategories.Add(category, category);
-            }
-        }
-        else if (strcmp(arg, "-api") == 0)
-        {
-            if (argCursor == argEnd)
-            {
-                stdError.print("error: expecting an api expression (eg 'vk+dx12' or '+dx11') '%s'\n", arg);
-                return SLANG_FAIL;
-            }
-            const char* apiList = *argCursor++;
-
-            SlangResult res = RenderApiUtil::parseApiFlags(UnownedStringSlice(apiList), g_options.enabledApis, &g_options.enabledApis);
-            if (SLANG_FAILED(res))
-            {
-                stdError.print("error: unable to parse api expression '%s'\n", apiList);
-                return res;
-            }
-        }
-        else if (strcmp(arg, "-synthesizedTestApi") == 0)
-        {
-            if (argCursor == argEnd)
-            {
-                stdError.print("error: expected an api expression (eg 'vk+dx12' or '+dx11') '%s'\n", arg);
-                return SLANG_FAIL;
-            }
-            const char* apiList = *argCursor++;
-
-            SlangResult res = RenderApiUtil::parseApiFlags(UnownedStringSlice(apiList), g_options.synthesizedTestApis, &g_options.synthesizedTestApis);
-            if (SLANG_FAILED(res))
-            {
-                stdError.print("error: unable to parse api expression '%s'\n", apiList);
-                return res;
-            }
-        }
-        else
-        {
-            stdError.print("unknown option '%s'\n", arg);
-            return SLANG_FAIL;
-        }
-    }
-    
-    {
-        // Find out what apis are available
-        const int availableApis = RenderApiUtil::getAvailableApis();
-        // Only allow apis we know are available
-        g_options.enabledApis &= availableApis;
-
-        // Can only synth for apis that are available
-        g_options.synthesizedTestApis &= g_options.enabledApis;
-    }
-
-
-    // first positional argument is source shader path
-    if (positionalArgs.Count())
-    {
-        g_options.testPrefix = positionalArgs[0];
-        positionalArgs.RemoveAt(0);
-    }
-
-    // any remaining arguments represent an error
-    if (positionalArgs.Count() != 0)
-    {
-        stdError.print("unexpected arguments\n");
-        return SLANG_FAIL;
-    }
-
-	return SLANG_OK;
-}
-
-// Called for an error in the test-runner (not for an error involving
-// a test itself).
-void error(char const* message, ...)
-{
-    fprintf(stderr, "error: ");
-
-    va_list args;
-    va_start(args, message);
-    vfprintf(stderr, message, args);
-    va_end(args);
-
-    fprintf(stderr, "\n");
-}
-
 bool match(char const** ioCursor, char const* expected)
 {
     char const* cursor = *ioCursor;
@@ -427,30 +154,10 @@ String collectRestOfLine(char const** ioCursor)
     return getString(textBegin, textEnd);
 }
 
-TestCategory* addTestCategory(String const& name, TestCategory* parent)
-{
-    TestCategory* category = new TestCategory();
-    category->name = name;
-
-    category->parent = parent;
-
-    g_testCategories.Add(name, category);
-
-    return category;
-}
 
-TestCategory* findTestCategory(String const& name)
-{
-    TestCategory* category = nullptr;
-    if( !g_testCategories.TryGetValue(name, category) )
-    {
-        error("unknown test category name '%s'\n", name.Buffer());
-        return nullptr;
-    }
-    return category;
-}
 
 TestResult gatherTestOptions(
+    TestCategorySet*    categorySet, 
     char const**    ioCursor,
     FileTestList*   testList)
 {
@@ -481,12 +188,13 @@ TestResult gatherTestOptions(
                     cursor++;
 
                     auto categoryName = getString(categoryStart, categoryEnd);
-                    TestCategory* category = findTestCategory(categoryName);
+                    TestCategory* category = categorySet->find(categoryName);
 
                     if(!category)
                     {
                         return TestResult::Fail;
                     }
+                    
 
                     testOptions.categories.Add(category);
 
@@ -510,7 +218,7 @@ TestResult gatherTestOptions(
     // If no categories were specified, then add the default category
     if(testOptions.categories.Count() == 0)
     {
-        testOptions.categories.Add(g_defaultTestCategory);
+        testOptions.categories.Add(categorySet->defaultCategory);
     }
 
     if(*cursor == ':')
@@ -594,6 +302,7 @@ TestResult gatherTestOptions(
 
 // Try to read command-line options from the test file itself
 TestResult gatherTestsForFile(
+    TestCategorySet*    categorySet,
     String				filePath,
     FileTestList*       testList)
 {
@@ -624,7 +333,7 @@ TestResult gatherTestsForFile(
         }
         else if(match(&cursor, "//TEST"))
         {
-            if(gatherTestOptions(&cursor, testList) != TestResult::Pass)
+            if(gatherTestOptions(categorySet, &cursor, testList) != TestResult::Pass)
                 return TestResult::Fail;
         }
         else
@@ -640,17 +349,17 @@ OSError spawnAndWait(TestContext* context, const String& testPath, OSProcessSpaw
 {
     SLANG_UNUSED(context);
 
-    if(context->m_isVerbose)
+    if(context->options.shouldBeVerbose)
     {
         String commandLine = spawner.getCommandLine();
-        context->messageFormat(TestMessageType::Info, "%s\n", commandLine.begin());
+        context->reporter->messageFormat(TestMessageType::Info, "%s\n", commandLine.begin());
     }
 
-    if (!context->m_useExes)
+    if (!context->options.useExes)
     {
         String exeName = Path::GetFileNameWithoutEXT(spawner.executableName_);
 
-        auto func = context->getInnerMainFunc(String(g_options.binDir), exeName);
+        auto func = context->getInnerMainFunc(String(context->options.binDir), exeName);
         if (func)
         {
             StringBuilder stdErrorString;
@@ -691,7 +400,7 @@ OSError spawnAndWait(TestContext* context, const String& testPath, OSProcessSpaw
     if (err != kOSError_None)
     {
 //        fprintf(stderr, "failed to run test '%S'\n", testPath.ToWString());
-        context->messageFormat(TestMessageType::RunError, "failed to run test '%S'", testPath.ToWString().begin());
+        context->reporter->messageFormat(TestMessageType::RunError, "failed to run test '%S'", testPath.ToWString().begin());
     }
     return err;
 }
@@ -763,7 +472,7 @@ TestResult runSimpleTest(TestContext* context, TestInput& input)
 
     OSProcessSpawner spawner;
 
-    spawner.pushExecutablePath(String(g_options.binDir) + "slangc" + osGetExecutableSuffix());
+    spawner.pushExecutablePath(String(context->options.binDir) + "slangc" + osGetExecutableSuffix());
     spawner.pushArgument(filePath999);
 
     for( auto arg : input.testOptions->args )
@@ -800,7 +509,7 @@ TestResult runSimpleTest(TestContext* context, TestInput& input)
     // Otherwise we compare to the expected output
     if (actualOutput != expectedOutput)
     {
-        context->dumpOutputDifference(expectedOutput, actualOutput);
+        context->reporter->dumpOutputDifference(expectedOutput, actualOutput);
         result = TestResult::Fail;
     }
 
@@ -812,7 +521,7 @@ TestResult runSimpleTest(TestContext* context, TestInput& input)
         String actualOutputPath = outputStem + ".actual";
         Slang::File::WriteAllText(actualOutputPath, actualOutput);
 
-        context->dumpOutputDifference(expectedOutput, actualOutput);
+        context->reporter->dumpOutputDifference(expectedOutput, actualOutput);
     }
 
     return result;
@@ -820,12 +529,13 @@ TestResult runSimpleTest(TestContext* context, TestInput& input)
 
 TestResult runReflectionTest(TestContext* context, TestInput& input)
 {
+    const auto& options = context->options;
     auto filePath = input.filePath;
     auto outputStem = input.outputStem;
 
     OSProcessSpawner spawner;
 
-    spawner.pushExecutablePath(String(g_options.binDir) + "slang-reflection-test" + osGetExecutableSuffix());
+    spawner.pushExecutablePath(String(options.binDir) + "slang-reflection-test" + osGetExecutableSuffix());
     spawner.pushArgument(filePath);
 
     for( auto arg : input.testOptions->args )
@@ -873,7 +583,7 @@ TestResult runReflectionTest(TestContext* context, TestInput& input)
         String actualOutputPath = outputStem + ".actual";
         Slang::File::WriteAllText(actualOutputPath, actualOutput);
 
-        context->dumpOutputDifference(expectedOutput, actualOutput);
+        context->reporter->dumpOutputDifference(expectedOutput, actualOutput);
     }
 
     return result;
@@ -932,8 +642,8 @@ TestResult runCrossCompilerTest(TestContext* context, TestInput& input)
     OSProcessSpawner actualSpawner;
     OSProcessSpawner expectedSpawner;
 
-    actualSpawner.pushExecutablePath(String(g_options.binDir) + "slangc" + osGetExecutableSuffix());
-    expectedSpawner.pushExecutablePath(String(g_options.binDir) + "slangc" + osGetExecutableSuffix());
+    actualSpawner.pushExecutablePath(String(context->options.binDir) + "slangc" + osGetExecutableSuffix());
+    expectedSpawner.pushExecutablePath(String(context->options.binDir) + "slangc" + osGetExecutableSuffix());
 
     actualSpawner.pushArgument(filePath);
 
@@ -1022,7 +732,7 @@ TestResult runCrossCompilerTest(TestContext* context, TestInput& input)
         String actualOutputPath = outputStem + ".actual";
         Slang::File::WriteAllText(actualOutputPath, actualOutput);
 
-        context->dumpOutputDifference(expectedOutput, actualOutput);
+        context->reporter->dumpOutputDifference(expectedOutput, actualOutput);
     }
 
     return result;
@@ -1036,7 +746,7 @@ TestResult generateHLSLBaseline(TestContext* context, TestInput& input)
     auto outputStem = input.outputStem;
 
     OSProcessSpawner spawner;
-    spawner.pushExecutablePath(String(g_options.binDir) + "slangc" + osGetExecutableSuffix());
+    spawner.pushExecutablePath(String(context->options.binDir) + "slangc" + osGetExecutableSuffix());
     spawner.pushArgument(filePath999);
 
     for( auto arg : input.testOptions->args )
@@ -1082,7 +792,7 @@ TestResult runHLSLComparisonTest(TestContext* context, TestInput& input)
 
     OSProcessSpawner spawner;
 
-    spawner.pushExecutablePath(String(g_options.binDir) + "slangc" + osGetExecutableSuffix());
+    spawner.pushExecutablePath(String(context->options.binDir) + "slangc" + osGetExecutableSuffix());
     spawner.pushArgument(filePath999);
 
     for( auto arg : input.testOptions->args )
@@ -1164,7 +874,7 @@ TestResult runHLSLComparisonTest(TestContext* context, TestInput& input)
         String actualOutputPath = outputStem + ".actual";
         Slang::File::WriteAllText(actualOutputPath, actualOutput);
 
-        context->dumpOutputDifference(expectedOutput, actualOutput);
+        context->reporter->dumpOutputDifference(expectedOutput, actualOutput);
     }
 
     return result;
@@ -1183,7 +893,7 @@ TestResult doGLSLComparisonTestRun(TestContext* context,
 
     OSProcessSpawner spawner;
 
-    spawner.pushExecutablePath(String(g_options.binDir) + "slangc" + osGetExecutableSuffix());
+    spawner.pushExecutablePath(String(context->options.binDir) + "slangc" + osGetExecutableSuffix());
     spawner.pushArgument(filePath999);
 
     if( langDefine )
@@ -1253,7 +963,7 @@ TestResult runGLSLComparisonTest(TestContext* context, TestInput& input)
 
     if (actualOutput != expectedOutput)
     {
-        context->dumpOutputDifference(expectedOutput, actualOutput);
+        context->reporter->dumpOutputDifference(expectedOutput, actualOutput);
 
         return TestResult::Fail;
     }
@@ -1276,7 +986,7 @@ TestResult runComputeComparisonImpl(TestContext* context, TestInput& input, cons
 
 	OSProcessSpawner spawner;
 
-	spawner.pushExecutablePath(String(g_options.binDir) + "render-test" + osGetExecutableSuffix());
+	spawner.pushExecutablePath(String(context->options.binDir) + "render-test" + osGetExecutableSuffix());
 	spawner.pushArgument(filePath999);
 
 	for (auto arg : input.testOptions->args)
@@ -1305,7 +1015,7 @@ TestResult runComputeComparisonImpl(TestContext* context, TestInput& input, cons
     auto expectedOutput = getExpectedOutput(outputStem);
     if (actualOutput != expectedOutput)
     {
-        context->dumpOutputDifference(expectedOutput, actualOutput);
+        context->reporter->dumpOutputDifference(expectedOutput, actualOutput);
 
         String actualOutputPath = outputStem + ".actual";
         Slang::File::WriteAllText(actualOutputPath, actualOutput);
@@ -1330,7 +1040,7 @@ TestResult runComputeComparisonImpl(TestContext* context, TestInput& input, cons
 	auto referenceProgramOutput = Split(File::ReadAllText(referenceOutput), '\n');
     auto printOutput = [&]()
     {
-        context->messageFormat(TestMessageType::TestFailure, "output mismatch! actual output: {\n%s\n}, \n%s\n", actualOutputContent.Buffer(), actualOutput.Buffer());
+        context->reporter->messageFormat(TestMessageType::TestFailure, "output mismatch! actual output: {\n%s\n}, \n%s\n", actualOutputContent.Buffer(), actualOutput.Buffer());
     };
     if (actualProgramOutput.Count() < referenceProgramOutput.Count())
     {
@@ -1390,7 +1100,7 @@ TestResult doRenderComparisonTestRun(TestContext* context, TestInput& input, cha
 
     OSProcessSpawner spawner;
 
-    spawner.pushExecutablePath(String(g_options.binDir) + "render-test" + osGetExecutableSuffix());
+    spawner.pushExecutablePath(String(context->options.binDir) + "render-test" + osGetExecutableSuffix());
     spawner.pushArgument(filePath);
 
     for( auto arg : input.testOptions->args )
@@ -1495,6 +1205,8 @@ bool STBImage::isComparable(const ThisType& rhs) const
 
 TestResult doImageComparison(TestContext* context, String const& filePath)
 {
+    auto reporter = context->reporter;
+
     // Allow a difference in the low bits of the 8-bit result, just to play it safe
     static const int kAbsoluteDiffCutoff = 2;
 
@@ -1507,20 +1219,20 @@ TestResult doImageComparison(TestContext* context, String const& filePath)
     STBImage expectedImage;
     if (SLANG_FAILED(expectedImage.read(expectedPath.Buffer())))
     {
-        context->messageFormat(TestMessageType::RunError, "Unable to load image ;%s'", expectedPath.Buffer());
+        reporter->messageFormat(TestMessageType::RunError, "Unable to load image ;%s'", expectedPath.Buffer());
         return TestResult::Fail;
     }
 
     STBImage actualImage;
     if (SLANG_FAILED(actualImage.read(actualPath.Buffer())))
     {
-        context->messageFormat(TestMessageType::RunError, "Unable to load image ;%s'", actualPath.Buffer());
+        reporter->messageFormat(TestMessageType::RunError, "Unable to load image ;%s'", actualPath.Buffer());
         return TestResult::Fail;
     }
 
     if (!expectedImage.isComparable(actualImage))
     {
-        context->messageFormat(TestMessageType::TestFailure, "Images are different sizes '%s' '%s'", actualPath.Buffer(), expectedPath.Buffer());
+        reporter->messageFormat(TestMessageType::TestFailure, "Images are different sizes '%s' '%s'", actualPath.Buffer(), expectedPath.Buffer());
         return TestResult::Fail;
     }
 
@@ -1568,7 +1280,7 @@ TestResult doImageComparison(TestContext* context, String const& filePath)
                 const int x = i / numChannels;
                 const int channelIndex = i % numChannels;
 
-                context->messageFormat(TestMessageType::TestFailure, "image compare failure at (%d,%d) channel %d. expected %d got %d (absolute error: %d, relative error: %f)\n",
+                reporter->messageFormat(TestMessageType::TestFailure, "image compare failure at (%d,%d) channel %d. expected %d got %d (absolute error: %d, relative error: %f)\n",
                     x, y, channelIndex,
                     expectedVal,
                     actualVal,
@@ -1610,7 +1322,7 @@ TestResult runHLSLRenderComparisonTestImpl(
 
     if (actualOutput != expectedOutput)
     {
-        context->dumpOutputDifference(expectedOutput, actualOutput);
+        context->reporter->dumpOutputDifference(expectedOutput, actualOutput);
 
         return TestResult::Fail;
     }
@@ -1644,7 +1356,6 @@ TestResult skipTest(TestContext* /* context */, TestInput& /*input*/)
     return TestResult::Ignored;
 }
 
-
 static bool hasRenderOption(RenderApiType apiType, const List<String>& options)
 {
     const RenderApiUtil::Info& info = RenderApiUtil::getInfo(apiType);
@@ -1721,14 +1432,14 @@ bool isRenderTest(const String& command)
         command == "COMPARE_HLSL_GLSL_RENDER";
 }
 
-static bool canIgnoreTestWithDisabledRenderer(const TestOptions& testOptions)
+static bool canIgnoreTestWithDisabledRenderer(const TestOptions& testOptions, const Options& appOptions)
 {
     for (int i = 0; i < int(RenderApiType::CountOf); ++i)
     {
         RenderApiType apiType = RenderApiType(i);
         RenderApiFlag::Enum apiFlag = RenderApiFlag::Enum(1 << i);
         
-        if (hasRenderOption(apiType, testOptions) && (g_options.enabledApis & apiFlag) == 0)
+        if (hasRenderOption(apiType, testOptions) && (appOptions.enabledApis & apiFlag) == 0)
         {
             return true;
         }
@@ -1745,7 +1456,7 @@ TestResult runTest(
     FileTestList const& testList)
 {
     // If this test can be ignored
-    if (canIgnoreTestWithDisabledRenderer(testOptions))
+    if (canIgnoreTestWithDisabledRenderer(testOptions, context->options))
     {
         return TestResult::Ignored;
     }
@@ -1798,10 +1509,11 @@ TestResult runTest(
         testInput.testList = &testList;
 
         {
-            TestContext::TestScope scope(context, outputStem);
+            TestReporter* reporter = context->reporter;
+            TestReporter::TestScope scope(reporter, outputStem);
 
             TestResult testResult = ii->callback(context, testInput);
-            context->addResult(testResult); 
+            reporter->addResult(testResult);
 
             return testResult;
        }
@@ -1840,20 +1552,20 @@ bool testCategoryMatches(
 }
 
 bool testPassesCategoryMask(
-    TestContext*        /*context*/,
+    TestContext*        context,
     TestOptions const&  test)
 {
     // Don't include a test we should filter out
     for( auto testCategory : test.categories )
     {
-        if(testCategoryMatches(testCategory, g_options.excludeCategories))
+        if(testCategoryMatches(testCategory, context->options.excludeCategories))
             return false;
     }
 
     // Otherwise inclue any test the user asked for
     for( auto testCategory : test.categories )
     {
-        if(testCategoryMatches(testCategory, g_options.includeCategories))
+        if(testCategoryMatches(testCategory, context->options.includeCategories))
             return true;
     }
 
@@ -1868,7 +1580,7 @@ void runTestsOnFile(
     // Gather a list of tests to run
     FileTestList testList;
 
-    if( gatherTestsForFile(filePath, &testList) == TestResult::Ignored )
+    if( gatherTestsForFile(&context->categorySet, filePath, &testList) == TestResult::Ignored )
     {
         // Test was explicitly ignored
         return;
@@ -1877,14 +1589,14 @@ void runTestsOnFile(
     // Note cases where a test file exists, but we found nothing to run
     if( testList.tests.Count() == 0 )
     {
-        context->addTest(filePath, TestResult::Ignored);
+        context->reporter->addTest(filePath, TestResult::Ignored);
         return;
     }
 
     List<TestOptions> synthesizedTests;
 
     // If dx12 is available synthesize Dx12 test
-    if ((g_options.synthesizedTestApis & RenderApiFlag::D3D12) != 0)
+    if ((context->options.synthesizedTestApis & RenderApiFlag::D3D12) != 0)
     {
         // If doesn't have option generate dx12 options from dx11
         if (!hasRenderOption(RenderApiType::D3D12, testList))
@@ -1907,7 +1619,7 @@ void runTestsOnFile(
     }
 
     // If Vulkan is available synthesize Vulkan test
-    if ((g_options.synthesizedTestApis & RenderApiFlag::Vulkan) != 0)
+    if ((context->options.synthesizedTestApis & RenderApiFlag::Vulkan) != 0)
     {
         // If doesn't have option generate dx12 options from dx11
         if (!hasRenderOption(RenderApiType::Vulkan, testList))
@@ -2004,9 +1716,9 @@ static bool shouldRunTest(
     if(!endsWithAllowedExtension(context, filePath))
         return false;
 
-    if( g_options.testPrefix )
+    if( context->options.testPrefix )
     {
-        if( strncmp(g_options.testPrefix, filePath.begin(), strlen(g_options.testPrefix)) != 0 )
+        if( strncmp(context->options.testPrefix, filePath.begin(), strlen(context->options.testPrefix)) != 0 )
         {
             return false;
         }
@@ -2033,74 +1745,57 @@ void runTestsInDirectory(
     }
 }
 
-
-//
-
-int main(
-    int		argcIn,
-    char**	argvIn)
+SlangResult innerMain(int argc, char** argv)
 {
     AppContext::initDefault();
 
-    // Set up our test categories here
+    // The context holds useful things used during testing
+    TestContext context;
+    auto& categorySet = context.categorySet;
 
-    auto fullTestCategory = addTestCategory("full", nullptr);
-
-    auto quickTestCategory = addTestCategory("quick", fullTestCategory);
-
-    /*auto smokeTestCategory = */addTestCategory("smoke", quickTestCategory);
-
-    auto renderTestCategory = addTestCategory("render", fullTestCategory);
-
-	/*auto computeTestCategory = */addTestCategory("compute", fullTestCategory);
-
-    auto vulkanTestCategory = addTestCategory("vulkan", fullTestCategory);
-
-    auto unitTestCatagory = addTestCategory("unit-test", fullTestCategory);
-
-    auto compatibilityIssueCatagory = addTestCategory("compatibility-issue", fullTestCategory);
+    // Set up our test categories here
+    auto fullTestCategory = categorySet.add("full", nullptr);
+    auto quickTestCategory = categorySet.add("quick", fullTestCategory);
+    /*auto smokeTestCategory = */categorySet.add("smoke", quickTestCategory);
+    auto renderTestCategory = categorySet.add("render", fullTestCategory);
+    /*auto computeTestCategory = */categorySet.add("compute", fullTestCategory);
+    auto vulkanTestCategory = categorySet.add("vulkan", fullTestCategory);
+    auto unitTestCatagory = categorySet.add("unit-test", fullTestCategory);
+    auto compatibilityIssueCatagory = categorySet.add("compatibility-issue", fullTestCategory);
 
     // An un-categorized test will always belong to the `full` category
-    g_defaultTestCategory = fullTestCategory;
+    categorySet.defaultCategory = fullTestCategory;
 
-    //
+    SLANG_RETURN_ON_FAIL(Options::parse(argc, argv, &categorySet, AppContext::getStdError(), &context.options));
+    SLANG_RETURN_ON_FAIL(SLANG_FAILED(context.init()))
 
-    if (SLANG_FAILED(parseOptions(argcIn, argvIn, AppContext::getStdError())))
+    Options& options = context.options;
+    if( options.includeCategories.Count() == 0 )
     {
-    	// Return exit code with error
-    	return 1;
-    }
-
-    if( g_options.includeCategories.Count() == 0 )
-    {
-        g_options.includeCategories.Add(fullTestCategory, fullTestCategory);
+        options.includeCategories.Add(fullTestCategory, fullTestCategory);
     }
 
     // Exclude rendering tests when building under AppVeyor.
     //
     // TODO: this is very ad hoc, and we should do something cleaner.
-    if( g_options.outputMode == TestOutputMode::AppVeyor )
+    if( options.outputMode == TestOutputMode::AppVeyor )
     {
-        g_options.excludeCategories.Add(renderTestCategory, renderTestCategory);
-        g_options.excludeCategories.Add(vulkanTestCategory, vulkanTestCategory);
+        options.excludeCategories.Add(renderTestCategory, renderTestCategory);
+        options.excludeCategories.Add(vulkanTestCategory, vulkanTestCategory);
     }
 
-    int returnCode = 0;
     {
-        // Setup the context 
-        TestContext context;
-        if (SLANG_FAILED(context.init(g_options.outputMode)))
-        {
-            // Unable to initialize context
-            return 1;
-        }
+        // Setup the reporter
+        TestReporter reporter;
+        SLANG_RETURN_ON_FAIL(reporter.init(options.outputMode));
+
+        context.reporter = &reporter;
 
-        context.m_dumpOutputOnFailure = g_options.dumpOutputOnFailure;
-        context.m_useExes = g_options.useExes;
-        context.m_isVerbose = g_options.shouldBeVerbose;
+        reporter.m_dumpOutputOnFailure = options.dumpOutputOnFailure;
+        reporter.m_isVerbose = options.shouldBeVerbose;
 
         {
-            TestContext::SuiteScope suiteScope(&context, "tests");
+            TestReporter::SuiteScope suiteScope(&reporter, "tests");
             // Enumerate test files according to policy
             // TODO: add more directories to this list
             // TODO: allow for a command-line argument to select a particular directory
@@ -2110,8 +1805,8 @@ int main(
         // Run the unit tests (these are internal C++ tests - not specified via files in a directory) 
         // They are registered with SLANG_UNIT_TEST macro
         {
-            TestContext::SuiteScope suiteScope(&context, "unit tests");
-            TestContext::set(&context);
+            TestReporter::SuiteScope suiteScope(&reporter, "unit tests");
+            TestReporter::set(&reporter);
 
             // Run the unit tests
             TestRegister* cur = TestRegister::s_first;
@@ -2128,14 +1823,14 @@ int main(
                 {
                     if (testPassesCategoryMask(&context, testOptions))
                     {
-                        context.startTest(testOptions.command);
+                        reporter.startTest(testOptions.command);
                         // Run the test function
                         cur->m_func();
-                        context.endTest();
+                        reporter.endTest();
                     }
                     else
                     {
-                        context.addTest(testOptions.command, TestResult::Ignored);
+                        reporter.addTest(testOptions.command, TestResult::Ignored);
                     }
                 }
 
@@ -2143,17 +1838,19 @@ int main(
                 cur = cur->m_next;
             }
 
-            TestContext::set(nullptr);
+            TestReporter::set(nullptr);
         }
 
-        context.outputSummary();
-
-        returnCode = context.didAllSucceed() ? 0 : 1;
+        reporter.outputSummary();
+        return reporter.didAllSucceed() ? SLANG_OK : SLANG_FAIL;
     }
+}
 
+int main(int argc, char** argv)
+{
+    const SlangResult res = innerMain(argc, argv);
 #ifdef _MSC_VER
     _CrtDumpMemoryLeaks();
 #endif
-
-    return returnCode; 
+    return SLANG_SUCCEEDED(res) ? 0 : 1;
 }