Test frame work improvements (#1452)

* Add -hide-ignored
Made API filter when enbled filter out non API tests.

* Add ability to set categories at file level.
Added wave, wave-mask and wave-active categories.

* Added -api-only flag.

* Don't synthesize tests from only CPU tests.

Co-authored-by: Tim Foley <tfoleyNV@users.noreply.github.com>

7 files changed, 197 insertions, 62 deletions

diff --git a/tools/slang-test/options.cpp b/tools/slang-test/options.cpp
index 4c8108f4a..27b759a0e 100644
--- a/tools/slang-test/options.cpp
+++ b/tools/slang-test/options.cpp
@@ -133,6 +133,14 @@ static bool _isSubCommand(const char* arg)
         {
             optionsOut->shouldBeVerbose = true;
         }
+        else if (strcmp(arg, "-hide-ignored") == 0)
+        {
+            optionsOut->hideIgnored = true;
+        }
+        else if (strcmp(arg, "-api-only") == 0)
+        {
+            optionsOut->apiOnly = true;
+        }
         else if (strcmp(arg, "-verbose-paths") == 0)
         {
             optionsOut->verbosePaths = true;
diff --git a/tools/slang-test/options.h b/tools/slang-test/options.h
index ffad16fdc..a8c2e3852 100644
--- a/tools/slang-test/options.h
+++ b/tools/slang-test/options.h
@@ -49,6 +49,12 @@ struct Options
     // generate extra output (notably: command lines we run)
     bool shouldBeVerbose = false;
 
+    // When true results from ignored tests are not shown
+    bool hideIgnored = false;
+
+    // When true only tests that use an api that matches the enabledApis flags will run
+    bool apiOnly = false;
+
     // Use verbose paths
     bool verbosePaths = false;
 
@@ -72,7 +78,8 @@ struct Options
     // Exclude test that match one these categories
     Slang::Dictionary<TestCategory*, TestCategory*> excludeCategories;
 
-    // By default we can test against all apis
+    // By default we can test against all apis. If is set to anything other than AllOf only tests that *require* the API
+    // will be run.
     Slang::RenderApiFlags enabledApis = Slang::RenderApiFlag::AllOf;
 
     // The subCommand to execute. Will be empty if there is no subCommand 
diff --git a/tools/slang-test/slang-test-main.cpp b/tools/slang-test/slang-test-main.cpp
index f85ade2ea..399aa52da 100644
--- a/tools/slang-test/slang-test-main.cpp
+++ b/tools/slang-test/slang-test-main.cpp
@@ -46,6 +46,21 @@ struct TestOptions
         Diagnostic,         ///< Diagnostic tests will always run (as form of failure is being tested)  
     };
 
+    void addCategory(TestCategory* category)
+    {
+        if (categories.indexOf(category) < 0)
+        {
+            categories.add(category);
+        }
+    }
+    void addCategories(TestCategory*const* inCategories, Index count)
+    {
+        for (Index i = 0; i < count; ++i)
+        {
+            addCategory(inCategories[i]);
+        }
+    }
+
     Type type = Type::Normal;
 
     String command;
@@ -191,32 +206,29 @@ String collectRestOfLine(char const** ioCursor)
     return getString(textBegin, textEnd);
 }
 
-
-static TestResult _gatherTestOptions(
-    TestCategorySet*    categorySet, 
-    char const**    ioCursor,
-    TestOptions&    outOptions)
+static SlangResult _parseCategories(TestCategorySet* categorySet, char const** ioCursor, TestOptions& out)
 {
     char const* cursor = *ioCursor;
 
     // Right after the `TEST` keyword, the user may specify
     // one or more categories for the test.
-    if(*cursor == '(')
+    if (*cursor == '(')
     {
         cursor++;
         // optional test category
         skipHorizontalSpace(&cursor);
         char const* categoryStart = cursor;
-        for(;;)
+        for (;;)
         {
-            switch( *cursor )
+            switch (*cursor)
             {
-            default:
-                cursor++;
-                continue;
-
-            case ',':
-            case ')':
+                default:
+                {
+                    cursor++;
+                    continue;
+                }
+                case ',':
+                case ')':
                 {
                     char const* categoryEnd = cursor;
                     cursor++;
@@ -224,38 +236,52 @@ static TestResult _gatherTestOptions(
                     auto categoryName = getString(categoryStart, categoryEnd);
                     TestCategory* category = categorySet->find(categoryName);
 
-                    if(!category)
+                    if (!category)
                     {
-                        return TestResult::Fail;
+                        // Failure if we don't find the category
+                        return SLANG_FAIL;
                     }
-                    
 
-                    outOptions.categories.add(category);
+                    out.addCategory(category);
 
-                    if( *categoryEnd == ',' )
+                    if (*categoryEnd == ',')
                     {
                         skipHorizontalSpace(&cursor);
                         categoryStart = cursor;
                         continue;
                     }
+
+                    *ioCursor = cursor;
+                    return SLANG_OK;
+                }
+                case 0: case '\r': case '\n':
+                {
+                    return SLANG_FAIL;
                 }
-                break;
-        
-            case 0: case '\r': case '\n':
-                return TestResult::Fail;
             }
 
             break;
         }
     }
 
-    // If no categories were specified, then add the default category
-    if(outOptions.categories.getCount() == 0)
+    *ioCursor = cursor;
+    return SLANG_OK;
+}
+
+
+static TestResult _gatherTestOptions(
+    TestCategorySet*    categorySet, 
+    char const**    ioCursor,
+    TestOptions&    outOptions)
+{
+    if (SLANG_FAILED(_parseCategories(categorySet, ioCursor, outOptions)))
     {
-        outOptions.categories.add(categorySet->defaultCategory);
+        return TestResult::Fail;
     }
 
-    if(*cursor == ':')
+    char const* cursor = *ioCursor;
+
+     if(*cursor == ':')
         cursor++;
     else
     {
@@ -335,6 +361,24 @@ static TestResult _gatherTestOptions(
     }
 }
 
+
+static RenderApiFlags _getRequiredRenderApisByCommand(const UnownedStringSlice& name);
+
+static void _combineOptions(
+    TestCategorySet* categorySet,
+    const TestOptions& fileOptions,
+    TestOptions& ioOptions)
+{
+    // And the file categories
+    ioOptions.addCategories(fileOptions.categories.getBuffer(), fileOptions.categories.getCount());
+
+    // If no categories were specified, then add the default category
+    if (ioOptions.categories.getCount() == 0)
+    {
+        ioOptions.categories.add(categorySet->defaultCategory);
+    }
+}
+
 // Try to read command-line options from the test file itself
 TestResult gatherTestsForFile(
     TestCategorySet*    categorySet,
@@ -354,6 +398,9 @@ TestResult gatherTestsForFile(
     // Walk through the lines of the file, looking for test commands
     char const* cursor = fileContents.begin();
 
+    // Options that are specified across all tests in the file.
+    TestOptions fileOptions;
+
     while(*cursor)
     {
         // We are at the start of a line of input.
@@ -378,11 +425,26 @@ TestResult gatherTestsForFile(
             testDetails.options.isEnabled = false;
         }
 
+        if (match(&cursor, "TEST_CATEGORY"))
+        {
+            if (SLANG_FAILED(_parseCategories(categorySet, &cursor, fileOptions)))
+            {
+                return TestResult::Fail;
+            }
+        }
+
         if(match(&cursor, "TEST"))
         { 
             if(_gatherTestOptions(categorySet, &cursor, testDetails.options) != TestResult::Pass)
                 return TestResult::Fail;
 
+            // See if the type of test needs certain APIs available
+            const RenderApiFlags testRequiredApis = _getRequiredRenderApisByCommand(testDetails.options.command.getUnownedSlice());
+            testDetails.requirements.addUsedRenderApis(testRequiredApis);
+
+            // Apply the file wide options
+            _combineOptions(categorySet, fileOptions, testDetails.options);
+
             testList->tests.add(testDetails);
         }
         else if (match(&cursor, "DIAGNOSTIC_TEST"))
@@ -390,6 +452,9 @@ TestResult gatherTestsForFile(
             if (_gatherTestOptions(categorySet, &cursor, testDetails.options) != TestResult::Pass)
                 return TestResult::Fail;
 
+            // Apply the file wide options
+            _combineOptions(categorySet, fileOptions, testDetails.options);
+
             // Mark that it is a diagnostic test
             testDetails.options.type = TestOptions::Type::Diagnostic;
             testList->tests.add(testDetails);
@@ -2611,33 +2676,52 @@ struct TestCommandInfo
 {
     char const*     name;
     TestCallback    callback;
+    RenderApiFlags  requiredRenderApiFlags;     ///< An RenderApi types that are needed to run the tests
 };
 
 static const TestCommandInfo s_testCommandInfos[] =
 {
-    { "SIMPLE",                                 &runSimpleTest},
-    { "SIMPLE_EX",                              &runSimpleTest},
-    { "REFLECTION",                             &runReflectionTest},
-    { "CPU_REFLECTION",                         &runReflectionTest},
-    { "COMMAND_LINE_SIMPLE",                    &runSimpleCompareCommandLineTest},
-    { "COMPARE_HLSL",                           &runDXBCComparisonTest},
-    { "COMPARE_DXIL",                           &runDXILComparisonTest},
-    { "COMPARE_HLSL_RENDER",                    &runHLSLRenderComparisonTest},
-    { "COMPARE_HLSL_CROSS_COMPILE_RENDER",      &runHLSLCrossCompileRenderComparisonTest},
-    { "COMPARE_HLSL_GLSL_RENDER",               &runHLSLAndGLSLRenderComparisonTest},
-    { "COMPARE_COMPUTE",                        &runSlangComputeComparisonTest},
-    { "COMPARE_COMPUTE_EX",                     &runSlangComputeComparisonTestEx},
-    { "HLSL_COMPUTE",                           &runHLSLComputeTest},
-    { "COMPARE_RENDER_COMPUTE",                 &runSlangRenderComputeComparisonTest},
-    { "COMPARE_GLSL",                           &runGLSLComparisonTest},
-    { "CROSS_COMPILE",                          &runCrossCompilerTest},
-    { "CPP_COMPILER_EXECUTE",                   &runCPPCompilerExecute},
-    { "CPP_COMPILER_SHARED_LIBRARY",            &runCPPCompilerSharedLibrary},
-    { "CPP_COMPILER_COMPILE",                   &runCPPCompilerCompile},
-    { "PERFORMANCE_PROFILE",                    &runPerformanceProfile},
-    { "COMPILE",                                &runCompile},
+    { "SIMPLE",                                 &runSimpleTest,                             0 },
+    { "SIMPLE_EX",                              &runSimpleTest,                             0 },
+    { "REFLECTION",                             &runReflectionTest,                         0 },
+    { "CPU_REFLECTION",                         &runReflectionTest,                         0 },
+    { "COMMAND_LINE_SIMPLE",                    &runSimpleCompareCommandLineTest,           0 },
+    { "COMPARE_HLSL",                           &runDXBCComparisonTest,                     0 },
+    { "COMPARE_DXIL",                           &runDXILComparisonTest,                     0 },
+    { "COMPARE_HLSL_RENDER",                    &runHLSLRenderComparisonTest,               0 },
+    { "COMPARE_HLSL_CROSS_COMPILE_RENDER",      &runHLSLCrossCompileRenderComparisonTest,   0 },
+    { "COMPARE_HLSL_GLSL_RENDER",               &runHLSLAndGLSLRenderComparisonTest,        0 },
+    { "COMPARE_COMPUTE",                        &runSlangComputeComparisonTest,             0 },
+    { "COMPARE_COMPUTE_EX",                     &runSlangComputeComparisonTestEx,           0 },
+    { "HLSL_COMPUTE",                           &runHLSLComputeTest,                        0 },
+    { "COMPARE_RENDER_COMPUTE",                 &runSlangRenderComputeComparisonTest,       0 },
+    { "COMPARE_GLSL",                           &runGLSLComparisonTest,                     0 },
+    { "CROSS_COMPILE",                          &runCrossCompilerTest,                      0 },
+    { "CPP_COMPILER_EXECUTE",                   &runCPPCompilerExecute,                     RenderApiFlag::CPU},
+    { "CPP_COMPILER_SHARED_LIBRARY",            &runCPPCompilerSharedLibrary,               RenderApiFlag::CPU},
+    { "CPP_COMPILER_COMPILE",                   &runCPPCompilerCompile,                     RenderApiFlag::CPU},
+    { "PERFORMANCE_PROFILE",                    &runPerformanceProfile,                     0 },
+    { "COMPILE",                                &runCompile,                                0 },
 };
 
+const TestCommandInfo* _findTestCommandInfoByCommand(const UnownedStringSlice& name)
+{
+    for (const auto& command : s_testCommandInfos)
+    {
+        if (name == command.name)
+        {
+            return &command;
+        }
+    }
+    return nullptr;
+}
+
+static RenderApiFlags _getRequiredRenderApisByCommand(const UnownedStringSlice& name)
+{
+    auto info = _findTestCommandInfoByCommand(name);
+    return info ? info->requiredRenderApiFlags : 0;
+}
+
 TestResult runTest(
     TestContext*        context, 
     String const&       filePath,
@@ -2654,18 +2738,17 @@ TestResult runTest(
 
     const SpawnType defaultSpawnType = context->options.useExes ? SpawnType::UseExe : SpawnType::UseSharedLibrary;
 
-    for( const auto& command : s_testCommandInfos)
-    {
-        if(testOptions.command != command.name)
-            continue;
+    auto testInfo = _findTestCommandInfoByCommand(testOptions.command.getUnownedSlice());
 
+    if (testInfo)
+    {
         TestInput testInput;
         testInput.filePath = filePath;
         testInput.outputStem = outputStem;
         testInput.testOptions = &testOptions;
         testInput.spawnType = defaultSpawnType;
 
-        return command.callback(context, testInput);
+        return testInfo->callback(context, testInput);
     }
 
     // No actual test runner found!
@@ -2771,7 +2854,9 @@ static void _calcSynthesizedTests(TestContext* context, RenderApiType synthRende
         {
             // TODO(JS): Arguably we should synthesize from explicit tests. In principal we can remove the explicit api apply another
             // although that may not always work.
+            // If it doesn't use any render API or only uses CPU, we don't synthesize
             if (requirements.usedRenderApiFlags == 0 ||
+                requirements.usedRenderApiFlags == RenderApiFlag::CPU ||
                 requirements.explicitRenderApi != RenderApiType::Unknown)
             {
                 continue;
@@ -2829,8 +2914,11 @@ static bool _canIgnore(TestContext* context, const TestDetails& details)
 
     const auto& requirements = details.requirements;
 
-    // Work out what render api flags are available
-    const RenderApiFlags availableRenderApiFlags = requirements.usedRenderApiFlags ? _getAvailableRenderApiFlags(context) : 0;
+    // Check if it's possible in principal to run this test with the render api flags used by this test
+    if (!context->canRunTestWithRenderApiFlags(requirements.usedRenderApiFlags))
+    {
+        return true;
+    }
 
     // Are all the required backends available?
     if (((requirements.usedBackendFlags & context->availableBackendFlags) != requirements.usedBackendFlags))
@@ -2838,18 +2926,15 @@ static bool _canIgnore(TestContext* context, const TestDetails& details)
         return true;
     }
 
+    // Work out what render api flags are actually available, lazily
+    const RenderApiFlags availableRenderApiFlags = requirements.usedRenderApiFlags ? _getAvailableRenderApiFlags(context) : 0;
+
     // Are all the required rendering apis available?
     if ((requirements.usedRenderApiFlags & availableRenderApiFlags) != requirements.usedRenderApiFlags)
     {
         return true;
     }
 
-    // Are the required rendering APIs enabled from the -api command line switch
-    if ((requirements.usedRenderApiFlags & context->options.enabledApis) != requirements.usedRenderApiFlags)
-    {
-        return true;
-    }
-
     return false;
 }
 
@@ -3113,6 +3198,10 @@ SlangResult innerMain(int argc, char** argv)
     auto cudaTestCategory = categorySet.add("cuda", fullTestCategory);
     auto optixTestCategory = categorySet.add("optix", cudaTestCategory);
 
+    auto waveTestCategory = categorySet.add("wave", fullTestCategory);
+    auto waveMaskCategory = categorySet.add("wave-mask", waveTestCategory);
+    auto waveActiveCategory = categorySet.add("wave-active", waveTestCategory);
+
     auto compatibilityIssueCategory = categorySet.add("compatibility-issue", fullTestCategory);
         
 #if SLANG_WINDOWS_FAMILY
@@ -3237,6 +3326,7 @@ SlangResult innerMain(int argc, char** argv)
 
         reporter.m_dumpOutputOnFailure = options.dumpOutputOnFailure;
         reporter.m_isVerbose = options.shouldBeVerbose;
+        reporter.m_hideIgnored = options.hideIgnored;
 
         {
             TestReporter::SuiteScope suiteScope(&reporter, "tests");
@@ -3248,6 +3338,9 @@ SlangResult innerMain(int argc, char** argv)
 
         // Run the unit tests (these are internal C++ tests - not specified via files in a directory) 
         // They are registered with SLANG_UNIT_TEST macro
+        //
+        // 
+        if (context.canRunUnitTests())
         {
             TestReporter::SuiteScope suiteScope(&reporter, "unit tests");
             TestReporter::set(&reporter);
diff --git a/tools/slang-test/test-context.cpp b/tools/slang-test/test-context.cpp
index 0a0931596..8fa5cb345 100644
--- a/tools/slang-test/test-context.cpp
+++ b/tools/slang-test/test-context.cpp
@@ -110,3 +110,15 @@ Slang::DownstreamCompiler* TestContext::getDefaultCompiler(SlangSourceLanguage s
     return set ? set->getDefaultCompiler(sourceLanguage) : nullptr;
 }
 
+bool TestContext::canRunTestWithRenderApiFlags(Slang::RenderApiFlags requiredFlags)
+{
+    // If only allow tests that use API - then the requiredFlags must be 0
+    if (options.apiOnly && requiredFlags == 0)
+    {
+        return false;
+    }
+    // Are the required rendering APIs enabled from the -api command line switch
+    return (requiredFlags & options.enabledApis) == requiredFlags;
+}
+
+
diff --git a/tools/slang-test/test-context.h b/tools/slang-test/test-context.h
index e10b8d60b..ab0a00b40 100644
--- a/tools/slang-test/test-context.h
+++ b/tools/slang-test/test-context.h
@@ -95,6 +95,15 @@ class TestContext
         /// If set, then tests are executed
     bool isExecuting() const { return testRequirements == nullptr; }
 
+        /// True if a render API filter is enabled
+    bool isRenderApiFilterEnabled() const { return options.enabledApis != Slang::RenderApiFlag::AllOf && options.enabledApis != 0; }
+
+        /// True if a test with the requiredFlags can in principal run (it may not be possible if the API is not available though)
+    bool canRunTestWithRenderApiFlags(Slang::RenderApiFlags requiredFlags);
+
+        /// True if can run unit tests
+    bool canRunUnitTests() const { return options.apiOnly == false; }
+
         /// Get compiler set
     Slang::DownstreamCompilerSet* getCompilerSet();
     Slang::DownstreamCompiler* getDefaultCompiler(SlangSourceLanguage sourceLanguage);
diff --git a/tools/slang-test/test-reporter.cpp b/tools/slang-test/test-reporter.cpp
index 9dd34739b..fd9748b28 100644
--- a/tools/slang-test/test-reporter.cpp
+++ b/tools/slang-test/test-reporter.cpp
@@ -286,6 +286,11 @@ static void _appendTime(double timeInSec, StringBuilder& out)
 
 void TestReporter::_addResult(const TestInfo& info)
 {
+    if (info.testResult == TestResult::Ignored && m_hideIgnored)
+    {
+        return;
+    }
+
     m_totalTestCount++;
 
     switch (info.testResult)
diff --git a/tools/slang-test/test-reporter.h b/tools/slang-test/test-reporter.h
index 6a862afef..1ebce81cd 100644
--- a/tools/slang-test/test-reporter.h
+++ b/tools/slang-test/test-reporter.h
@@ -166,6 +166,7 @@ class TestReporter
     TestOutputMode m_outputMode = TestOutputMode::Default;
     bool m_dumpOutputOnFailure;
     bool m_isVerbose = false;
+    bool m_hideIgnored = false;
 
 protected: