AppVeyor: Run tests as part of AppVeyor builds

This includes a bunch of related changes:

- `slang-test`
  - Add a notion of an "output mode" that specifies whether we output to console (the default), or invoke the apprpriate AppVeyor command to update test status
  - Add a notion of test categories, so that tests can be tagged with categories, and then we can invoke only those tets in a given category, or choose to *exclude* tests with specific categories
  - Allow the `OSProcessSpawner` to look up an executable by "path" (meaning a full path is expected) or by "name" (meaning it should be allowed to look in the current directory, `PATH` environment variable, etc.). This was important to make sure that I can run `appveyor` without having to know its absolute path.

- AppVeyor configuration
  - Change badge to reflect new build account for organization (rather than a single-user account)
  - Remove attempt to set AppVeyor build version in a clever way, since it breaks links from GitHub to AppVeyor
  - Change order or configurations in the build matrix to front-load the Release build (which has the main tests)
  - Turn on `fast_finish` flag so we don't have to wait as long for failed builds
  - Turn on `parallel` builds
  - Set `verbosity: minimal` to avoid getting build spew about Xamarin stuff I'm not using
  - Add custom `test_script` to invoke `test.bat`
    - Sets the test category based on teh build configuration, so we don't run the full test suite on every input.

- `test.bat`
  - Allow for `-platform` and `-configuration` arguments
  - Rewrute a platform of `Win32` over to `x86` to match how the output directories are named
  - Futz around with how the directories are being passed along to work around annoying `.bat` file quoting behavior (I still don't get how batch files work)

- Tests
  - Mark a bunch of tests as `smoke` tests
  - Mark the relevant tests as `render` tests
    (these get filtered out for AppVeyor builds)

3 files changed, 400 insertions, 35 deletions

diff --git a/tools/slang-test/main.cpp b/tools/slang-test/main.cpp
index ed419f946..4940228fe 100644
--- a/tools/slang-test/main.cpp
+++ b/tools/slang-test/main.cpp
@@ -22,6 +22,20 @@ using namespace CoreLib::IO;
 #include <stdlib.h>
 #include <stdarg.h>
 
+enum OutputMode
+{
+    // Default mode is to write test results to the console
+    kOutputMode_Default = 0,
+
+    // When running under AppVeyor contiuous integration, we
+    // need to output test results in a way that the AppVeyor
+    // environment can pick up and display.
+    kOutputMode_AppVeyor,
+};
+
+struct TestCategory;
+TestCategory* findTestCategory(String const& name);
+
 struct Options
 {
     char const* appName = "slang-test";
@@ -37,6 +51,15 @@ struct Options
 
     // force generation of baselines for HLSL tests
     bool generateHLSLBaselines = false;
+
+    // kind of output to generate
+    OutputMode outputMode = kOutputMode_Default;
+
+    // Only run tests that match one of the given categories
+    Dictionary<TestCategory*, TestCategory*> includeCategories;
+
+    // Exclude test taht match one these categories
+    Dictionary<TestCategory*, TestCategory*> excludeCategories;
 };
 Options options;
 
@@ -74,7 +97,7 @@ void parseOptions(int* argc, char** argv)
             break;
         }
 
-        if( strcmp(arg, "--bindir") == 0 )
+        if( strcmp(arg, "-bindir") == 0 )
         {
             if( argCursor == argEnd )
             {
@@ -99,6 +122,56 @@ void parseOptions(int* argc, char** argv)
         {
             // Assumed to be handle by .bat file that called us
         }
+        else if( strcmp(arg, "-configuration") == 0 )
+        {
+            if( argCursor == argEnd )
+            {
+                fprintf(stderr, "error: expected operand for '%s'\n", arg);
+                exit(1);
+            }
+            argCursor++;
+            // Assumed to be handle by .bat file that called us
+        }
+        else if( strcmp(arg, "-platform") == 0 )
+        {
+            if( argCursor == argEnd )
+            {
+                fprintf(stderr, "error: expected operand for '%s'\n", arg);
+                exit(1);
+            }
+            argCursor++;
+            // Assumed to be handle by .bat file that called us
+        }
+        else if( strcmp(arg, "-appveyor") == 0 )
+        {
+            options.outputMode = kOutputMode_AppVeyor;
+        }
+        else if( strcmp(arg, "-category") == 0 )
+        {
+            if( argCursor == argEnd )
+            {
+                fprintf(stderr, "error: expected operand for '%s'\n", arg);
+                exit(1);
+            }
+            auto category = findTestCategory(*argCursor++);
+            if(category)
+            {
+                options.includeCategories.Add(category, category);
+            }
+        }
+        else if( strcmp(arg, "-exclude") == 0 )
+        {
+            if( argCursor == argEnd )
+            {
+                fprintf(stderr, "error: expected operand for '%s'\n", arg);
+                exit(1);
+            }
+            auto category = findTestCategory(*argCursor++);
+            if(category)
+            {
+                options.excludeCategories.Add(category, category);
+            }
+        }
         else
         {
             fprintf(stderr, "unknown option '%s'\n", arg);
@@ -231,11 +304,53 @@ String collectRestOfLine(char const** ioCursor)
     return getString(textBegin, textEnd);
 }
 
+// A category that a test can be tagged with
+struct TestCategory
+{
+    // The name of the category, from the user perspective
+    String name;
+
+    // The logical "super-category" of this category
+    TestCategory* parent;
+
+    // A list of categories that we explcitly want to exclude
+    List<TestCategory*> prohibitedCategories;
+};
+
+Dictionary<String, TestCategory*> testCategories;
+TestCategory* defaultTestCategory;
+
+TestCategory* addTestCategory(String const& name, TestCategory* parent)
+{
+    TestCategory* category = new TestCategory();
+    category->name = name;
+
+    category->parent = parent;
+
+    testCategories.Add(name, category);
+
+    return category;
+}
+
+TestCategory* findTestCategory(String const& name)
+{
+    TestCategory* category = nullptr;
+    if( !testCategories.TryGetValue(name, category) )
+    {
+        error("unknown test category name '%s'\n", name.Buffer());
+        return nullptr;
+    }
+    return category;
+}
+
 // Optiosn for a particular test
 struct TestOptions
 {
     String command;
     List<String> args;
+
+    // The categories that this test was assigned to
+    List<TestCategory*> categories;
 };
 
 // Information on tests to run for a particular file
@@ -250,7 +365,71 @@ TestResult gatherTestOptions(
 {
     char const* cursor = *ioCursor;
 
-    // Start by scanning for the sub-command name:
+    TestOptions testOptions;
+
+    // Right after the `TEST` keyword, the user may specify
+    // one or more categories for the test.
+    if(*cursor == '(')
+    {
+        cursor++;
+        // optional test category
+        skipHorizontalSpace(&cursor);
+        char const* categoryStart = cursor;
+        for(;;)
+        {
+            switch( *cursor )
+            {
+            default:
+                cursor++;
+                continue;
+
+            case ',':
+            case ')':
+                {
+                    char const* categoryEnd = cursor;
+                    cursor++;
+
+                    auto categoryName = getString(categoryStart, categoryEnd);
+                    TestCategory* category = findTestCategory(categoryName);
+
+                    if(!category)
+                    {
+                        return kTestResult_Fail;
+                    }
+
+                    testOptions.categories.Add(category);
+
+                    if( *categoryEnd == ',' )
+                    {
+                        skipHorizontalSpace(&cursor);
+                        categoryStart = cursor;
+                        continue;
+                    }
+                }
+                break;
+        
+            case 0: case '\r': case '\n':
+                return kTestResult_Fail;
+            }
+
+            break;
+        }
+    }
+
+    // If no categories were specified, then add the default category
+    if(testOptions.categories.Count() == 0)
+    {
+        testOptions.categories.Add(defaultTestCategory);
+    }
+
+    if(*cursor == ':')
+        cursor++;
+    else
+    {
+        return kTestResult_Fail;
+    }
+
+    // Next scan for a sub-command name
     char const* commandStart = cursor;
     for(;;)
     {
@@ -270,12 +449,16 @@ TestResult gatherTestOptions(
         break;
     }
     char const* commandEnd = cursor;
-    if(*cursor == ':')
-        cursor++;
 
-    TestOptions testOptions;
     testOptions.command = getString(commandStart, commandEnd);
 
+    if(*cursor == ':')
+        cursor++;
+    else
+    {
+        return kTestResult_Fail;
+    }
+
     // Now scan for arguments. For now we just assume that
     // any whitespace separation indicates a new argument
     // (we don't support quoting)
@@ -344,14 +527,14 @@ TestResult gatherTestsForFile(
         skipHorizontalSpace(&cursor);
 
         // Look for a pattern that matches what we want
-        if(match(&cursor, "//TEST:"))
+        if(match(&cursor, "//TEST_IGNORE_FILE"))
         {
-            if(gatherTestOptions(&cursor, testList) != kTestResult_Pass)
-                return kTestResult_Fail;
+            return kTestResult_Ignored;
         }
-        else if(match(&cursor, "//TEST_IGNORE_FILE"))
+        else if(match(&cursor, "//TEST"))
         {
-            return kTestResult_Ignored;
+            if(gatherTestOptions(&cursor, testList) != kTestResult_Pass)
+                return kTestResult_Fail;
         }
         else
         {
@@ -372,6 +555,7 @@ OSError spawnAndWait(String	testPath, OSProcessSpawner& spawner)
     OSError err = spawner.spawnAndWaitForCompletion();
     if (err != kOSError_None)
     {
+//        fprintf(stderr, "failed to run test '%S'\n", testPath.ToWString());
         error("failed to run test '%S'", testPath.ToWString());
     }
     return err;
@@ -426,7 +610,7 @@ TestResult runSimpleTest(TestInput& input)
 
     OSProcessSpawner spawner;
 
-    spawner.pushExecutableName(String(options.binDir) + "slangc.exe");
+    spawner.pushExecutablePath(String(options.binDir) + "slangc.exe");
     spawner.pushArgument(filePath999);
 
     for( auto arg : input.testOptions->args )
@@ -485,7 +669,7 @@ TestResult generateHLSLBaseline(TestInput& input)
     auto outputStem = input.outputStem;
 
     OSProcessSpawner spawner;
-    spawner.pushExecutableName(String(options.binDir) + "slangc.exe");
+    spawner.pushExecutablePath(String(options.binDir) + "slangc.exe");
     spawner.pushArgument(filePath999);
 
     for( auto arg : input.testOptions->args )
@@ -531,7 +715,7 @@ TestResult runHLSLComparisonTest(TestInput& input)
 
     OSProcessSpawner spawner;
 
-    spawner.pushExecutableName(String(options.binDir) + "slangc.exe");
+    spawner.pushExecutablePath(String(options.binDir) + "slangc.exe");
     spawner.pushArgument(filePath999);
 
     for( auto arg : input.testOptions->args )
@@ -621,7 +805,7 @@ TestResult doGLSLComparisonTestRun(
 
     OSProcessSpawner spawner;
 
-    spawner.pushExecutableName(String(options.binDir) + "slangc.exe");
+    spawner.pushExecutablePath(String(options.binDir) + "slangc.exe");
     spawner.pushArgument(filePath999);
 
     if( langDefine )
@@ -710,7 +894,7 @@ TestResult doRenderComparisonTestRun(TestInput& input, char const* langOption, c
 
     OSProcessSpawner spawner;
 
-    spawner.pushExecutableName(String(options.binDir) + "render-test.exe");
+    spawner.pushExecutablePath(String(options.binDir) + "render-test.exe");
     spawner.pushArgument(filePath999);
 
     for( auto arg : input.testOptions->args )
@@ -913,6 +1097,150 @@ struct TestContext
     int failedTestCount;
 };
 
+// deal with the fallout of a test having completed, whether
+// passed or failed or who-knows-what.
+void handleTestResult(
+    TestContext*    context,
+    String const&   testName,
+    TestResult      testResult)
+{
+    switch( testResult )
+    {
+    case kTestResult_Fail:
+        context->failedTestCount++;
+        break;
+
+    case kTestResult_Pass:
+        context->passedTestCount++;
+        break;
+
+    case kTestResult_Ignored:
+        // Note that we don't currently add ignored tests into
+        // the totals, which is kind of inaccurate.
+        break;
+
+    default:
+        assert(!"unexpected");
+        break;
+    }
+
+//    printf("OUTPUT_MODE: %d\n", options.outputMode);
+    switch( options.outputMode )
+    {
+    case kOutputMode_Default:
+        {
+            char const* resultString = "UNEXPECTED";
+            switch( testResult )
+            {
+            case kTestResult_Fail:      resultString = "FAILED";  break;
+            case kTestResult_Pass:      resultString = "passed";  break;
+            case kTestResult_Ignored:   resultString = "ignored"; break;
+            default:
+                assert(!"unexpected");
+                break;
+            }
+
+            printf("%s test: '%S'\n", resultString, testName.ToWString());
+        }
+        break;
+
+    case kOutputMode_AppVeyor:
+        {
+            char const* resultString = "None";
+            switch( testResult )
+            {
+            case kTestResult_Fail:      resultString = "Failed";  break;
+            case kTestResult_Pass:      resultString = "Passed";  break;
+            case kTestResult_Ignored:   resultString = "Ignored"; break;
+            default:
+                assert(!"unexpected");
+                break;
+            }
+
+
+            OSProcessSpawner spawner;
+            spawner.pushExecutableName("appveyor");
+            spawner.pushArgument("AddTest");
+            spawner.pushArgument(testName);
+            spawner.pushArgument("-FileName");
+            // TODO: this isn't actually a file name in all cases
+            spawner.pushArgument(testName);
+            spawner.pushArgument("-Framework");
+            spawner.pushArgument("slang-test");
+            spawner.pushArgument("-Outcome");
+            spawner.pushArgument(resultString);
+
+            auto err = spawner.spawnAndWaitForCompletion();
+
+            if( err != kOSError_None )
+            {
+                error("failed to add appveyor test results for '%S'\n", testName.ToWString());
+
+#if 0
+                fprintf(stderr, "[%d] TEST RESULT: %s {%d} {%s} {%s}\n", err, spawner.commandLine_.Buffer(),
+                    spawner.getResultCode(),
+                    spawner.getStandardOutput().begin(),
+                    spawner.getStandardError().begin());
+#endif
+            }
+        }
+        break;
+
+    default:
+        assert(!"unexpected");
+        break;
+    }
+}
+
+bool testCategoryMatches(
+    TestCategory*   sub,
+    TestCategory*   sup)
+{
+    auto ss = sub;
+    while(ss)
+    {
+        if(ss == sup)
+            return true;
+
+        ss = ss->parent;
+    }
+    return false;
+}
+
+bool testCategoryMatches(
+    TestCategory*                           categoryToMatch,
+    Dictionary<TestCategory*, TestCategory*> categorySet)
+{
+    for( auto item : categorySet )
+    {
+        if(testCategoryMatches(categoryToMatch, item.Value))
+            return true;
+    }
+    return false;
+}
+
+bool testPassesCategoryMask(
+    TestContext*        context,
+    TestOptions const&  test)
+{
+    // Don't include a test we should filter out
+    for( auto testCategory : test.categories )
+    {
+        if(testCategoryMatches(testCategory, options.excludeCategories))
+            return false;
+    }
+
+    // Otherwise inclue any test the user asked for
+    for( auto testCategory : test.categories )
+    {
+        if(testCategoryMatches(testCategory, options.includeCategories))
+            return true;
+    }
+
+    // skip by default
+    return false;
+}
+
 void runTestsOnFile(
     TestContext*    context,
     String          filePath)
@@ -929,10 +1257,7 @@ void runTestsOnFile(
     // Note cases where a test file exists, but we found nothing to run
     if( testList.tests.Count() == 0 )
     {
-        context->totalTestCount++;
-        context->failedTestCount++;
-
-        printf("FAILED test: '%S' (no test commands found)\n", filePath.ToWString());
+        handleTestResult(context, filePath, kTestResult_Ignored);
         return;
     }
 
@@ -940,9 +1265,16 @@ void runTestsOnFile(
     int subTestCount = 0;
     for( auto& tt : testList.tests )
     {
+        int subTestIndex = subTestCount++;
+
+        // Check that the test passes our current category mask
+        if(!testPassesCategoryMask(context, tt))
+        {
+            continue;
+        }
+
         context->totalTestCount++;
 
-        int subTestIndex = subTestCount++;
 
         String outputStem = filePath;
         if(subTestIndex != 0)
@@ -951,22 +1283,9 @@ void runTestsOnFile(
         }
 
         TestResult result = runTest(filePath, outputStem, tt, testList);
-        if(result == kTestResult_Ignored)
-            return;
 
-        if (result == kTestResult_Pass)
-        {
-            printf("passed");
-            context->passedTestCount++;
-        }
-        else
-        {
-            printf("FAILED");
-            context->failedTestCount++;
-        }
+        handleTestResult(context, outputStem, result);
 
-        printf(" test: '%S'", outputStem.ToWString());
-        printf("\n");
     }
 }
 
@@ -1038,8 +1357,37 @@ int main(
     int		argc,
     char**	argv)
 {
+    // Set up our test categories here
+
+    auto fullTestCategory = addTestCategory("full", nullptr);
+
+    auto quickTestCategory = addTestCategory("quick", fullTestCategory);
+
+    auto smokeTestCategory = addTestCategory("smoke", quickTestCategory);
+
+    auto renderTestCategory = addTestCategory("render", fullTestCategory);
+
+    // An un-categorized test will always belong to the `full` category
+    defaultTestCategory = fullTestCategory;
+
+    //
+
+
     parseOptions(&argc, argv);
 
+    if( options.includeCategories.Count() == 0 )
+    {
+        options.includeCategories.Add(fullTestCategory, fullTestCategory);
+    }
+
+    // Exclude rendering tests when building under AppVeyor.
+    //
+    // TODO: this is very ad hoc, and we should do something cleaner.
+    if( options.outputMode == kOutputMode_AppVeyor )
+    {
+        options.excludeCategories.Add(renderTestCategory, renderTestCategory);
+    }
+
     TestContext context = { 0 };
 
     // Enumerate test files according to policy
diff --git a/tools/slang-test/os.cpp b/tools/slang-test/os.cpp
index 8f5172ae6..4bc3d8e6d 100644
--- a/tools/slang-test/os.cpp
+++ b/tools/slang-test/os.cpp
@@ -205,6 +205,15 @@ void OSProcessSpawner::pushExecutableName(
 {
     executableName_ = executableName;
     commandLine_.Append(executableName);
+    isExecutablePath_ = false;
+}
+
+void OSProcessSpawner::pushExecutablePath(
+    CoreLib::Basic::String executablePath)
+{
+    executableName_ = executablePath;
+    commandLine_.Append(executablePath);
+    isExecutablePath_ = true;
 }
 
 void OSProcessSpawner::pushArgument(
@@ -305,7 +314,7 @@ OSError OSProcessSpawner::spawnAndWaitForCompletion()
 
     // `CreateProcess` requires write access to this, for some reason...
     BOOL success = CreateProcessW(
-        executableName_.ToWString(),
+        isExecutablePath_ ? executableName_.ToWString() : nullptr,
         (LPWSTR)commandLine_.ToString().ToWString(),
         nullptr,
         nullptr,
diff --git a/tools/slang-test/os.h b/tools/slang-test/os.h
index 2996001e7..5471e7e1a 100644
--- a/tools/slang-test/os.h
+++ b/tools/slang-test/os.h
@@ -129,6 +129,11 @@ struct OSProcessSpawner
     void pushExecutableName(
         CoreLib::Basic::String executableName);
 
+    // Set the executable name for the process to be spawned.
+    // Note: this call must be made before any arguments are pushed.
+    void pushExecutablePath(
+        CoreLib::Basic::String executablePath);
+
     // Append an argument for the process to be spawned.
     void pushArgument(
         CoreLib::Basic::String argument);
@@ -155,6 +160,9 @@ struct OSProcessSpawner
 #ifdef WIN32
     CoreLib::Basic::String executableName_;
     CoreLib::Basic::StringBuilder commandLine_;
+
+    // Is the executable specified by path, rather than just by name?
+    bool isExecutablePath_;
 #else
 #endif
 };