Feature/teamcity output (#715)

* First pass support for TeamCity compatible output.

* Remove reset of counters on starting suite - so summary is over all suites.

3 files changed, 194 insertions, 16 deletions

diff --git a/tools/slang-test/main.cpp b/tools/slang-test/main.cpp
index 22749c206..30fa709d7 100644
--- a/tools/slang-test/main.cpp
+++ b/tools/slang-test/main.cpp
@@ -222,6 +222,10 @@ Result parseOptions(int* argc, char** argv)
         {
             g_options.outputMode = TestOutputMode::XUnit2;
         }
+        else if (strcmp(arg, "-teamcity") == 0)
+        {
+            g_options.outputMode = TestOutputMode::TeamCity;
+        }
         else if( strcmp(arg, "-category") == 0 )
         {
             if( argCursor == argEnd )
@@ -1670,7 +1674,7 @@ TestResult runTest(
         testInput.testList = &testList;
 
         {
-            TestContext::Scope scope(context, outputStem);
+            TestContext::TestScope scope(context, outputStem);
 
             TestResult testResult = ii->callback(context, testInput);
             context->addResult(testResult); 
@@ -1957,18 +1961,21 @@ int main(
 
     // Setup the context 
     TestContext context(g_options.outputMode);
-   
     context.m_dumpOutputOnFailure = g_options.dumpOutputOnFailure;
     context.m_isVerbose = g_options.shouldBeVerbose;
- 
-    // Enumerate test files according to policy
-    // TODO: add more directories to this list
-    // TODO: allow for a command-line argument to select a particular directory
-    runTestsInDirectory(&context, "tests/");
-    
+
+    { 
+        TestContext::SuiteScope suiteScope(&context, "tests");
+        // Enumerate test files according to policy
+        // TODO: add more directories to this list
+        // TODO: allow for a command-line argument to select a particular directory
+        runTestsInDirectory(&context, "tests/");
+    }
+
     // Run the unit tests (these are internal C++ tests - not specified via files in a directory) 
     // They are registered with SLANG_UNIT_TEST macro
     {
+        TestContext::SuiteScope suiteScope(&context, "unit tests");
         TestContext::set(&context);
 
         // Run the unit tests
@@ -2003,7 +2010,6 @@ int main(
 
         TestContext::set(nullptr);
     }
-        
 
     context.outputSummary();
 
diff --git a/tools/slang-test/test-context.cpp b/tools/slang-test/test-context.cpp
index 822a5953c..158258873 100644
--- a/tools/slang-test/test-context.cpp
+++ b/tools/slang-test/test-context.cpp
@@ -99,7 +99,10 @@ bool TestContext::canWriteStdError() const
 
 void TestContext::startTest(const String& testName)
 {
+    // Must be in a suite
+    assert(m_suiteStack.Count());
     assert(!m_inTest);
+
     m_inTest = true;
 
     m_numCurrentResults = 0;
@@ -112,6 +115,7 @@ void TestContext::startTest(const String& testName)
 
 void TestContext::endTest()
 {
+    assert(m_suiteStack.Count());
     assert(m_inTest);
 
     m_currentInfo.message = m_currentMessage;
@@ -196,6 +200,51 @@ void TestContext::dumpOutputDifference(const String& expectedOutput, const Strin
     message(TestMessageType::TestFailure, builder);
 }
 
+static char _getTeamCityEscapeChar(char c)
+{
+    switch (c)
+    {
+        case '|':   return '|';
+        case '\'':  return '\''; 
+        case '\n':  return 'n';
+        case '\r':  return 'r'; 
+        case '[':   return '[';
+        case ']':   return ']';
+        default:    return 0;
+    }
+}
+
+static void _appendEncodedTeamCityString(const UnownedStringSlice& in, StringBuilder& builder)
+{
+    const char* start = in.begin();
+    const char* cur = start;
+    const char* end = in.end();
+    
+    for (const char* cur = start; cur < end; cur++)
+    {
+        const char c = *cur;
+        const char escapeChar = _getTeamCityEscapeChar(c);
+        if (escapeChar)
+        {
+            // Flush
+            if (cur > start)
+            {
+                builder.Append(start, UInt(cur - start));
+            }
+
+            builder.Append('|');
+            builder.Append(escapeChar);
+            start = cur + 1;
+        }
+    }    
+
+    // Flush the end
+    if (end > start)
+    {
+        builder.Append(start, UInt(end - start));
+    }
+}
+
 void TestContext::_addResult(const TestInfo& info)
 {
     m_totalTestCount++;
@@ -239,6 +288,63 @@ void TestContext::_addResult(const TestInfo& info)
             printf("%s test: '%S'\n", resultString, info.name.ToWString().begin());
             break;
         }
+        case TestOutputMode::TeamCity:
+        {
+            StringBuilder escapedTestName;
+            _appendEncodedTeamCityString(info.name.getUnownedSlice(), escapedTestName);
+
+            printf("##teamcity[testStarted name='%s']\n", escapedTestName.begin());
+             
+            switch (info.testResult)
+            {
+                case TestResult::Fail:      
+                {
+                    if (info.message.Length())
+                    {
+                        StringBuilder escapedMessage;
+                        _appendEncodedTeamCityString(info.message.getUnownedSlice(), escapedMessage);            
+                        printf("##teamcity[testFailed name='%s' message='%s']\n", escapedTestName.begin(), escapedMessage.begin());
+                    }
+                    else
+                    {
+                        printf("##teamcity[testFailed name='%s']\n", escapedTestName.begin());
+                    }
+                    break;
+                }
+                case TestResult::Pass:     
+                {
+                    if (info.message.Length())
+                    {
+                        StringBuilder escapedMessage;
+                        _appendEncodedTeamCityString(info.message.getUnownedSlice(), escapedMessage);
+                        printf("##teamcity[testStdOut name='%s' out='%s']\n", escapedTestName.begin(), escapedMessage.begin());
+                    }
+                    break;
+                }
+                case TestResult::Ignored:  
+                {
+                    if (info.message.Length())
+                    {
+                        StringBuilder escapedMessage;
+                        _appendEncodedTeamCityString(info.message.getUnownedSlice(), escapedMessage);
+
+                        printf("##teamcity[testIgnored name='%s' message='%s']\n", escapedTestName.begin(), escapedMessage.begin());
+                    }
+                    else
+                    {
+                        printf("##teamcity[testIgnored name='%s']\n", escapedTestName.begin());
+                    }
+                    break;
+                }
+                default:
+                    assert(!"unexpected");
+                    break;
+            }
+
+            printf("##teamcity[testFinished name='%s']\n", escapedTestName.begin());
+            fflush(stdout);
+            break;
+        }
         case TestOutputMode::XUnit2:
         case TestOutputMode::XUnit:
         {
@@ -397,6 +503,7 @@ void TestContext::outputSummary()
             }
             break;
         }
+        
         case TestOutputMode::XUnit:
         {
             // xUnit 1.0 format  
@@ -451,5 +558,47 @@ void TestContext::outputSummary()
             assert("Not currently supported");
             break;
         }
+        case TestOutputMode::TeamCity:
+        {
+            // Don't output a summary
+            break;
+        }
+    }
+}
+
+void TestContext::startSuite(const String& name)
+{
+    m_suiteStack.Add(name);
+
+    switch (m_outputMode)
+    {
+        case TestOutputMode::TeamCity:
+        {
+            StringBuilder escapedSuiteName;
+            _appendEncodedTeamCityString(name.getUnownedSlice(), escapedSuiteName);
+            printf("##teamcity[testSuiteStarted name='%s']\n", escapedSuiteName.begin());
+            break;
+        }
+        default: break;
+    }
+}
+
+void TestContext::endSuite()
+{
+    assert(m_suiteStack.Count());
+
+    switch (m_outputMode)
+    {
+        case TestOutputMode::TeamCity:
+        {
+            const String& name = m_suiteStack.Last();
+            StringBuilder escapedSuiteName;
+            _appendEncodedTeamCityString(name.getUnownedSlice(), escapedSuiteName);
+            printf("##teamcity[testSuiteFinished name='%s']\n", escapedSuiteName.begin());
+            break;
+        }
+        default: break;
     }
+    
+    m_suiteStack.RemoveLast();
 }
\ No newline at end of file
diff --git a/tools/slang-test/test-context.h b/tools/slang-test/test-context.h
index 853bd3015..94f8c689f 100644
--- a/tools/slang-test/test-context.h
+++ b/tools/slang-test/test-context.h
@@ -32,6 +32,7 @@ enum class TestOutputMode
     Travis,        ///< We currently don't specialize for Travis, but maybe we should.
     XUnit,         ///< xUnit original format  https://nose.readthedocs.io/en/latest/plugins/xunit.html
     XUnit2,        ///< https://xunit.github.io/docs/format-xml-v2
+    TeamCity,      ///< Output suitable for teamcity
 };
 
 enum class TestResult
@@ -61,15 +62,15 @@ class TestContext
         Slang::String message;                 ///< Message that is specific for the testResult
     };
     
-    class Scope
+    class TestScope
     {
     public:
-        Scope(TestContext* context, const Slang::String& testName) :
+        TestScope(TestContext* context, const Slang::String& testName) :
             m_context(context)
         {
             context->startTest(testName);
         }
-        ~Scope()
+        ~TestScope()
         {
             m_context->endTest();
         }
@@ -78,6 +79,26 @@ class TestContext
         TestContext* m_context;
     };
 
+    class SuiteScope
+    {
+    public:
+        SuiteScope(TestContext* context, const Slang::String& suiteName) :
+            m_context(context)
+        {
+            context->startSuite(suiteName);
+        }
+        ~SuiteScope()
+        {
+            m_context->endSuite();
+        }
+
+    protected:
+        TestContext* m_context;
+    };
+
+    void startSuite(const Slang::String& name);
+    void endSuite();
+
     void startTest(const Slang::String& testName);
     void addResult(TestResult result);
     void addResultWithLocation(TestResult result, const char* testText, const char* file, int line);
@@ -99,9 +120,7 @@ class TestContext
         /// True if can write output directly to stderr
     bool canWriteStdError() const;
 
-        /// Call at end of tests 
-    void outputSummary();
-
+    
         /// Returns true if all run tests succeeded
     bool didAllSucceed() const;
 
@@ -115,6 +134,8 @@ class TestContext
 
     Slang::List<TestInfo> m_testInfos;
 
+    Slang::List<Slang::String> m_suiteStack;
+
     int m_totalTestCount;
     int m_passedTestCount;
     int m_failedTestCount;
@@ -126,9 +147,11 @@ class TestContext
     bool m_dumpOutputOnFailure;
     bool m_isVerbose;
 
+    void outputSummary();
+
 protected:
     void _addResult(const TestInfo& info);
-
+    
     Slang::StringBuilder m_currentMessage;
     TestInfo m_currentInfo;
     int m_numCurrentResults;