yum-mirror/slang

Making it easier to work with shaders

git clone https://git.yummers.dev/yum-mirror/slang

Yong HeRelax restriction on using link-time types for shader parameters. (#8387)bc6b82666

master
branchesmasterall refs →
22.4 KiB851 linesraw
1// test-reporter.cpp
2#include "test-reporter.h"
3
4#include "../../source/core/slang-process-util.h"
5#include "../../source/core/slang-string-util.h"
6#include "options.h"
7
8#include <mutex>
9#include <stdio.h>
10#include <stdlib.h>
11
12using namespace Slang;
13
14/* static */ TestReporter* TestReporter::s_reporter = nullptr;
15
16static void appendXmlEncode(char c, StringBuilder& out)
17{
18    switch (c)
19    {
20    case '&':
21        out << "&amp;";
22        break;
23    case '<':
24        out << "&lt;";
25        break;
26    case '>':
27        out << "&gt;";
28        break;
29    case '\'':
30        out << "&apos;";
31        break;
32    case '"':
33        out << "&quot;";
34        break;
35    default:
36        out.append(c);
37    }
38}
39
40static bool isXmlEncodeChar(char c)
41{
42    switch (c)
43    {
44    case '&':
45    case '<':
46    case '>':
47        {
48            return true;
49        }
50    }
51    return false;
52}
53
54static void appendXmlEncode(const String& in, StringBuilder& out)
55{
56    const char* cur = in.getBuffer();
57    const char* end = cur + in.getLength();
58
59    while (cur < end)
60    {
61        const char* start = cur;
62        // Look for a run of non encoded
63        while (cur < end && !isXmlEncodeChar(*cur))
64        {
65            cur++;
66        }
67        // Write it
68        if (cur > start)
69        {
70            out.append(start, UInt(end - start));
71        }
72
73        // if not at the end, we must be on an xml encoded character, so just output it xml encoded.
74        if (cur < end)
75        {
76            const char encodeChar = *cur++;
77            assert(isXmlEncodeChar(encodeChar));
78            appendXmlEncode(encodeChar, out);
79        }
80    }
81}
82
83TestReporter::TestReporter()
84    : m_outputMode(TestOutputMode::Default)
85{
86    m_totalTestCount = 0;
87    m_passedTestCount = 0;
88    m_failedTestCount = 0;
89    m_ignoredTestCount = 0;
90    m_expectedFailedTestCount = 0;
91    m_maxFailTestResults = 10;
92
93    m_inTest = false;
94    m_dumpOutputOnFailure = false;
95    m_verbosity = VerbosityLevel::Info;
96}
97
98Result TestReporter::init(
99    TestOutputMode outputMode,
100    const HashSet<String>& expectedFailureList,
101    bool isSubReporter)
102{
103    m_outputMode = outputMode;
104    m_isSubReporter = isSubReporter;
105    m_expectedFailureList = expectedFailureList;
106    return SLANG_OK;
107}
108
109TestReporter::~TestReporter() {}
110
111bool TestReporter::canWriteStdError() const
112{
113    switch (m_outputMode)
114    {
115    case TestOutputMode::XUnit:
116    case TestOutputMode::XUnit2:
117        {
118            return false;
119        }
120    default:
121        return true;
122    }
123}
124
125void TestReporter::startTest(const char* testName)
126{
127    // Must be in a suite
128    assert(m_suiteStack.getCount());
129    assert(!m_inTest);
130
131    m_inTest = true;
132
133    m_numCurrentResults = 0;
134    m_numFailResults = 0;
135
136    m_currentInfo = TestInfo();
137    m_currentInfo.name = testName;
138    m_currentMessage.clear();
139}
140
141void TestReporter::endTest()
142{
143    assert(m_suiteStack.getCount());
144    assert(m_inTest);
145
146    m_currentInfo.message = m_currentMessage;
147
148    _addResult(m_currentInfo);
149
150    m_inTest = false;
151}
152
153void TestReporter::addResult(TestResult result)
154{
155    assert(m_inTest);
156
157    std::lock_guard<std::recursive_mutex> lock(m_mutex);
158    if (result == TestResult::Fail && m_expectedFailureList.contains(m_currentInfo.name))
159        result = TestResult::ExpectedFail;
160    m_currentInfo.testResult = combine(m_currentInfo.testResult, result);
161    m_numCurrentResults++;
162}
163
164TestResult TestReporter::getResult() const
165{
166    return m_currentInfo.testResult;
167}
168
169void TestReporter::addExecutionTime(double time)
170{
171    std::lock_guard<std::recursive_mutex> lock(m_mutex);
172
173    m_currentInfo.executionTime = time;
174}
175
176void TestReporter::addResultWithLocation(
177    TestResult result,
178    const char* testText,
179    const char* file,
180    int line)
181{
182    assert(m_inTest);
183
184    std::lock_guard<std::recursive_mutex> lock(m_mutex);
185    result = adjustResult(m_currentInfo.name.getUnownedSlice(), result);
186
187    m_numCurrentResults++;
188
189    m_currentInfo.testResult = combine(m_currentInfo.testResult, result);
190    if (result != TestResult::Fail)
191    {
192        // We don't need to output the result if it
193        return;
194    }
195
196    m_numFailResults++;
197
198    if (m_maxFailTestResults > 0)
199    {
200        if (m_numFailResults > m_maxFailTestResults)
201        {
202            if (m_numFailResults == m_maxFailTestResults + 1)
203            {
204                // It's a failure, but to show that there are more than are going to be shown, just
205                // show '...'
206                message(TestMessageType::TestFailure, "...");
207            }
208            return;
209        }
210    }
211
212    StringBuilder buf;
213    buf << testText << " - " << file << " (" << line << ")";
214
215    message(TestMessageType::TestFailure, buf);
216}
217
218void TestReporter::addResultWithLocation(
219    bool testSucceeded,
220    const char* testText,
221    const char* file,
222    int line)
223{
224    addResultWithLocation(
225        testSucceeded ? TestResult::Pass : TestResult::Fail,
226        testText,
227        file,
228        line);
229}
230
231TestResult TestReporter::addTest(const String& testName, bool isPass)
232{
233    const TestResult res = isPass ? TestResult::Pass : TestResult::Fail;
234    addTest(testName, res);
235    return res;
236}
237
238void TestReporter::consolidateWith(TestReporter* other)
239{
240    m_testInfos.addRange(other->m_testInfos);
241    m_failedTestCount += other->m_failedTestCount;
242    m_ignoredTestCount += other->m_ignoredTestCount;
243    m_passedTestCount += other->m_passedTestCount;
244    m_expectedFailedTestCount += other->m_expectedFailedTestCount;
245    m_totalTestCount += other->m_totalTestCount;
246}
247
248void TestReporter::dumpOutputDifference(const String& expectedOutput, const String& actualOutput)
249{
250    StringBuilder builder;
251
252    StringUtil::appendFormat(
253        builder,
254        "ERROR:\n"
255        "EXPECTED{{{\n%s}}}\n"
256        "ACTUAL{{{\n%s}}}\n",
257        expectedOutput.getBuffer(),
258        actualOutput.getBuffer());
259
260    // Add to the m_currentInfo
261    message(TestMessageType::TestFailure, builder);
262}
263
264static char _getTeamCityEscapeChar(char c)
265{
266    switch (c)
267    {
268    case '|':
269        return '|';
270    case '\'':
271        return '\'';
272    case '\n':
273        return 'n';
274    case '\r':
275        return 'r';
276    case '[':
277        return '[';
278    case ']':
279        return ']';
280    default:
281        return 0;
282    }
283}
284
285static void _appendEncodedTeamCityString(const UnownedStringSlice& in, StringBuilder& builder)
286{
287    const char* start = in.begin();
288    const char* cur = start;
289    const char* end = in.end();
290
291    for (const char* cur = start; cur < end; cur++)
292    {
293        const char c = *cur;
294        const char escapeChar = _getTeamCityEscapeChar(c);
295        if (escapeChar)
296        {
297            // Flush
298            if (cur > start)
299            {
300                builder.append(start, UInt(cur - start));
301            }
302
303            builder.append('|');
304            builder.append(escapeChar);
305            start = cur + 1;
306        }
307    }
308
309    // Flush the end
310    if (end > start)
311    {
312        builder.append(start, UInt(end - start));
313    }
314}
315
316static void _appendTime(double timeInSec, StringBuilder& out)
317{
318    SLANG_ASSERT(timeInSec >= 0.0);
319    if (timeInSec == 0.0 || timeInSec >= 1.0)
320    {
321        out << timeInSec << "s";
322        return;
323    }
324    timeInSec *= 1000.0f;
325    if (timeInSec > 1.0f)
326    {
327        out << timeInSec << "ms";
328        return;
329    }
330    timeInSec *= 1000.0f;
331    if (timeInSec > 1.0f)
332    {
333        out << timeInSec << "us";
334        return;
335    }
336
337    timeInSec *= 1000.0f;
338    out << timeInSec << "ns";
339}
340
341void TestReporter::_addResult(TestInfo info)
342{
343    if (info.testResult == TestResult::Ignored && m_hideIgnored)
344    {
345        return;
346    }
347    info.testResult = adjustResult(info.name.getUnownedSlice(), info.testResult);
348
349    m_totalTestCount++;
350
351    switch (info.testResult)
352    {
353    case TestResult::Fail:
354        m_failedTestCount++;
355        break;
356
357    case TestResult::Pass:
358        m_passedTestCount++;
359        break;
360    case TestResult::ExpectedFail:
361        m_expectedFailedTestCount++;
362        break;
363
364    case TestResult::Ignored:
365        m_ignoredTestCount++;
366        break;
367
368    default:
369        assert(!"unexpected");
370        break;
371    }
372
373    m_testInfos.add(info);
374
375    auto defaultOutputFunc = [this](const TestInfo& info)
376    {
377        // Skip output for passed/ignored tests if verbosity < Info
378        if (m_verbosity < VerbosityLevel::Info)
379        {
380            if (info.testResult == TestResult::Pass || info.testResult == TestResult::Ignored)
381            {
382                return;
383            }
384        }
385
386        char const* resultString = "UNEXPECTED";
387        switch (info.testResult)
388        {
389        case TestResult::Fail:
390            resultString = "FAILED";
391            break;
392        case TestResult::ExpectedFail:
393            resultString = "failed(expected)";
394            break;
395        case TestResult::Pass:
396            resultString = "passed";
397            break;
398        case TestResult::Ignored:
399            resultString = "ignored";
400            break;
401        default:
402            assert(!"unexpected");
403            break;
404        }
405
406        StringBuilder buffer;
407        if (info.executionTime > 0.0f)
408        {
409            _appendTime(info.executionTime, buffer);
410        }
411        printf(
412            "%s test: '%S' %s\n",
413            resultString,
414            info.name.toWString().begin(),
415            buffer.getBuffer());
416        fflush(stdout);
417    };
418
419    switch (m_outputMode)
420    {
421    default:
422        {
423            defaultOutputFunc(info);
424            break;
425        }
426    case TestOutputMode::TeamCity:
427        {
428            StringBuilder escapedTestName;
429            _appendEncodedTeamCityString(info.name.getUnownedSlice(), escapedTestName);
430
431            printf("##teamcity[testStarted name='%s']\n", escapedTestName.begin());
432
433            switch (info.testResult)
434            {
435            case TestResult::Fail:
436                {
437                    if (info.message.getLength())
438                    {
439                        StringBuilder escapedMessage;
440                        _appendEncodedTeamCityString(
441                            info.message.getUnownedSlice(),
442                            escapedMessage);
443                        printf(
444                            "##teamcity[testFailed name='%s' message='%s']\n",
445                            escapedTestName.begin(),
446                            escapedMessage.begin());
447                    }
448                    else
449                    {
450                        printf("##teamcity[testFailed name='%s']\n", escapedTestName.begin());
451                    }
452                    break;
453                }
454            case TestResult::Pass:
455            case TestResult::ExpectedFail:
456                {
457                    StringBuilder message;
458                    message << info.message;
459                    // Add execution time if one is set
460                    if (info.executionTime > 0.0)
461                    {
462                        if (message.getLength())
463                        {
464                            message << " ";
465                        }
466                        _appendTime(info.executionTime, message);
467                    }
468
469                    if (message.getLength())
470                    {
471                        StringBuilder escapedMessage;
472                        _appendEncodedTeamCityString(message.getUnownedSlice(), escapedMessage);
473                        printf(
474                            "##teamcity[testStdOut name='%s' out='%s']\n",
475                            escapedTestName.begin(),
476                            escapedMessage.begin());
477                    }
478                    break;
479                }
480            case TestResult::Ignored:
481                {
482                    if (info.message.getLength())
483                    {
484                        StringBuilder escapedMessage;
485                        _appendEncodedTeamCityString(
486                            info.message.getUnownedSlice(),
487                            escapedMessage);
488
489                        printf(
490                            "##teamcity[testIgnored name='%s' message='%s']\n",
491                            escapedTestName.begin(),
492                            escapedMessage.begin());
493                    }
494                    else
495                    {
496                        printf("##teamcity[testIgnored name='%s']\n", escapedTestName.begin());
497                    }
498                    break;
499                }
500            default:
501                assert(!"unexpected");
502                break;
503            }
504
505            printf("##teamcity[testFinished name='%s']\n", escapedTestName.begin());
506            fflush(stdout);
507            break;
508        }
509    case TestOutputMode::XUnit2:
510    case TestOutputMode::XUnit:
511        {
512            // Don't output anything -> we'll output all in one go at the end
513            break;
514        }
515    case TestOutputMode::AppVeyor:
516        {
517            char const* resultString = "None";
518            switch (info.testResult)
519            {
520            case TestResult::Fail:
521                resultString = "Failed";
522                break;
523            case TestResult::Pass:
524                resultString = "Passed";
525                break;
526            case TestResult::Ignored:
527                resultString = "Ignored";
528                break;
529            case TestResult::ExpectedFail:
530                resultString = "ExpectedFail";
531                break;
532
533            default:
534                assert(!"unexpected");
535                break;
536            }
537
538            // https://www.appveyor.com/docs/build-worker-api/#add-tests
539
540            CommandLine cmdLine;
541            cmdLine.setExecutableLocation(ExecutableLocation("appveyor"));
542            cmdLine.addArg("AddTest");
543            cmdLine.addArg(info.name);
544            cmdLine.addArg("-FileName");
545            // TODO: this isn't actually a file name in all cases
546            cmdLine.addArg(info.name);
547            cmdLine.addArg("-Framework");
548            cmdLine.addArg("slang-test");
549            cmdLine.addArg("-Outcome");
550            cmdLine.addArg(resultString);
551
552            // If has execution time output it
553            if (info.executionTime > 0.0)
554            {
555                StringBuilder builder;
556                _appendTime(info.executionTime, builder);
557                cmdLine.addArg("-StdOut");
558                cmdLine.addArg(builder);
559            }
560
561            ExecuteResult exeRes;
562            SlangResult res = ProcessUtil::execute(cmdLine, exeRes);
563
564            if (SLANG_FAILED(res))
565            {
566                messageFormat(
567                    TestMessageType::Info,
568                    "failed to add appveyor test results for '%S'\n",
569                    info.name.toWString().begin());
570
571#if 0
572                String cmdLineString = ProcessUtil::getCommandLineString(cmdLine);
573                fprintf(stderr, "[%d] TEST RESULT: %s {%d} {%s} {%s}\n", err, cmdLineString.getBuffer(),
574                    exeRes.resultCode,
575                    exeRes.standardOutput.begin(),
576                    exeRes.standardError.begin());
577#endif
578            }
579            defaultOutputFunc(info);
580            break;
581        }
582    }
583}
584
585void TestReporter::addTest(const String& testName, TestResult testResult)
586{
587    // Can't add this way if in test
588    assert(!m_inTest);
589
590    TestInfo info;
591    info.name = testName;
592    info.testResult = testResult;
593    _addResult(info);
594}
595
596void TestReporter::message(TestMessageType type, const String& message)
597{
598    std::lock_guard<std::recursive_mutex> lock(m_mutex);
599
600    if (type == TestMessageType::Info)
601    {
602        if (m_verbosity == VerbosityLevel::Verbose && canWriteStdError())
603        {
604            fputs(message.getBuffer(), stderr);
605        }
606        fflush(stderr);
607        // Just dump out if can dump out
608        return;
609    }
610
611    if (canWriteStdError())
612    {
613        fprintf(stderr, "[%s] ", m_currentInfo.name.getBuffer());
614        if (type == TestMessageType::RunError || type == TestMessageType::TestFailure)
615        {
616            fputs(message.getBuffer(), stderr);
617            fprintf(stderr, "\n");
618        }
619        else
620        {
621            fputs(message.getBuffer(), stderr);
622        }
623        fflush(stderr);
624    }
625
626    if (m_currentMessage.getLength() > 0)
627    {
628        m_currentMessage << "\n";
629    }
630
631    if (m_currentInfo.name.getLength())
632        m_currentMessage << "[" << m_currentInfo.name << "] ";
633    m_currentMessage.append(message);
634}
635
636void TestReporter::messageFormat(TestMessageType type, char const* format, ...)
637{
638    StringBuilder builder;
639
640    va_list args;
641    va_start(args, format);
642    StringUtil::append(format, args, builder);
643    va_end(args);
644
645    message(type, builder);
646}
647
648void TestReporter::message(TestMessageType type, const char* messageContent)
649{
650    message(type, String(messageContent));
651}
652
653
654bool TestReporter::didAllSucceed() const
655{
656    return m_failedTestCount == 0;
657}
658
659void TestReporter::outputSummary()
660{
661    auto passCount = m_passedTestCount;
662    auto rawTotal = m_totalTestCount;
663    auto ignoredCount = m_ignoredTestCount;
664
665    auto runTotal = rawTotal - ignoredCount;
666
667    switch (m_outputMode)
668    {
669    default:
670        {
671            if (!m_totalTestCount)
672            {
673                printf("no tests run\n");
674                return;
675            }
676
677            int percentPassed = 0;
678            if (runTotal > 0)
679            {
680                percentPassed = (passCount * 100) / runTotal;
681            }
682
683            printf("\n===\n%d%% of tests passed (%d/%d)", percentPassed, passCount, runTotal);
684            if (ignoredCount)
685            {
686                printf(", %d tests ignored", ignoredCount);
687            }
688            if (m_expectedFailedTestCount)
689            {
690                printf(", %d tests failed expectedly", m_expectedFailedTestCount);
691                printf("\n===\n\n");
692                printf("\npassing tests that are expected to fail:\n");
693                printf("---\n");
694                for (const auto& testInfo : m_testInfos)
695                {
696                    if (testInfo.testResult == TestResult::Pass)
697                    {
698                        if (m_expectedFailureList.contains(testInfo.name))
699                        {
700                            printf("%s\n", testInfo.name.getBuffer());
701                        }
702                    }
703                }
704                printf("---\n");
705            }
706            printf("\n===\n\n");
707            if (m_failedTestCount)
708            {
709                printf("%d failing tests:\n", m_failedTestCount);
710                printf("---\n");
711                for (const auto& testInfo : m_testInfos)
712                {
713                    if (testInfo.testResult == TestResult::Fail)
714                    {
715                        printf("%s\n", testInfo.name.getBuffer());
716                    }
717                }
718                printf("---\n");
719            }
720
721            break;
722        }
723
724    case TestOutputMode::XUnit:
725        {
726            // xUnit 1.0 format
727
728            printf("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
729            printf(
730                "<testsuites tests=\"%d\" failures=\"%d\" disabled=\"%d\" errors=\"0\" "
731                "name=\"AllTests\">\n",
732                m_totalTestCount,
733                m_failedTestCount,
734                m_ignoredTestCount);
735            printf(
736                "  <testsuite name=\"all\" tests=\"%d\" failures=\"%d\" disabled=\"%d\" "
737                "errors=\"0\" time=\"0\">\n",
738                m_totalTestCount,
739                m_failedTestCount,
740                m_ignoredTestCount);
741
742            for (const auto& testInfo : m_testInfos)
743            {
744                const int numFailed = (testInfo.testResult == TestResult::Fail);
745                const int numIgnored = (testInfo.testResult == TestResult::Ignored);
746                // int numPassed = (testInfo.testResult == TestResult::ePass);
747
748                if (testInfo.testResult == TestResult::Pass)
749                {
750                    printf(
751                        "    <testcase name=\"%s\" status=\"run\"/>\n",
752                        testInfo.name.getBuffer());
753                }
754                else
755                {
756                    printf(
757                        "    <testcase name=\"%s\" status=\"run\">\n",
758                        testInfo.name.getBuffer());
759                    switch (testInfo.testResult)
760                    {
761                    case TestResult::Fail:
762                        {
763                            StringBuilder buf;
764                            appendXmlEncode(testInfo.message, buf);
765
766                            printf("      <error>\n");
767                            printf("%s", buf.getBuffer());
768                            printf("      </error>\n");
769                            break;
770                        }
771                    case TestResult::Ignored:
772                        {
773                            printf("      <skip>Ignored</skip>\n");
774                            break;
775                        }
776                    default:
777                        break;
778                    }
779                    printf("    </testcase>\n");
780                }
781            }
782
783            printf("  </testsuite>\n");
784            printf("</testSuites>\n");
785            break;
786        }
787    case TestOutputMode::XUnit2:
788        {
789            // https://xunit.github.io/docs/format-xml-v2
790            assert("Not currently supported");
791            break;
792        }
793    case TestOutputMode::TeamCity:
794        {
795            // Don't output a summary
796            break;
797        }
798    }
799}
800
801void TestReporter::startSuite(const String& name)
802{
803    m_suiteStack.add(name);
804
805    switch (m_outputMode)
806    {
807    case TestOutputMode::TeamCity:
808        {
809            if (!m_isSubReporter)
810            {
811                StringBuilder escapedSuiteName;
812                _appendEncodedTeamCityString(name.getUnownedSlice(), escapedSuiteName);
813                printf("##teamcity[testSuiteStarted name='%s']\n", escapedSuiteName.begin());
814            }
815            break;
816        }
817    default:
818        break;
819    }
820}
821
822void TestReporter::endSuite()
823{
824    assert(m_suiteStack.getCount());
825
826    switch (m_outputMode)
827    {
828    case TestOutputMode::TeamCity:
829        {
830            if (!m_isSubReporter)
831            {
832                const String& name = m_suiteStack.getLast();
833                StringBuilder escapedSuiteName;
834                _appendEncodedTeamCityString(name.getUnownedSlice(), escapedSuiteName);
835                printf("##teamcity[testSuiteFinished name='%s']\n", escapedSuiteName.begin());
836            }
837            break;
838        }
839    default:
840        break;
841    }
842
843    m_suiteStack.removeLast();
844}
845
846TestResult TestReporter::adjustResult(UnownedStringSlice testName, TestResult result)
847{
848    if (result == TestResult::Fail && m_expectedFailureList.contains(testName))
849        result = TestResult::ExpectedFail;
850    return result;
851}