yum-mirror/slang

git clone https://git.yummers.dev/yum-mirror/slang

Janne Kiviluoto (NVIDIA)Add deterministic shuffling of tests in directory (#8622)e4d1200cb2025-10-08

// options.cpp
#include "options.h"

#include "../../source/core/slang-io.h"
#include "../../source/core/slang-string-util.h"

#include <stdio.h>
#include <stdlib.h>

using namespace Slang;

/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!! CategorySet !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */

TestCategory* TestCategorySet::add(String const& name, TestCategory* parent)
{
    RefPtr<TestCategory> category(new TestCategory);
    category->name = name;
    category->parent = parent;

    m_categoryMap.add(name, category);
    return category;
}

TestCategory* TestCategorySet::find(String const& name)
{
    if (auto category = m_categoryMap.tryGetValue(name))
    {
        return category->Ptr();
    }
    return nullptr;
}

TestCategory* TestCategorySet::findOrError(String const& name)
{
    TestCategory* category = find(name);
    if (!category)
    {
        StdWriters::getError().print("error: unknown test category name '%s'\n", name.getBuffer());
    }
    return category;
}

/* We need a way to differentiate a subCommand from say a test prefix. Here
we assume a command is just alpha characters or -, and this would differentiate it from
typical prefix usage (which is generally a directory). */
static bool _isSubCommand(const char* arg)
{
    for (; *arg; arg++)
    {
        const char c = *arg;
        // A command is just letters
        if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '-'))
        {
            return false;
        }
    }
    return true;
}

/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!! Options !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */

/* static */ void Options::showHelp(WriterHelper stdOut)
{
    stdOut.print(
        "Usage: slang-test [options] [test-prefix...]\n"
        "\n"
        "Options:\n"
        "  -h, --help                     Show this help message\n"
        "  -bindir <path>                 Set directory for binaries (default: the path to the "
        "slang-test executable)\n"
        "  -test-dir <path>               Set directory for test files (default: tests/)\n"
        "  -v [level]                     Set verbosity level (verbose, info, failure)\n"
        "                                 Default: verbose when -v used, info otherwise\n"
        "  -hide-ignored                  Hide results from ignored tests\n"
        "  -api-only                      Only run tests that use specified APIs\n"
        "  -verbose-paths                 Use verbose paths in output\n"
        "  -category <name>               Only run tests in specified category\n"
        "  -exclude <name>                Exclude tests in specified category\n"
        "  -exclude-prefix <prefix>       Exclude tests with specified path prefix\n"
        "  -api <expr>                    Enable specific APIs (e.g., 'vk+dx12' or '+dx11')\n"
        "  -synthesizedTestApi <expr>     Set APIs for synthesized tests\n"
        "  -skip-api-detection            Skip API availability detection\n"
        "  -server-count <n>              Set number of test servers (default: 1)\n"
        "  -show-adapter-info             Show detailed adapter information\n"
        "  -generate-hlsl-baselines       Generate HLSL test baselines\n"
        "  -skip-reference-image-generation Skip generating reference images for render tests\n"
        "  -emit-spirv-via-glsl           Emit SPIR-V through GLSL instead of directly\n"
        "  -expected-failure-list <file>  Specify file containing expected failures\n"
        "  -use-shared-library            Run tests in-process using shared library\n"
        "  -use-test-server               Run tests using test server\n"
        "  -use-fully-isolated-test-server  Run each test in isolated server\n"
        "  -capability <name>             Compile with the given capability\n"
        "  -shuffle-tests                 Shuffle tests in directories\n"
        "  -shuffle-seed <seed>           Set shuffle seed (default: 1)\n"

        // Recent Windows runtime versions started opening a dialog popup window when
        // `abort()` is called, which breaks the CI workflow and some scripts that
        // expect a normal termination.
        // It can be helpful for debugging but we should ignore it for CI.
        "  -ignore-abort-msg              Ignore abort message dialog popup on Windows\n"

        "  -enable-debug-layers [true|false] Enable or disable Validation Layer for Vulkan\n"
        "                                 and Debug Device for DX\n"
        "  -cache-rhi-device [true|false] Enable or disable RHI device caching (default: true)\n"
#if _DEBUG
        "  -disable-debug-layers          Disable the debug layers (default enabled in debug "
        "build)\n"
#endif
        "\n"
        "Output modes:\n"
        "  -appveyor                      Use AppVeyor output format\n"
        "  -travis                        Use Travis CI output format\n"
        "  -teamcity                      Use TeamCity output format\n"
        "  -xunit                         Use xUnit output format\n"
        "  -xunit2                        Use xUnit 2 output format\n"
        "\n"
        "Test prefixes are used to filter which tests to run. If no prefix is specified,\n"
        "all tests will be run.\n");
}

/* static */ Result Options::parse(
    int argc,
    char** argv,
    TestCategorySet* categorySet,
    Slang::WriterHelper stdOut,
    Slang::WriterHelper stdError,
    Options* optionsOut)
{
    // Reset the options
    *optionsOut = Options();

    List<const char*> positionalArgs;

    int argCount = argc;
    char const* const* argCursor = argv;
    char const* const* argEnd = argCursor + argCount;

#if _DEBUG
    // Enabling debug layers by default in debug builds.
    // For DX12 it will use the debug layer, for Vulkan it will enable validation layers.
    //
    // CI/CD will explicitly disable this until we address all of VUID errors.
    // https://github.com/shader-slang/slang/issues/4798
    //
    // When you run the Debug build locally, you may see more errors if not disabled with
    // '-enable-debug-layers false'.
    //
    optionsOut->enableDebugLayers = true;
#endif

    // first argument is the application name
    if (argCursor != argEnd)
    {
        optionsOut->appName = *argCursor++;
    }

    // Check for help flags first
    for (int i = 1; i < argc; i++)
    {
        if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0)
        {
            showHelp(stdOut);
            return SLANG_FAIL;
        }
    }

    // now iterate over arguments to collect options
    while (argCursor != argEnd)
    {
        char const* arg = *argCursor++;

        if (arg[0] != '-')
        {
            // We need to determine if this is a command, the confusion is that
            // previously we can specify a test prefix as just a single positional arg.
            // To rule this out, here it can only be a subCommand if it is just text

            if (_isSubCommand(arg))
            {
                optionsOut->subCommand = arg;
                // Make the first arg the command name
                optionsOut->subCommandArgs.add(optionsOut->subCommand);

                // Add all the remaining commands to subCommands
                for (; argCursor != argEnd; ++argCursor)
                {
                    optionsOut->subCommandArgs.add(*argCursor);
                }
                // Done
                return SLANG_OK;
            }

            positionalArgs.add(arg);
            continue;
        }

        if (strcmp(arg, "--") == 0)
        {
            // Add all positional args at the end
            while (argCursor != argEnd)
            {
                positionalArgs.add(*argCursor++);
            }
            break;
        }

        if (strcmp(arg, "-bindir") == 0)
        {
            if (argCursor == argEnd)
            {
                stdError.print("error: expected operand for '%s'\n", arg);
                showHelp(stdError);
                return SLANG_FAIL;
            }
            optionsOut->binDir = *argCursor++;
        }
        else if (strcmp(arg, "-use-shared-library") == 0)
        {
            optionsOut->defaultSpawnType = SpawnType::UseSharedLibrary;
        }
        else if (strcmp(arg, "-use-test-server") == 0)
        {
            optionsOut->defaultSpawnType = SpawnType::UseTestServer;
        }
        else if (strcmp(arg, "-use-fully-isolated-test-server") == 0)
        {
            optionsOut->defaultSpawnType = SpawnType::UseFullyIsolatedTestServer;
        }
        else if (strcmp(arg, "-v") == 0)
        {
            if (argCursor == argEnd)
            {
                // Default to verbose if no argument provided (backward compatibility)
                optionsOut->verbosity = VerbosityLevel::Verbose;
            }
            else
            {
                const char* verbosityArg = *argCursor;
                if (strcmp(verbosityArg, "verbose") == 0)
                {
                    optionsOut->verbosity = VerbosityLevel::Verbose;
                    argCursor++;
                }
                else if (strcmp(verbosityArg, "info") == 0)
                {
                    optionsOut->verbosity = VerbosityLevel::Info;
                    argCursor++;
                }
                else if (strcmp(verbosityArg, "failure") == 0)
                {
                    optionsOut->verbosity = VerbosityLevel::Failure;
                    argCursor++;
                }
                else
                {
                    // Not a verbosity level, treat as old-style -v
                    optionsOut->verbosity = VerbosityLevel::Verbose;
                }
            }
        }
        else if (strcmp(arg, "-hide-ignored") == 0)
        {
            optionsOut->hideIgnored = true;
        }
        else if (strcmp(arg, "-api-only") == 0)
        {
            optionsOut->apiOnly = true;
        }
        else if (strcmp(arg, "-verbose-paths") == 0)
        {
            optionsOut->verbosePaths = true;
        }
        else if (strcmp(arg, "-generate-hlsl-baselines") == 0)
        {
            optionsOut->generateHLSLBaselines = true;
        }
        else if (strcmp(arg, "-shuffle-tests") == 0)
        {
            optionsOut->shuffleTests = true;
        }
        else if (strcmp(arg, "-shuffle-seed") == 0)
        {
            if (argCursor == argEnd)
            {
                stdError.print("error: expected operand for '%s'\n", arg);
                showHelp(stdError);
                return SLANG_FAIL;
            }
            optionsOut->shuffleSeed = stringToInt(*argCursor++);
            if (optionsOut->shuffleSeed <= 0)
            {
                optionsOut->shuffleSeed = 1;
            }
        }
        else if (strcmp(arg, "-release") == 0)
        {
            // Assumed to be handle by .bat file that called us
        }
        else if (strcmp(arg, "-debug") == 0)
        {
            // Assumed to be handle by .bat file that called us
        }
        else if (strcmp(arg, "-configuration") == 0)
        {
            if (argCursor == argEnd)
            {
                stdError.print("error: expected operand for '%s'\n", arg);
                showHelp(stdError);
                return SLANG_FAIL;
            }
            argCursor++;
            // Assumed to be handle by .bat file that called us
        }
        else if (strcmp(arg, "-platform") == 0)
        {
            if (argCursor == argEnd)
            {
                stdError.print("error: expected operand for '%s'\n", arg);
                showHelp(stdError);
                return SLANG_FAIL;
            }
            argCursor++;
            // Assumed to be handle by .bat file that called us
        }
        else if (strcmp(arg, "-server-count") == 0)
        {
            if (argCursor == argEnd)
            {
                stdError.print("error: expected operand for '%s'\n", arg);
                showHelp(stdError);
                return SLANG_FAIL;
            }
            optionsOut->serverCount = stringToInt(*argCursor++);
            if (optionsOut->serverCount <= 0)
            {
                optionsOut->serverCount = 1;
            }
        }
        else if (strcmp(arg, "-appveyor") == 0)
        {
            optionsOut->outputMode = TestOutputMode::AppVeyor;
            optionsOut->dumpOutputOnFailure = true;
        }
        else if (strcmp(arg, "-travis") == 0)
        {
            optionsOut->outputMode = TestOutputMode::Travis;
            optionsOut->dumpOutputOnFailure = true;
        }
        else if (strcmp(arg, "-xunit") == 0)
        {
            optionsOut->outputMode = TestOutputMode::XUnit;
        }
        else if (strcmp(arg, "-xunit2") == 0)
        {
            optionsOut->outputMode = TestOutputMode::XUnit2;
        }
        else if (strcmp(arg, "-teamcity") == 0)
        {
            optionsOut->outputMode = TestOutputMode::TeamCity;
        }
        else if (strcmp(arg, "-category") == 0)
        {
            if (argCursor == argEnd)
            {
                stdError.print("error: expected operand for '%s'\n", arg);
                showHelp(stdError);
                return SLANG_FAIL;
            }
            auto category = categorySet->findOrError(*argCursor++);
            if (category)
            {
                optionsOut->includeCategories.add(category, category);
            }
        }
        else if (strcmp(arg, "-exclude") == 0)
        {
            if (argCursor == argEnd)
            {
                stdError.print("error: expected operand for '%s'\n", arg);
                showHelp(stdError);
                return SLANG_FAIL;
            }
            auto category = categorySet->findOrError(*argCursor++);
            if (category)
            {
                optionsOut->excludeCategories.add(category, category);
            }
        }
        else if (strcmp(arg, "-exclude-prefix") == 0)
        {
            if (argCursor == argEnd)
            {
                stdError.print("error: expected operand for '%s'\n", arg);
                showHelp(stdError);
                return SLANG_FAIL;
            }
            Slang::StringBuilder sb;
            Slang::Path::simplify(*argCursor++, Slang::Path::SimplifyStyle::NoRoot, sb);
            optionsOut->excludePrefixes.add(sb);
        }
        else if (strcmp(arg, "-api") == 0)
        {
            if (argCursor == argEnd)
            {
                stdError.print(
                    "error: expecting an api expression (eg 'vk+dx12' or '+dx11') '%s'\n",
                    arg);
                showHelp(stdError);
                return SLANG_FAIL;
            }
            const char* apiList = *argCursor++;

            SlangResult res = RenderApiUtil::parseApiFlags(
                UnownedStringSlice(apiList),
                optionsOut->enabledApis,
                &optionsOut->enabledApis);
            if (SLANG_FAILED(res))
            {
                stdError.print("error: unable to parse api expression '%s'\n", apiList);
                return res;
            }
        }
        else if (strcmp(arg, "-synthesizedTestApi") == 0)
        {
            if (argCursor == argEnd)
            {
                stdError.print(
                    "error: expected an api expression (eg 'vk+dx12' or '+dx11') '%s'\n",
                    arg);
                showHelp(stdError);
                return SLANG_FAIL;
            }
            const char* apiList = *argCursor++;

            SlangResult res = RenderApiUtil::parseApiFlags(
                UnownedStringSlice(apiList),
                optionsOut->synthesizedTestApis,
                &optionsOut->synthesizedTestApis);
            if (SLANG_FAILED(res))
            {
                stdError.print("error: unable to parse api expression '%s'\n", apiList);
                return res;
            }
        }
        else if (strcmp(arg, "-skip-api-detection") == 0)
        {
            optionsOut->skipApiDetection = true;
        }
        else if (strcmp(arg, "-emit-spirv-via-glsl") == 0)
        {
            optionsOut->emitSPIRVDirectly = false;
        }
        else if (strcmp(arg, "-capability") == 0)
        {
            if (argCursor == argEnd)
            {
                stdError.print("error: expected operand for '%s'\n", arg);
                showHelp(stdError);
                return SLANG_FAIL;
            }
            optionsOut->capabilities.add(*argCursor++);
        }
        else if (strcmp(arg, "-ignore-abort-msg") == 0)
        {
            optionsOut->ignoreAbortMsg = true;
#ifdef _MSC_VER
            _set_abort_behavior(0, _WRITE_ABORT_MSG);
#endif
        }
        else if (strcmp(arg, "-expected-failure-list") == 0)
        {
            if (argCursor == argEnd)
            {
                stdError.print("error: expected operand for '%s'\n", arg);
                showHelp(stdError);
                return SLANG_FAIL;
            }
            auto fileName = *argCursor++;
            String text;
            File::readAllText(fileName, text);
            List<UnownedStringSlice> lines;
            StringUtil::split(text.getUnownedSlice(), '\n', lines);
            for (auto line : lines)
            {
                // Remove comments (everything after '#' character)
                auto trimmedLine = line;
                auto commentIndex = line.indexOf('#');
                if (commentIndex != -1)
                {
                    trimmedLine = line.head(commentIndex);
                }

                // Trim whitespace and skip empty lines
                trimmedLine = trimmedLine.trim();
                if (trimmedLine.getLength() > 0)
                {
                    optionsOut->expectedFailureList.add(trimmedLine);
                }
            }
        }
        else if (strcmp(arg, "-test-dir") == 0)
        {
            if (argCursor == argEnd)
            {
                stdError.print("error: expected operand for '%s'\n", arg);
                showHelp(stdError);
                return SLANG_FAIL;
            }
            optionsOut->testDir = *argCursor++;
        }
        else if (strcmp(arg, "-show-adapter-info") == 0)
        {
            optionsOut->showAdapterInfo = true;
        }
        else if (strcmp(arg, "-skip-reference-image-generation") == 0)
        {
            optionsOut->skipReferenceImageGeneration = true;
        }
        else if (strcmp(arg, "-enable-debug-layers") == 0)
        {
            optionsOut->enableDebugLayers = true;

            if (argCursor == argEnd)
            {
                stdError.print("error: expected operand for '%s'\n", arg);
                showHelp(stdError);
                return SLANG_FAIL;
            }

            // Check for false variants
            const char* value = *argCursor++;
            if (value[0] == 'f' || value[0] == 'F' || value[0] == 'n' || value[0] == 'N' ||
                value[0] == '0' ||
                ((value[0] == 'o' || value[0] == 'O') && (value[1] == 'f' || value[1] == 'F')))
            {
                optionsOut->enableDebugLayers = false;
            }
        }
        else if (strcmp(arg, "-cache-rhi-device") == 0)
        {
            optionsOut->cacheRhiDevice = true;

            if (argCursor == argEnd)
            {
                stdError.print("error: expected operand for '%s'\n", arg);
                showHelp(stdError);
                return SLANG_FAIL;
            }

            // Check for false variants
            const char* value = *argCursor++;
            if (value[0] == 'f' || value[0] == 'F' || value[0] == 'n' || value[0] == 'N' ||
                value[0] == '0' ||
                ((value[0] == 'o' || value[0] == 'O') && (value[1] == 'f' || value[1] == 'F')))
            {
                optionsOut->cacheRhiDevice = false;
            }
        }
#if _DEBUG
        else if (strcmp(arg, "-disable-debug-layers") == 0)
        {
            stdError.print("warning: '-disable-debug-layers' is deprecated, use "
                           "'-enable-debug-layers false'\n");
            optionsOut->enableDebugLayers = false;
        }
#endif
        else
        {
            stdError.print("unknown option '%s'\n", arg);
            showHelp(stdError);
            return SLANG_FAIL;
        }
    }

    {
        // Find out what apis are available
        const int availableApis = RenderApiUtil::getAvailableApis();
        // Only allow apis we know are available
        optionsOut->enabledApis &= availableApis;

        // Can only synth for apis that are available
        optionsOut->synthesizedTestApis &= optionsOut->enabledApis;
    }


    // first positional argument is source shader path
    optionsOut->testPrefixes.clear();
    optionsOut->testPrefixes.reserve(positionalArgs.getCount());
    for (auto testPrefix : positionalArgs)
    {
        Slang::StringBuilder sb;
        Slang::Path::simplify(testPrefix, Slang::Path::SimplifyStyle::NoRoot, sb);
        optionsOut->testPrefixes.add(sb);
    }

    if (optionsOut->binDir.getLength() == 0)
    {
        // If the binDir isn't set try using the path to the executable
        String exePath = Path::getExecutablePath();
        if (exePath.getLength())
        {
            optionsOut->binDir = Path::getParentDirectory(exePath);
        }
    }

    if (optionsOut->testDir.getLength() == 0)
    {
        // If the test directory isn't set, use the "tests" directory
        optionsOut->testDir = String("tests/");
    }

    return SLANG_OK;
}