Improve slang-test output verbosity control (#7625)

* Improve slang-test output verbosity control

This commit improves the existing command-line argument for slang-test,
"-v". Previously it printed more information when "-v" was used.

This commit adds a new option to silence the information output so that
LLM processes less tokens when things are working as expected.

* format code (#74)

---------

Co-authored-by: slangbot <ellieh+slangbot@nvidia.com>

5 files changed, 65 insertions, 13 deletions

diff --git a/tools/slang-test/options.cpp b/tools/slang-test/options.cpp
index aa538e75e..4103d56b5 100644
--- a/tools/slang-test/options.cpp
+++ b/tools/slang-test/options.cpp
@@ -69,7 +69,8 @@ static bool _isSubCommand(const char* arg)
         "  -bindir <path>                 Set directory for binaries (default: the path to the "
         "slang-test executable)\n"
         "  -test-dir <path>               Set directory for test files (default: tests/)\n"
-        "  -v                             Enable verbose output\n"
+        "  -v [level]                     Set verbosity level (verbose, info, failure)\n"
+        "                                 Default: verbose when -v used, info otherwise\n"
         "  -hide-ignored                  Hide results from ignored tests\n"
         "  -api-only                      Only run tests that use specified APIs\n"
         "  -verbose-paths                 Use verbose paths in output\n"
@@ -215,7 +216,35 @@ static bool _isSubCommand(const char* arg)
         }
         else if (strcmp(arg, "-v") == 0)
         {
-            optionsOut->shouldBeVerbose = true;
+            if (argCursor == argEnd)
+            {
+                // Default to verbose if no argument provided (backward compatibility)
+                optionsOut->verbosity = VerbosityLevel::Verbose;
+            }
+            else
+            {
+                const char* verbosityArg = *argCursor;
+                if (strcmp(verbosityArg, "verbose") == 0)
+                {
+                    optionsOut->verbosity = VerbosityLevel::Verbose;
+                    argCursor++;
+                }
+                else if (strcmp(verbosityArg, "info") == 0)
+                {
+                    optionsOut->verbosity = VerbosityLevel::Info;
+                    argCursor++;
+                }
+                else if (strcmp(verbosityArg, "failure") == 0)
+                {
+                    optionsOut->verbosity = VerbosityLevel::Failure;
+                    argCursor++;
+                }
+                else
+                {
+                    // Not a verbosity level, treat as old-style -v
+                    optionsOut->verbosity = VerbosityLevel::Verbose;
+                }
+            }
         }
         else if (strcmp(arg, "-hide-ignored") == 0)
         {
diff --git a/tools/slang-test/options.h b/tools/slang-test/options.h
index b2f71d3e3..8ae89d084 100644
--- a/tools/slang-test/options.h
+++ b/tools/slang-test/options.h
@@ -44,6 +44,13 @@ enum class SpawnType
     UseFullyIsolatedTestServer, ///< Uses a test server for each test (slow!)
 };
 
+enum class VerbosityLevel
+{
+    Failure, ///< Only show failures and errors
+    Info,    ///< Show test discovery and results (default)
+    Verbose, ///< Show detailed output including command lines
+};
+
 struct Options
 {
     char const* appName = "slang-test";
@@ -57,8 +64,8 @@ struct Options
     // only run test cases with names have one of these prefixes.
     Slang::List<Slang::String> testPrefixes;
 
-    // generate extra output (notably: command lines we run)
-    bool shouldBeVerbose = false;
+    // verbosity level for output
+    VerbosityLevel verbosity = VerbosityLevel::Info;
 
     // When true results from ignored tests are not shown
     bool hideIgnored = false;
diff --git a/tools/slang-test/slang-test-main.cpp b/tools/slang-test/slang-test-main.cpp
index 14c9b52ef..4eab84340 100644
--- a/tools/slang-test/slang-test-main.cpp
+++ b/tools/slang-test/slang-test-main.cpp
@@ -782,7 +782,7 @@ Result spawnAndWaitExe(
 
     const auto& options = context->options;
 
-    if (options.shouldBeVerbose)
+    if (options.verbosity == VerbosityLevel::Verbose)
     {
         String commandLine = cmdLine.toString();
         context->getTestReporter()->messageFormat(
@@ -815,7 +815,7 @@ Result spawnAndWaitSharedLibrary(
     const auto& options = context->options;
     String exeName = Path::getFileNameWithoutExt(cmdLine.m_executableLocation.m_pathOrName);
 
-    if (options.shouldBeVerbose)
+    if (options.verbosity == VerbosityLevel::Verbose)
     {
         CommandLine testCmdLine;
 
@@ -908,7 +908,7 @@ Result spawnAndWaitProxy(
     cmdLine.setExecutableLocation(ExecutableLocation(context->exeDirectoryPath, "test-proxy"));
 
     const auto& options = context->options;
-    if (options.shouldBeVerbose)
+    if (options.verbosity == VerbosityLevel::Verbose)
     {
         String commandLine = cmdLine.toString();
         context->getTestReporter()->messageFormat(
@@ -4624,7 +4624,10 @@ void runTestsInDirectory(TestContext* context)
     {
         if (shouldRunTest(context, file))
         {
-            printf("found test: '%s'\n", file.getBuffer());
+            if (context->options.verbosity >= VerbosityLevel::Info)
+            {
+                printf("found test: '%s'\n", file.getBuffer());
+            }
             if (SLANG_FAILED(_runTestsOnFile(context, file)))
             {
                 {
@@ -5071,7 +5074,7 @@ SlangResult innerMain(int argc, char** argv)
         context.setTestReporter(&reporter);
 
         reporter.m_dumpOutputOnFailure = options.dumpOutputOnFailure;
-        reporter.m_isVerbose = options.shouldBeVerbose;
+        reporter.m_verbosity = options.verbosity;
         reporter.m_hideIgnored = options.hideIgnored;
 
         {
diff --git a/tools/slang-test/test-reporter.cpp b/tools/slang-test/test-reporter.cpp
index 352245028..3aba10566 100644
--- a/tools/slang-test/test-reporter.cpp
+++ b/tools/slang-test/test-reporter.cpp
@@ -3,6 +3,7 @@
 
 #include "../../source/core/slang-process-util.h"
 #include "../../source/core/slang-string-util.h"
+#include "options.h"
 
 #include <mutex>
 #include <stdio.h>
@@ -91,7 +92,7 @@ TestReporter::TestReporter()
 
     m_inTest = false;
     m_dumpOutputOnFailure = false;
-    m_isVerbose = false;
+    m_verbosity = VerbosityLevel::Info;
 }
 
 Result TestReporter::init(
@@ -371,8 +372,17 @@ void TestReporter::_addResult(TestInfo info)
 
     m_testInfos.add(info);
 
-    auto defaultOutputFunc = [](const TestInfo& info)
+    auto defaultOutputFunc = [this](const TestInfo& info)
     {
+        // Skip output for passed/ignored tests if verbosity < Info
+        if (m_verbosity < VerbosityLevel::Info)
+        {
+            if (info.testResult == TestResult::Pass || info.testResult == TestResult::Ignored)
+            {
+                return;
+            }
+        }
+
         char const* resultString = "UNEXPECTED";
         switch (info.testResult)
         {
@@ -589,7 +599,7 @@ void TestReporter::message(TestMessageType type, const String& message)
 
     if (type == TestMessageType::Info)
     {
-        if (m_isVerbose && canWriteStdError())
+        if (m_verbosity == VerbosityLevel::Verbose && canWriteStdError())
         {
             fputs(message.getBuffer(), stderr);
         }
diff --git a/tools/slang-test/test-reporter.h b/tools/slang-test/test-reporter.h
index 321e65ae2..9c1384615 100644
--- a/tools/slang-test/test-reporter.h
+++ b/tools/slang-test/test-reporter.h
@@ -11,6 +11,9 @@
 
 #include <mutex>
 
+// Forward declaration
+enum class VerbosityLevel;
+
 enum class TestOutputMode
 {
     Default = 0, ///< Default mode is to write test results to the console
@@ -139,7 +142,7 @@ public:
 
     TestOutputMode m_outputMode = TestOutputMode::Default;
     bool m_dumpOutputOnFailure;
-    bool m_isVerbose = false;
+    VerbosityLevel m_verbosity;
     bool m_hideIgnored = false;
     bool m_isSubReporter = false;
     Slang::HashSet<Slang::String> m_expectedFailureList;