Running tests in slang-test process (#740)

* First pass at having an interface to write text to that can be replaced.
Simplifed and made more rigerous the interface used to write formatted strings.

* Added AppContext to simplify setting up and parsing around of streams.

* Added more simplified way to get the std error/out from AppContext.

* Work in progress using dll for tools to speed up testing.

* First pass at ISlangWriter interface.

* Added support for writing VaArgs.
Added NullWriter.

* Use ISlangWriter for output.

* Use ISlangWriter for output - replacing OutputCallback.
Make IRDump go to ISlangWriter

* SlangWriterTargetType -> SlangWriterChannel
Improvements around AppContext

* Shared library working with slang-reflection-test.

* Dll testing working for render-test.

* Include va_list definintion from header.

* Fix errors from clang.

* Fix typo for linux.

* Added -usexes option

* Fix typo.

* Fix arguments problem on linux.

* Fix typo for linux.

* Add windows tool shared library projects.

* Fix warning from x86 win build.
Fix signed warning from slang-test/main.cpp

* First attempt at getting premake to work on travis, and run tests.

* Try moving build out into script.

* Invoke bash scripts so they don't have to be executable.

* Drive configuration/tests from env parameters set by travis

* Try using source to run travis tests.

* Remove the build.linux directory - but doing so will overwrite Makefile.

* Made -fno-delete-null-pointer-checks gcc only.

* Try to fix warning from -fno-delete-null-pointer-checks

* Turn of warnings for unknown switches.

* Try to make premake choose the correct tooling.

* Disabled missing braces warning.

* Disable -Wundefined-var-template on clang.

* -Wunused-function disabled for clang.

* Fix typo due to SlangBool.

* Remove this nullptr tests.

* "-Wno-unused-private-field" for clang.

* Added "-Wno-undefined-bool-conversion"

* Add DominatorList::end fix.

* Split scripts into travis_build.sh travis_test.sh

* Fix gcc/clang template pre-declaration issue around QualType.

* Fix premake to build such that pthread correctly links with slang-glslang

5 files changed, 152 insertions, 43 deletions

diff --git a/tools/slang-test/main.cpp b/tools/slang-test/main.cpp
index 943e82cef..77005d03b 100644
--- a/tools/slang-test/main.cpp
+++ b/tools/slang-test/main.cpp
@@ -2,6 +2,7 @@
 
 #include "../../source/core/slang-io.h"
 #include "../../source/core/token-reader.h"
+#include "../../source/core/slang-app-context.h"
 
 #include "../../slang-com-helper.h"
 
@@ -95,6 +96,9 @@ struct Options
     // integration builds.
     bool dumpOutputOnFailure = false;
 
+    // If set, will force using of executables (not shared library) for tests
+    bool useExes = false;
+
     // kind of output to generate
     TestOutputMode outputMode = TestOutputMode::Default;
 
@@ -114,6 +118,8 @@ struct Options
 
 // Globals
 
+static const Options g_defaultOptions;
+
 Options g_options;
 Dictionary<String, TestCategory*> g_testCategories;
 TestCategory* g_defaultTestCategory;
@@ -125,14 +131,16 @@ TestCategory* findTestCategory(String const& name);
 /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!! Functions !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */
 
 
-Result parseOptions(int* argc, char** argv)
+Result parseOptions(int argc, char** argv, Slang::WriterHelper stdError)
 {
-    int argCount = *argc;
+    g_options = g_defaultOptions;
+
+    List<const char*> positionalArgs;
+    
+    int argCount = argc;
     char const* const* argCursor = argv;
     char const* const* argEnd = argCursor + argCount;
 
-    char const** writeCursor = (char const**) argv;
-
     // first argument is the application name
     if( argCursor != argEnd )
     {
@@ -145,7 +153,7 @@ Result parseOptions(int* argc, char** argv)
         char const* arg = *argCursor++;
         if( arg[0] != '-' )
         {
-            *writeCursor++ = arg;
+            positionalArgs.Add(arg);
             continue;
         }
 
@@ -153,8 +161,7 @@ Result parseOptions(int* argc, char** argv)
         {
             while(argCursor != argEnd)
             {
-                char const* nxtArg = *argCursor++;
-                *writeCursor++ = nxtArg;
+                positionalArgs.Add(*argCursor++);
             }
             break;
         }
@@ -163,11 +170,15 @@ Result parseOptions(int* argc, char** argv)
         {
             if( argCursor == argEnd )
             {
-                fprintf(stderr, "error: expected operand for '%s'\n", arg);
+                stdError.print("error: expected operand for '%s'\n", arg);
                 return SLANG_FAIL;
             }
             g_options.binDir = *argCursor++;
         }
+        else if (strcmp(arg, "-useexes") == 0)
+        {
+            g_options.useExes = true;
+        }
         else if( strcmp(arg, "-v") == 0 )
         {
             g_options.shouldBeVerbose = true;
@@ -188,7 +199,7 @@ Result parseOptions(int* argc, char** argv)
         {
             if( argCursor == argEnd )
             {
-                fprintf(stderr, "error: expected operand for '%s'\n", arg);
+                stdError.print("error: expected operand for '%s'\n", arg);
 				return SLANG_FAIL;
             }
             argCursor++;
@@ -198,7 +209,7 @@ Result parseOptions(int* argc, char** argv)
         {
             if( argCursor == argEnd )
             {
-                fprintf(stderr, "error: expected operand for '%s'\n", arg);
+                stdError.print("error: expected operand for '%s'\n", arg);
 				return SLANG_FAIL;
             }
             argCursor++;
@@ -230,7 +241,7 @@ Result parseOptions(int* argc, char** argv)
         {
             if( argCursor == argEnd )
             {
-                fprintf(stderr, "error: expected operand for '%s'\n", arg);
+                stdError.print("error: expected operand for '%s'\n", arg);
                 return SLANG_FAIL;
             }
             auto category = findTestCategory(*argCursor++);
@@ -243,7 +254,7 @@ Result parseOptions(int* argc, char** argv)
         {
             if( argCursor == argEnd )
             {
-                fprintf(stderr, "error: expected operand for '%s'\n", arg);
+                stdError.print("error: expected operand for '%s'\n", arg);
                 return SLANG_FAIL;
             }
             auto category = findTestCategory(*argCursor++);
@@ -256,7 +267,7 @@ Result parseOptions(int* argc, char** argv)
         {
             if (argCursor == argEnd)
             {
-                fprintf(stderr, "error: expecting an api expression (eg 'vk+dx12' or '+dx11') '%s'\n", arg);
+                stdError.print("error: expecting an api expression (eg 'vk+dx12' or '+dx11') '%s'\n", arg);
                 return SLANG_FAIL;
             }
             const char* apiList = *argCursor++;
@@ -264,7 +275,7 @@ Result parseOptions(int* argc, char** argv)
             SlangResult res = RenderApiUtil::parseApiFlags(UnownedStringSlice(apiList), g_options.enabledApis, &g_options.enabledApis);
             if (SLANG_FAILED(res))
             {
-                fprintf(stderr, "error: unable to parse api expression '%s'\n", apiList);
+                stdError.print("error: unable to parse api expression '%s'\n", apiList);
                 return res;
             }
         }
@@ -272,7 +283,7 @@ Result parseOptions(int* argc, char** argv)
         {
             if (argCursor == argEnd)
             {
-                fprintf(stderr, "error: expected an api expression (eg 'vk+dx12' or '+dx11') '%s'\n", arg);
+                stdError.print("error: expected an api expression (eg 'vk+dx12' or '+dx11') '%s'\n", arg);
                 return SLANG_FAIL;
             }
             const char* apiList = *argCursor++;
@@ -280,13 +291,13 @@ Result parseOptions(int* argc, char** argv)
             SlangResult res = RenderApiUtil::parseApiFlags(UnownedStringSlice(apiList), g_options.synthesizedTestApis, &g_options.synthesizedTestApis);
             if (SLANG_FAILED(res))
             {
-                fprintf(stderr, "error: unable to parse api expression '%s'\n", apiList);
+                stdError.print("error: unable to parse api expression '%s'\n", apiList);
                 return res;
             }
         }
         else
         {
-            fprintf(stderr, "unknown option '%s'\n", arg);
+            stdError.print("unknown option '%s'\n", arg);
             return SLANG_FAIL;
         }
     }
@@ -301,25 +312,21 @@ Result parseOptions(int* argc, char** argv)
         g_options.synthesizedTestApis &= g_options.enabledApis;
     }
 
-    // any arguments left over were positional arguments
-    argCount = (int)((char**)writeCursor - argv);
-    argCursor = argv;
-    argEnd = argCursor + argCount;
 
-    // first positional argument is a "filter" to apply
-    if( argCursor != argEnd )
+    // first positional argument is source shader path
+    if (positionalArgs.Count())
     {
-        g_options.testPrefix = *argCursor++;
+        g_options.testPrefix = positionalArgs[0];
+        positionalArgs.RemoveAt(0);
     }
 
     // any remaining arguments represent an error
-    if(argCursor != argEnd)
+    if (positionalArgs.Count() != 0)
     {
-        fprintf(stderr, "unexpected arguments\n");
+        stdError.print("unexpected arguments\n");
         return SLANG_FAIL;
     }
 
-    *argc = 0;
 	return SLANG_OK;
 }
 
@@ -638,7 +645,48 @@ OSError spawnAndWait(TestContext* context, const String& testPath, OSProcessSpaw
         String commandLine = spawner.getCommandLine();
         context->messageFormat(TestMessageType::Info, "%s\n", commandLine.begin());
     }
-    
+
+    if (!context->m_useExes)
+    {
+        String exeName = Path::GetFileNameWithoutEXT(spawner.executableName_);
+
+        auto func = context->getInnerMainFunc(String(g_options.binDir), exeName);
+        if (func)
+        {
+            StringBuilder stdErrorString;
+            StringBuilder stdOutString;
+
+            // Say static so not released
+            StringWriter stdError(&stdErrorString, WriterFlag::IsConsole | WriterFlag::IsStatic);
+            StringWriter stdOut(&stdOutString, WriterFlag::IsConsole | WriterFlag::IsStatic);
+
+            AppContext appContext;
+            appContext.setWriter(SLANG_WRITER_CHANNEL_STD_ERROR, &stdError);
+            appContext.setWriter(SLANG_WRITER_CHANNEL_STD_OUTPUT, &stdOut);
+
+            if (exeName == "slangc")
+            {
+                appContext.setWriter(SLANG_WRITER_CHANNEL_DIAGNOSTIC, &stdError);
+            }
+            appContext.setReplaceWriterFlagsAll();
+
+            List<const char*> args;
+            args.Add(exeName.Buffer());
+            for (int i = 0; i < int(spawner.argumentList_.Count()); ++i)
+            {
+                args.Add(spawner.argumentList_[i].Buffer());
+            }
+
+            SlangResult res = func(&appContext, context->getSession(), int(args.Count()), args.begin());
+
+            spawner.standardError_ = stdErrorString;
+            spawner.standardOutput_ = stdOutString;
+            spawner.resultCode_ = AppContext::getReturnCode(res);
+
+            return kOSError_None;
+        }
+    }
+
     OSError err = spawner.spawnAndWaitForCompletion();
     if (err != kOSError_None)
     {
@@ -1214,7 +1262,7 @@ TestResult runGLSLComparisonTest(TestContext* context, TestInput& input)
 }
 
 
-TestResult runComputeComparisonImpl(TestContext* context, TestInput& input, const char * langOption)
+TestResult runComputeComparisonImpl(TestContext* context, TestInput& input, const char *const* langOpts, size_t numLangOpts)
 {
 	// TODO: delete any existing files at the output path(s) to avoid stale outputs leading to a false pass
 	auto filePath999 = input.filePath;
@@ -1236,7 +1284,10 @@ TestResult runComputeComparisonImpl(TestContext* context, TestInput& input, cons
 		spawner.pushArgument(arg);
 	}
 
-	spawner.pushArgument(langOption);
+    for (int i = 0; i < int(numLangOpts); ++i)
+    {
+        spawner.pushArgument(langOpts[i]);
+    }
 	spawner.pushArgument("-o");
     auto actualOutputFile = outputStem + ".actual.txt";
 	spawner.pushArgument(actualOutputFile);
@@ -1309,22 +1360,25 @@ TestResult runComputeComparisonImpl(TestContext* context, TestInput& input, cons
 
 TestResult runSlangComputeComparisonTest(TestContext* context, TestInput& input)
 {
-	return runComputeComparisonImpl(context, input, "-slang -compute");
+    const char* langOpts[] = { "-slang", "-compute" };
+	return runComputeComparisonImpl(context, input, langOpts, SLANG_COUNT_OF(langOpts));
 }
 
 TestResult runSlangComputeComparisonTestEx(TestContext* context, TestInput& input)
 {
-	return runComputeComparisonImpl(context, input, "");
+	return runComputeComparisonImpl(context, input, nullptr, 0);
 }
 
 TestResult runHLSLComputeTest(TestContext* context, TestInput& input)
 {
-    return runComputeComparisonImpl(context, input, "-hlsl-rewrite -compute");
+    const char* langOpts[] = { "--hlsl-rewrite", "-compute" };
+    return runComputeComparisonImpl(context, input, langOpts, SLANG_COUNT_OF(langOpts));
 }
 
 TestResult runSlangRenderComputeComparisonTest(TestContext* context, TestInput& input)
 {
-    return runComputeComparisonImpl(context, input, "-slang -gcompute");
+    const char* langOpts[] = { "-slang", "-gcompute" };
+    return runComputeComparisonImpl(context, input, langOpts, SLANG_COUNT_OF(langOpts));
 }
 
 TestResult doRenderComparisonTestRun(TestContext* context, TestInput& input, char const* langOption, char const* outputKind, String* outOutput)
@@ -1913,9 +1967,11 @@ void runTestsInDirectory(
 //
 
 int main(
-    int		argc,
-    char**	argv)
+    int		argcIn,
+    char**	argvIn)
 {
+    AppContext::initDefault();
+
     // Set up our test categories here
 
     auto fullTestCategory = addTestCategory("full", nullptr);
@@ -1939,7 +1995,7 @@ int main(
 
     //
 
-    if (SLANG_FAILED(parseOptions(&argc, argv)))
+    if (SLANG_FAILED(parseOptions(argcIn, argvIn, AppContext::getStdError())))
     {
     	// Return exit code with error
     	return 1;
@@ -1968,6 +2024,7 @@ int main(
     }
 
     context.m_dumpOutputOnFailure = g_options.dumpOutputOnFailure;
+    context.m_useExes = g_options.useExes;
     context.m_isVerbose = g_options.shouldBeVerbose;
 
     { 
diff --git a/tools/slang-test/os.cpp b/tools/slang-test/os.cpp
index 8cb49248c..9ecf50557 100644
--- a/tools/slang-test/os.cpp
+++ b/tools/slang-test/os.cpp
@@ -222,6 +222,8 @@ void OSProcessSpawner::pushArgument(
     // TODO(tfoley): handle cases where arguments need some escaping
     commandLine_.Append(" ");
     commandLine_.Append(argument);
+
+    argumentList_.Add(argument);
 }
 
 Slang::String OSProcessSpawner::getCommandLine()
@@ -480,7 +482,7 @@ void OSProcessSpawner::pushExecutableName(
     Slang::String executableName)
 {
     executableName_ = executableName;
-    pushArgument(executableName);
+    arguments_.Add(executableName);
     isExecutablePath_ = false;
 }
 
@@ -488,7 +490,7 @@ void OSProcessSpawner::pushExecutablePath(
     Slang::String executablePath)
 {
     executableName_ = executablePath;
-    pushArgument(executablePath);
+    arguments_.Add(executablePath);
     isExecutablePath_ = true;
 }
 
@@ -496,6 +498,7 @@ void OSProcessSpawner::pushArgument(
     Slang::String argument)
 {
     arguments_.Add(argument);
+    argumentList_.Add(argument);
 }
 
 Slang::String OSProcessSpawner::getCommandLine()
@@ -507,7 +510,6 @@ Slang::String OSProcessSpawner::getCommandLine()
     {
         if(ii != 0) sb << " ";
         sb << arguments_[ii];
-
     }
     return sb.ProduceString();
 }
diff --git a/tools/slang-test/os.h b/tools/slang-test/os.h
index 3466f818c..3e6d93942 100644
--- a/tools/slang-test/os.h
+++ b/tools/slang-test/os.h
@@ -178,8 +178,11 @@ struct OSProcessSpawner
     Slang::StringBuilder commandLine_;
 #else
     Slang::List<Slang::String>  arguments_;
-
 #endif
+
+    // Only holds the argumements in order
+    Slang::List<Slang::String>  argumentList_;          
+
     // Is the executable specified by path, rather than just by name?
     bool isExecutablePath_;
 };
diff --git a/tools/slang-test/test-context.cpp b/tools/slang-test/test-context.cpp
index b94435a8e..efbbb4f65 100644
--- a/tools/slang-test/test-context.cpp
+++ b/tools/slang-test/test-context.cpp
@@ -82,6 +82,7 @@ TestContext::TestContext() :
     m_inTest = false;
     m_dumpOutputOnFailure = false;
     m_isVerbose = false;
+    m_useExes = false;
 
     m_session = nullptr;
 }
@@ -480,6 +481,35 @@ bool TestContext::didAllSucceed() const
     return m_passedTestCount == (m_totalTestCount - m_ignoredTestCount);
 }
 
+TestContext::InnerMainFunc TestContext::getInnerMainFunc(const String& dirPath, const String& name)
+{
+    {
+        SharedLibraryTool* tool = m_sharedLibTools.TryGetValue(name);
+        if (tool)
+        {
+            return tool->m_func;
+        }
+    }
+
+    StringBuilder sharedLibToolBuilder;
+    sharedLibToolBuilder.append(name);
+    sharedLibToolBuilder.append("-shared-library");
+
+    StringBuilder builder;
+    SharedLibrary::appendPlatformFileName(sharedLibToolBuilder.getUnownedSlice(), builder);
+    String path = Path::Combine(dirPath, builder);
+
+    SharedLibraryTool tool = {};
+
+    if (SLANG_SUCCEEDED(SharedLibrary::loadWithPlatformFilename(path.begin(), tool.m_sharedLibrary)))
+    {
+        tool.m_func = (InnerMainFunc)SharedLibrary::findFuncByName(tool.m_sharedLibrary, "innerMain");
+    }
+
+    m_sharedLibTools.Add(name, tool);
+    return tool.m_func;
+}
+
 void TestContext::outputSummary()
 {
     auto passCount = m_passedTestCount;
diff --git a/tools/slang-test/test-context.h b/tools/slang-test/test-context.h
index a81473efa..cf047cec7 100644
--- a/tools/slang-test/test-context.h
+++ b/tools/slang-test/test-context.h
@@ -1,6 +1,10 @@
 // test-context.h
 
 #include "../../source/core/slang-string-util.h"
+#include "../../source/core/platform.h"
+#include "../../source/core/slang-app-context.h"
+#include "../../source/core/dictionary.h"
+
 
 #define SLANG_CHECK(x) TestContext::get()->addResultWithLocation((x), #x, __FILE__, __LINE__); 
 
@@ -54,6 +58,7 @@ enum class TestMessageType
 class TestContext
 {
     public:
+    typedef SlangResult(*InnerMainFunc)(Slang::AppContext* appContext, SlangSession* session, int argc, const char*const* argv);
 
     struct TestInfo
     {
@@ -124,6 +129,9 @@ class TestContext
         /// Returns true if all run tests succeeded
     bool didAllSucceed() const;
 
+        /// Get the InnerMain function from a shared library tool
+    InnerMainFunc getInnerMainFunc(const Slang::String& dirPath, const Slang::String& name);
+
         /// Get the slang session
     SlangSession* getSession() const { return m_session;  }
 
@@ -134,7 +142,6 @@ class TestContext
         /// Dtor
     ~TestContext();
 
-
     static TestResult combine(TestResult a, TestResult b) { return (a > b) ? a : b; }
 
     static TestContext* get() { return s_context; }
@@ -155,9 +162,17 @@ class TestContext
     bool m_dumpOutputOnFailure;
     bool m_isVerbose;
 
+    bool m_useExes;
+
     void outputSummary();
 
 protected:
+    struct SharedLibraryTool
+    {
+        Slang::SharedLibrary::Handle m_sharedLibrary;
+        InnerMainFunc m_func;
+    };
+
     void _addResult(const TestInfo& info);
     
     Slang::StringBuilder m_currentMessage;
@@ -168,7 +183,9 @@ protected:
     bool m_inTest;
 
     SlangSession* m_session;
-    
+
+    Slang::Dictionary<Slang::String, SharedLibraryTool> m_sharedLibTools;
+
     static TestContext* s_context;
 };