From 604e0fc8dc859e5d3954fd1e074e5288659cb439 Mon Sep 17 00:00:00 2001 From: Tim Foley Date: Wed, 14 Jun 2017 08:23:49 -0700 Subject: Testing: Add support for multiple tests per input file. Now if there are multiple tests listed for an input file like `foo.slang`, they will be output as: passed test: 'foo.slang' passed test: 'foo.slang.1' passed test: 'foo.slang.2' This isn't a perfect solution, but it should work well enough for now. This change doesn't make any tests actually use the new capability. --- tools/slang-test/main.cpp | 96 +++++++++++++++++++++++++++++------------------ 1 file changed, 59 insertions(+), 37 deletions(-) diff --git a/tools/slang-test/main.cpp b/tools/slang-test/main.cpp index ffb0f2951..ed419f946 100644 --- a/tools/slang-test/main.cpp +++ b/tools/slang-test/main.cpp @@ -399,8 +399,19 @@ String getOutput(OSProcessSpawner& spawner) struct TestInput { + // Path to the input file for the test String filePath; + + // Prefix for the path that test output should write to + // (usually the same as `filePath`, but will differ when + // we run multiple tests out of the same file) + String outputStem; + + // Arguments for the test (usually to be interpreted + // as command line args) TestOptions const* testOptions; + + // The list of tests that will be run on this file FileTestList const* testList; }; @@ -410,26 +421,27 @@ TestResult runSimpleTest(TestInput& input) { // need to execute the stand-alone Slang compiler on the file, and compare its output to what we expect - auto filePath = input.filePath; + auto filePath999 = input.filePath; + auto outputStem = input.outputStem; OSProcessSpawner spawner; spawner.pushExecutableName(String(options.binDir) + "slangc.exe"); - spawner.pushArgument(filePath); + spawner.pushArgument(filePath999); for( auto arg : input.testOptions->args ) { spawner.pushArgument(arg); } - if (spawnAndWait(filePath, spawner) != kOSError_None) + if (spawnAndWait(outputStem, spawner) != kOSError_None) { return kTestResult_Fail; } String actualOutput = getOutput(spawner); - String expectedOutputPath = filePath + ".expected"; + String expectedOutputPath = outputStem + ".expected"; String expectedOutput; try { @@ -459,7 +471,7 @@ TestResult runSimpleTest(TestInput& input) // diagnose the problem. if (result == kTestResult_Fail) { - String actualOutputPath = filePath + ".actual"; + String actualOutputPath = outputStem + ".actual"; CoreLib::IO::File::WriteAllText(actualOutputPath, actualOutput); } @@ -469,11 +481,12 @@ TestResult runSimpleTest(TestInput& input) #ifdef SLANG_TEST_SUPPORT_HLSL TestResult generateHLSLBaseline(TestInput& input) { - auto filePath = input.filePath; + auto filePath999 = input.filePath; + auto outputStem = input.outputStem; OSProcessSpawner spawner; spawner.pushExecutableName(String(options.binDir) + "slangc.exe"); - spawner.pushArgument(filePath); + spawner.pushArgument(filePath999); for( auto arg : input.testOptions->args ) { @@ -485,13 +498,13 @@ TestResult generateHLSLBaseline(TestInput& input) spawner.pushArgument("-pass-through"); spawner.pushArgument("fxc"); - if (spawnAndWait(filePath, spawner) != kOSError_None) + if (spawnAndWait(outputStem, spawner) != kOSError_None) { return kTestResult_Fail; } String expectedOutput = getOutput(spawner); - String expectedOutputPath = filePath + ".expected"; + String expectedOutputPath = outputStem + ".expected"; try { CoreLib::IO::File::WriteAllText(expectedOutputPath, expectedOutput); @@ -505,10 +518,11 @@ TestResult generateHLSLBaseline(TestInput& input) TestResult runHLSLComparisonTest(TestInput& input) { - auto filePath = input.filePath; + auto filePath999 = input.filePath; + auto outputStem = input.outputStem; // We will use the Microsoft compiler to generate out expected output here - String expectedOutputPath = filePath + ".expected"; + String expectedOutputPath = outputStem + ".expected"; // Generate the expected output using standard HLSL compiler generateHLSLBaseline(input); @@ -518,7 +532,7 @@ TestResult runHLSLComparisonTest(TestInput& input) OSProcessSpawner spawner; spawner.pushExecutableName(String(options.binDir) + "slangc.exe"); - spawner.pushArgument(filePath); + spawner.pushArgument(filePath999); for( auto arg : input.testOptions->args ) { @@ -532,7 +546,7 @@ TestResult runHLSLComparisonTest(TestInput& input) spawner.pushArgument("-target"); spawner.pushArgument("dxbc-assembly"); - if (spawnAndWait(filePath, spawner) != kOSError_None) + if (spawnAndWait(outputStem, spawner) != kOSError_None) { return kTestResult_Fail; } @@ -587,7 +601,7 @@ TestResult runHLSLComparisonTest(TestInput& input) // diagnose the problem. if (result == kTestResult_Fail) { - String actualOutputPath = filePath + ".actual"; + String actualOutputPath = outputStem + ".actual"; CoreLib::IO::File::WriteAllText(actualOutputPath, actualOutput); } @@ -602,12 +616,13 @@ TestResult doGLSLComparisonTestRun( char const* outputKind, String* outOutput) { - auto filePath = input.filePath; + auto filePath999 = input.filePath; + auto outputStem = input.outputStem; OSProcessSpawner spawner; spawner.pushExecutableName(String(options.binDir) + "slangc.exe"); - spawner.pushArgument(filePath); + spawner.pushArgument(filePath999); if( langDefine ) { @@ -631,7 +646,7 @@ TestResult doGLSLComparisonTestRun( spawner.pushArgument(arg); } - if (spawnAndWait(filePath, spawner) != kOSError_None) + if (spawnAndWait(outputStem, spawner) != kOSError_None) { return kTestResult_Fail; } @@ -651,7 +666,7 @@ TestResult doGLSLComparisonTestRun( outputBuilder.Append(standardOuptut); outputBuilder.Append("}\n"); - String outputPath = filePath + outputKind; + String outputPath = outputStem + outputKind; String output = outputBuilder.ProduceString(); *outOutput = output; @@ -661,7 +676,8 @@ TestResult doGLSLComparisonTestRun( TestResult runGLSLComparisonTest(TestInput& input) { - auto filePath = input.filePath; + auto filePath999 = input.filePath; + auto outputStem = input.outputStem; String expectedOutput; String actualOutput; @@ -669,8 +685,8 @@ TestResult runGLSLComparisonTest(TestInput& input) TestResult hlslResult = doGLSLComparisonTestRun(input, "__GLSL__", "glslang", ".expected", &expectedOutput); TestResult slangResult = doGLSLComparisonTestRun(input, "__SLANG__", nullptr, ".actual", &actualOutput); - CoreLib::IO::File::WriteAllText(filePath + ".expected", expectedOutput); - CoreLib::IO::File::WriteAllText(filePath + ".actual", actualOutput); + CoreLib::IO::File::WriteAllText(outputStem + ".expected", expectedOutput); + CoreLib::IO::File::WriteAllText(outputStem + ".actual", actualOutput); if( hlslResult == kTestResult_Fail ) return kTestResult_Fail; if( slangResult == kTestResult_Fail ) return kTestResult_Fail; @@ -689,12 +705,13 @@ TestResult doRenderComparisonTestRun(TestInput& input, char const* langOption, c { // TODO: delete any existing files at the output path(s) to avoid stale outputs leading to a false pass - auto filePath = input.filePath; + auto filePath999 = input.filePath; + auto outputStem = input.outputStem; OSProcessSpawner spawner; spawner.pushExecutableName(String(options.binDir) + "render-test.exe"); - spawner.pushArgument(filePath); + spawner.pushArgument(filePath999); for( auto arg : input.testOptions->args ) { @@ -703,9 +720,9 @@ TestResult doRenderComparisonTestRun(TestInput& input, char const* langOption, c spawner.pushArgument(langOption); spawner.pushArgument("-o"); - spawner.pushArgument(filePath + outputKind + ".png"); + spawner.pushArgument(outputStem + outputKind + ".png"); - if (spawnAndWait(filePath, spawner) != kOSError_None) + if (spawnAndWait(outputStem, spawner) != kOSError_None) { return kTestResult_Fail; } @@ -725,7 +742,7 @@ TestResult doRenderComparisonTestRun(TestInput& input, char const* langOption, c outputBuilder.Append(standardOuptut); outputBuilder.Append("}\n"); - String outputPath = filePath + outputKind; + String outputPath = outputStem + outputKind; String output = outputBuilder.ProduceString(); *outOutput = output; @@ -804,16 +821,17 @@ TestResult runHLSLRenderComparisonTestImpl( char const* expectedArg, char const* actualArg) { - auto filePath = input.filePath; + auto filePath999 = input.filePath; + auto outputStem = input.outputStem; String expectedOutput; String actualOutput; TestResult hlslResult = doRenderComparisonTestRun(input, expectedArg, ".expected", &expectedOutput); - TestResult slangResult = doRenderComparisonTestRun(input, actualArg, ".actual", &actualOutput); + TestResult slangResult = doRenderComparisonTestRun(input, actualArg, ".actual", &actualOutput); - CoreLib::IO::File::WriteAllText(filePath + ".expected", expectedOutput); - CoreLib::IO::File::WriteAllText(filePath + ".actual", actualOutput); + CoreLib::IO::File::WriteAllText(outputStem + ".expected", expectedOutput); + CoreLib::IO::File::WriteAllText(outputStem + ".actual", actualOutput); if( hlslResult == kTestResult_Fail ) return kTestResult_Fail; if( slangResult == kTestResult_Fail ) return kTestResult_Fail; @@ -825,7 +843,7 @@ TestResult runHLSLRenderComparisonTestImpl( // Next do an image comparison on the expected output images! - TestResult imageCompareResult = doImageComparison(filePath); + TestResult imageCompareResult = doImageComparison(outputStem); if(imageCompareResult != kTestResult_Pass) return imageCompareResult; @@ -849,6 +867,7 @@ TestResult runHLSLAndGLSLComparisonTest(TestInput& input) TestResult runTest( String const& filePath, + String const& outputStem, TestOptions const& testOptions, FileTestList const& testList) { @@ -874,6 +893,7 @@ TestResult runTest( TestInput testInput; testInput.filePath = filePath; + testInput.outputStem = outputStem; testInput.testOptions = &testOptions; testInput.testList = &testList; @@ -924,7 +944,13 @@ void runTestsOnFile( int subTestIndex = subTestCount++; - TestResult result = runTest(filePath, tt, testList); + String outputStem = filePath; + if(subTestIndex != 0) + { + outputStem = outputStem + "." + String(subTestIndex); + } + + TestResult result = runTest(filePath, outputStem, tt, testList); if(result == kTestResult_Ignored) return; @@ -939,11 +965,7 @@ void runTestsOnFile( context->failedTestCount++; } - printf(" test: '%S'", filePath.ToWString()); - if( subTestIndex ) - { - printf(" subtest:%d", subTestIndex); - } + printf(" test: '%S'", outputStem.ToWString()); printf("\n"); } } -- cgit v1.2.3 From 34c7cdc68fd5b7b8132250c4de0d562c9ba0adbb Mon Sep 17 00:00:00 2001 From: Tim Foley Date: Wed, 14 Jun 2017 08:35:43 -0700 Subject: Testing: Adding binding-generation tests for some GLSL shaders Most of the Vulkan GLSL shaders in the `sascha-willems` corpus use completely regular parameter bindings, e.g.: ``` layout (binding = 0) uniform sampler2D samplerPositionDepth; layout (binding = 1) uniform sampler2D samplerNormal; layout (binding = 2) uniform sampler2D ssaoNoise; ``` With the Slang compiler, we can write this kind of stuff more compactly as: ``` uniform sampler2D samplerPositionDepth; uniform sampler2D samplerNormal; uniform sampler2D ssaoNoise; ``` and get the exact same result (in fact, we will generate output GLSL matching the first example). There are a few spot tests for this in the HLSL case, but I hadn't done anything for GLSL yet. Now that we have the ability to specify multiple tests to run on each input file, it was easy enough to go in and tweak some of these files to be usable as binding-generation tests. I didn't ammend all of the GLSL shaders for two reasons: 1. Not all of the shaders will work with completely automatic binding generation done on a per-file basis. In some cases, the parameter layout needs to consider multiple files (and the order in which the files are supplied could matter). This is probably best handled with more directed tests. 2. There is going to be a ton of duplication if I just have 100s of tests that all confirm that, yes, the Slang compiler can count vertex inputs starting from zero. These shaders aren't really presenting a whole lot of unique cases to work with. That said, a level of confidence greater than zero is always better than zero confidence. --- .gitignore | 12 +++-------- tests/glsl/sascha-willems/base/textoverlay.frag | 13 +++++++++--- tests/glsl/sascha-willems/base/textoverlay.vert | 13 +++++++++--- tests/glsl/sascha-willems/bloom/phongpass.vert | 27 ++++++++++++++++--------- tests/glsl/sascha-willems/ssao/ssao.frag | 23 +++++++++++++-------- 5 files changed, 55 insertions(+), 33 deletions(-) diff --git a/.gitignore b/.gitignore index a0a57b5aa..e62d06013 100644 --- a/.gitignore +++ b/.gitignore @@ -10,15 +10,9 @@ intermediate/ # files generated by test runner *.actual -*.hlsl.expected -*.fx.expected -*.glsl.expected -*.vert.expected -*.frag.expected -*.geom.expected -*.tesc.expected -*.tese.expected -*.comp.expected +*.expected +!*.slang.expected +!*.slang.*.expected *.expected.png *.actual.png diff --git a/tests/glsl/sascha-willems/base/textoverlay.frag b/tests/glsl/sascha-willems/base/textoverlay.frag index e5dbb08de..c77198765 100644 --- a/tests/glsl/sascha-willems/base/textoverlay.frag +++ b/tests/glsl/sascha-willems/base/textoverlay.frag @@ -1,11 +1,18 @@ #version 450 core //TEST:COMPARE_GLSL: +//TEST:COMPARE_GLSL:-DBINDING -layout (location = 0) in vec2 inUV; +#if defined(__SLANG__) && defined(BINDING) +#define LAYOUT(X) /* empty */ +#else +#define LAYOUT(X) layout(X) +#endif -layout (binding = 0) uniform sampler2D samplerFont; +LAYOUT(location = 0) in vec2 inUV; -layout (location = 0) out vec4 outFragColor; +LAYOUT(binding = 0) uniform sampler2D samplerFont; + +LAYOUT(location = 0) out vec4 outFragColor; void main(void) { diff --git a/tests/glsl/sascha-willems/base/textoverlay.vert b/tests/glsl/sascha-willems/base/textoverlay.vert index 8a20fd8b2..5d5dacab6 100644 --- a/tests/glsl/sascha-willems/base/textoverlay.vert +++ b/tests/glsl/sascha-willems/base/textoverlay.vert @@ -1,10 +1,17 @@ #version 450 core //TEST:COMPARE_GLSL: +//TEST:COMPARE_GLSL:-DBINDING -layout (location = 0) in vec2 inPos; -layout (location = 1) in vec2 inUV; +#if defined(__SLANG__) && defined(BINDING) +#define LAYOUT(X) /* empty */ +#else +#define LAYOUT(X) layout(X) +#endif -layout (location = 0) out vec2 outUV; +LAYOUT(location = 0) in vec2 inPos; +LAYOUT(location = 1) in vec2 inUV; + +LAYOUT(location = 0) out vec2 outUV; out gl_PerVertex { diff --git a/tests/glsl/sascha-willems/bloom/phongpass.vert b/tests/glsl/sascha-willems/bloom/phongpass.vert index ac0a77ab9..a3c77cb48 100644 --- a/tests/glsl/sascha-willems/bloom/phongpass.vert +++ b/tests/glsl/sascha-willems/bloom/phongpass.vert @@ -1,26 +1,33 @@ #version 450 //TEST:COMPARE_GLSL: +//TEST:COMPARE_GLSL:-DBINDING + +#if defined(__SLANG__) && defined(BINDING) +#define LAYOUT(X) /* empty */ +#else +#define LAYOUT(X) layout(X) +#endif #extension GL_ARB_separate_shader_objects : enable #extension GL_ARB_shading_language_420pack : enable -layout (location = 0) in vec4 inPos; -layout (location = 1) in vec2 inUV; -layout (location = 2) in vec3 inColor; -layout (location = 3) in vec3 inNormal; +LAYOUT(location = 0) in vec4 inPos; +LAYOUT(location = 1) in vec2 inUV; +LAYOUT(location = 2) in vec3 inColor; +LAYOUT(location = 3) in vec3 inNormal; -layout (binding = 0) uniform UBO +LAYOUT(binding = 0) uniform UBO { mat4 projection; mat4 view; mat4 model; } ubo; -layout (location = 0) out vec3 outNormal; -layout (location = 1) out vec2 outUV; -layout (location = 2) out vec3 outColor; -layout (location = 3) out vec3 outViewVec; -layout (location = 4) out vec3 outLightVec; +LAYOUT(location = 0) out vec3 outNormal; +LAYOUT(location = 1) out vec2 outUV; +LAYOUT(location = 2) out vec3 outColor; +LAYOUT(location = 3) out vec3 outViewVec; +LAYOUT(location = 4) out vec3 outLightVec; out gl_PerVertex { diff --git a/tests/glsl/sascha-willems/ssao/ssao.frag b/tests/glsl/sascha-willems/ssao/ssao.frag index cdcbfd3ec..ef0035013 100644 --- a/tests/glsl/sascha-willems/ssao/ssao.frag +++ b/tests/glsl/sascha-willems/ssao/ssao.frag @@ -1,29 +1,36 @@ -//TEST:COMPARE_GLSL: #version 450 +//TEST:COMPARE_GLSL: +//TEST:COMPARE_GLSL:-DBINDING + +#if defined(__SLANG__) && defined(BINDING) +#define LAYOUT(X) /* empty */ +#else +#define LAYOUT(X) layout(X) +#endif #extension GL_ARB_separate_shader_objects : enable #extension GL_ARB_shading_language_420pack : enable -layout (binding = 0) uniform sampler2D samplerPositionDepth; -layout (binding = 1) uniform sampler2D samplerNormal; -layout (binding = 2) uniform sampler2D ssaoNoise; +LAYOUT(binding = 0) uniform sampler2D samplerPositionDepth; +LAYOUT(binding = 1) uniform sampler2D samplerNormal; +LAYOUT(binding = 2) uniform sampler2D ssaoNoise; layout (constant_id = 0) const int SSAO_KERNEL_SIZE = 64; layout (constant_id = 1) const float SSAO_RADIUS = 0.5; -layout (binding = 3) uniform UBOSSAOKernel +LAYOUT(binding = 3) uniform UBOSSAOKernel { vec4 samples[SSAO_KERNEL_SIZE]; } uboSSAOKernel; -layout (binding = 4) uniform UBO +LAYOUT(binding = 4) uniform UBO { mat4 projection; } ubo; -layout (location = 0) in vec2 inUV; +LAYOUT(location = 0) in vec2 inUV; -layout (location = 0) out float outFragColor; +LAYOUT(location = 0) out float outFragColor; void main() { -- cgit v1.2.3