From 7d3bfe403362b294cc2a1f2607d51dfcd447aafd Mon Sep 17 00:00:00 2001 From: Tim Foley Date: Mon, 26 Jun 2017 09:32:40 -0700 Subject: Replace "auto-import" with `#import` Right now `#import` only differs from `#include` in that it takes a string literal for a file name instead of a raw identifier (to which `.slang` gets appended). The next step is to make `#import` respect preprocessor state, while `__import` doesn't. --- tests/render/auto-import.hlsl | 147 -------------------------------------- tests/render/auto-import.slang.h | 21 ------ tests/render/pound-import.hlsl | 147 ++++++++++++++++++++++++++++++++++++++ tests/render/pound-import.slang.h | 21 ++++++ 4 files changed, 168 insertions(+), 168 deletions(-) delete mode 100644 tests/render/auto-import.hlsl delete mode 100644 tests/render/auto-import.slang.h create mode 100644 tests/render/pound-import.hlsl create mode 100644 tests/render/pound-import.slang.h (limited to 'tests') diff --git a/tests/render/auto-import.hlsl b/tests/render/auto-import.hlsl deleted file mode 100644 index 588ebc612..000000000 --- a/tests/render/auto-import.hlsl +++ /dev/null @@ -1,147 +0,0 @@ -//TEST(smoke,render):COMPARE_HLSL_GLSL_RENDER: -xslang -auto-import-dir -xslang tests/render/ - -// This is a basic test case for cross-compilation behavior. -// -// We will define distinct HLSL and GLSL entry points, -// but the two will share a dependency on a file of -// pure Spire code that provides the actual shading logic. - - -// Pull in Spire code depdendency using extended syntax: -#include "auto-import.slang.h" - -#if defined(__HLSL__) - -cbuffer Uniforms -{ - float4x4 modelViewProjection; -}; - -struct AssembledVertex -{ - float3 position; - float3 color; -}; - -struct CoarseVertex -{ - float3 color; -}; - -struct Fragment -{ - float4 color; -}; - -// Vertex Shader - -struct VertexStageInput -{ - AssembledVertex assembledVertex : A; -}; - -struct VertexStageOutput -{ - CoarseVertex coarseVertex : CoarseVertex; - float4 sv_position : SV_Position; -}; - -VertexStageOutput vertexMain(VertexStageInput input) -{ - VertexStageOutput output; - - float3 position = input.assembledVertex.position; - float3 color = input.assembledVertex.color; - - output.coarseVertex.color = color; - output.sv_position = mul(modelViewProjection, float4(position, 1.0)); - - return output; - -} - -// Fragment Shader - -struct FragmentStageInput -{ - CoarseVertex coarseVertex : CoarseVertex; -}; - -struct FragmentStageOutput -{ - Fragment fragment : SV_Target; -}; - -FragmentStageOutput fragmentMain(FragmentStageInput input) -{ - FragmentStageOutput output; - - float3 color = input.coarseVertex.color; - - color = transformColor(color); - - output.fragment.color = float4(color, 1.0); - - return output; -} - -#elif defined(__GLSL__) - -#version 420 - -uniform Uniforms -{ - mat4x4 modelViewProjection; -}; - -#define ASSEMBLED_VERTEX(QUAL) \ - /* */ - -#define V2F(QUAL) \ - QUAL vec3 coarse_color; \ - /* */ - -// Vertex Shader - -#ifdef __GLSL_VERTEX__ - -layout(location = 0) -in vec3 assembled_position; - -layout(location = 1) -in vec3 assembled_color; - -V2F(out) - -void main() -{ - vec3 position = assembled_position; - vec3 color = assembled_color; - - coarse_color = color; -// gl_Position = modelViewProjection * vec4(position, 1.0); - gl_Position = vec4(position, 1.0) * modelViewProjection; -} - -#endif - -#ifdef __GLSL_FRAGMENT__ - -V2F(in) - -layout(location = 0) -out vec4 fragment_color; - -void main() -{ - vec3 color = coarse_color; - - color = transformColor(color); - - fragment_color = vec4(color, 1.0); -} - - -#endif - -#endif diff --git a/tests/render/auto-import.slang.h b/tests/render/auto-import.slang.h deleted file mode 100644 index d53005688..000000000 --- a/tests/render/auto-import.slang.h +++ /dev/null @@ -1,21 +0,0 @@ -//TEST_IGNORE_FILE: - -// This file implements the "library" code -// that both the HLSL and GLSL shaders share. -// -// This code is written in Slang (more or less -// just HLSL), and will be translated as needed -// for each of the targets. - -float3 transformColor(float3 color) -{ - float3 result; - - result.x = sin(20.0 * (color.x + color.y)); - result.y = saturate(cos(color.z * 30.0)); - result.z = sin(color.x * color.y * color.z * 100.0); - - result = 0.5 * (result + 1); - - return result; -} \ No newline at end of file diff --git a/tests/render/pound-import.hlsl b/tests/render/pound-import.hlsl new file mode 100644 index 000000000..a9b625fb6 --- /dev/null +++ b/tests/render/pound-import.hlsl @@ -0,0 +1,147 @@ +//TEST(smoke,render):COMPARE_HLSL_GLSL_RENDER: + +// This is a basic test case for cross-compilation behavior. +// +// We will define distinct HLSL and GLSL entry points, +// but the two will share a dependency on a file of +// pure Spire code that provides the actual shading logic. + + +// Pull in Spire code depdendency using extended syntax: +#import "pound-import.slang.h" + +#if defined(__HLSL__) + +cbuffer Uniforms +{ + float4x4 modelViewProjection; +}; + +struct AssembledVertex +{ + float3 position; + float3 color; +}; + +struct CoarseVertex +{ + float3 color; +}; + +struct Fragment +{ + float4 color; +}; + +// Vertex Shader + +struct VertexStageInput +{ + AssembledVertex assembledVertex : A; +}; + +struct VertexStageOutput +{ + CoarseVertex coarseVertex : CoarseVertex; + float4 sv_position : SV_Position; +}; + +VertexStageOutput vertexMain(VertexStageInput input) +{ + VertexStageOutput output; + + float3 position = input.assembledVertex.position; + float3 color = input.assembledVertex.color; + + output.coarseVertex.color = color; + output.sv_position = mul(modelViewProjection, float4(position, 1.0)); + + return output; + +} + +// Fragment Shader + +struct FragmentStageInput +{ + CoarseVertex coarseVertex : CoarseVertex; +}; + +struct FragmentStageOutput +{ + Fragment fragment : SV_Target; +}; + +FragmentStageOutput fragmentMain(FragmentStageInput input) +{ + FragmentStageOutput output; + + float3 color = input.coarseVertex.color; + + color = transformColor(color); + + output.fragment.color = float4(color, 1.0); + + return output; +} + +#elif defined(__GLSL__) + +#version 420 + +uniform Uniforms +{ + mat4x4 modelViewProjection; +}; + +#define ASSEMBLED_VERTEX(QUAL) \ + /* */ + +#define V2F(QUAL) \ + QUAL vec3 coarse_color; \ + /* */ + +// Vertex Shader + +#ifdef __GLSL_VERTEX__ + +layout(location = 0) +in vec3 assembled_position; + +layout(location = 1) +in vec3 assembled_color; + +V2F(out) + +void main() +{ + vec3 position = assembled_position; + vec3 color = assembled_color; + + coarse_color = color; +// gl_Position = modelViewProjection * vec4(position, 1.0); + gl_Position = vec4(position, 1.0) * modelViewProjection; +} + +#endif + +#ifdef __GLSL_FRAGMENT__ + +V2F(in) + +layout(location = 0) +out vec4 fragment_color; + +void main() +{ + vec3 color = coarse_color; + + color = transformColor(color); + + fragment_color = vec4(color, 1.0); +} + + +#endif + +#endif diff --git a/tests/render/pound-import.slang.h b/tests/render/pound-import.slang.h new file mode 100644 index 000000000..d53005688 --- /dev/null +++ b/tests/render/pound-import.slang.h @@ -0,0 +1,21 @@ +//TEST_IGNORE_FILE: + +// This file implements the "library" code +// that both the HLSL and GLSL shaders share. +// +// This code is written in Slang (more or less +// just HLSL), and will be translated as needed +// for each of the targets. + +float3 transformColor(float3 color) +{ + float3 result; + + result.x = sin(20.0 * (color.x + color.y)); + result.y = saturate(cos(color.z * 30.0)); + result.z = sin(color.x * color.y * color.z * 100.0); + + result = 0.5 * (result + 1); + + return result; +} \ No newline at end of file -- cgit v1.2.3 From 6e99b81c98f8c76444563d959536073befc7d8ca Mon Sep 17 00:00:00 2001 From: Tim Foley Date: Mon, 26 Jun 2017 10:11:00 -0700 Subject: Make `#import` work with preprocessor macros With this change, there is now a meaningful semantic difference between `__import` and `#import`. An `__import` compiles the target file in a fresh environment, only providing it any macro definitions passed via command line or API. Any macros defined in the imported file are not made visible at the import site. One can think of an `__import` as a bit like `using namespace` in C++. A `#import` will tokenize the input in the same preprocessor environment as the importing file, and any macros defined along the way will be visible in the parent file. It is a *bit* like a `#include` with two big differences: - The imported code is always parsed as Slang, and as its own module with default flags, etc. (so semantic checks are on even if we are in "rewriter" mode). It is pulled into the outer namespace just as for `__import`. - A given file will only get `#import`ed once for a translation unit, so it behaves a bit like there is an implicit `#pragma once` in the target file --- source/slang/compiler.h | 6 +- source/slang/preprocessor.cpp | 217 +++++++++++++++++++++----------------- source/slang/slang.cpp | 47 +++++---- tests/preprocessor/import.hlsl | 18 ++++ tests/preprocessor/import.slang.h | 12 +++ 5 files changed, 183 insertions(+), 117 deletions(-) create mode 100644 tests/preprocessor/import.hlsl create mode 100644 tests/preprocessor/import.slang.h (limited to 'tests') diff --git a/source/slang/compiler.h b/source/slang/compiler.h index 89bb01738..7c4a17607 100644 --- a/source/slang/compiler.h +++ b/source/slang/compiler.h @@ -247,10 +247,10 @@ namespace Slang String const& source, CodePosition const& loc); - String autoImportModule( + void handlePoundImport( + String const& name, String const& path, - String const& source, - CodePosition const& loc); + TokenList const& tokens); RefPtr findOrImportModule( String const& name, diff --git a/source/slang/preprocessor.cpp b/source/slang/preprocessor.cpp index b93e47ec2..c3cc8cb0a 100644 --- a/source/slang/preprocessor.cpp +++ b/source/slang/preprocessor.cpp @@ -1,12 +1,11 @@ -// Preprocessor.cpp -#include "Preprocessor.h" +// preprocessor.cpp +#include "preprocessor.h" -#include "Diagnostics.h" -#include "Lexer.h" - -// Needed so that we can construct modifier syntax -// to represent GLSL directives -#include "Syntax.h" +#include "compiler.h" +#include "diagnostics.h" +#include "lexer.h" +// Needed so that we can construct modifier syntax to represent GLSL directives +#include "syntax.h" #include @@ -18,15 +17,6 @@ namespace Slang { -// Forward declaration for code provided in `slang.cpp` -// -// TODO: Need an appropriate header for this. -String autoImportModule( - CompileRequest* request, - String const& path, - String const& source, - CodePosition const& loc); - // State of a preprocessor conditional, which can change when // we encounter directives like `#elif` or `#endif` enum class PreprocessorConditionalState @@ -315,11 +305,6 @@ static Token AdvanceRawToken(Preprocessor* preprocessor) EndInputStream(preprocessor, inputStream); continue; } - else - { - // HACK(tfoley): A place to fall into debugger... - int f = 0; - } } // Everything worked, so read a token from the top-most stream @@ -351,11 +336,6 @@ static Token PeekRawToken(Preprocessor* preprocessor) inputStream = inputStream->parent; continue; } - else - { - // HACK(tfoley): A place to fall into debugger... - int f = 0; - } } // Everything worked, so the token we just peeked is fine. @@ -1490,74 +1470,7 @@ static void expectEndOfDirective(PreprocessorDirectiveContext* context) } // Handle a `#import` directive -static void HandleImportDirective(PreprocessorDirectiveContext* context) -{ - Token pathToken; - if(!Expect(context, TokenType::StringLiterial, Diagnostics::expectedTokenInPreprocessorDirective, &pathToken)) - return; - - String path = getFileNameTokenValue(pathToken); - - // TODO(tfoley): make this robust in presence of `#line` - String pathIncludedFrom = GetDirectiveLoc(context).FileName; - String foundPath; - String foundSource; - - IncludeHandler* includeHandler = context->preprocessor->includeHandler; - if (!includeHandler) - { - GetSink(context)->diagnose(pathToken.Position, Diagnostics::importFailed, path); - GetSink(context)->diagnose(pathToken.Position, Diagnostics::noIncludeHandlerSpecified); - return; - } - auto includeResult = includeHandler->TryToFindIncludeFile(path, pathIncludedFrom, &foundPath, &foundSource); - - switch (includeResult) - { - case IncludeResult::NotFound: - case IncludeResult::Error: - GetSink(context)->diagnose(pathToken.Position, Diagnostics::importFailed, path); - return; - - case IncludeResult::Found: - break; - } - - // Do all checking related to the end of this directive before we push a new stream, - // just to avoid complications where that check would need to deal with - // a switch of input stream - expectEndOfDirective(context); - - // Import code from the chosen file - String autoImportName = autoImportModule( - context->preprocessor->compileRequest, - foundPath, - foundSource, - GetDirectiveLoc(context)); - - // Now create a dummy token stream to represent the import request, - // so that it can be manifest in the user's program - SourceTextInputStream* inputStream = new SourceTextInputStream(); - - Token token; - token.Type = TokenType::PoundImport; - token.Position = GetDirectiveLoc(context); - token.flags = 0; - token.Content = autoImportName; - - inputStream->lexedTokens.mTokens.Add(token); - - token.Type = TokenType::EndOfFile; - token.flags = TokenFlag::AfterWhitespace | TokenFlag::AtStartOfLine; - inputStream->lexedTokens.mTokens.Add(token); - - inputStream->tokenReader = TokenReader(inputStream->lexedTokens); - - inputStream->parent = context->preprocessor->inputStream; - context->preprocessor->inputStream = inputStream; -} - - +static void HandleImportDirective(PreprocessorDirectiveContext* context); // Handle a `#include` directive static void HandleIncludeDirective(PreprocessorDirectiveContext* context) @@ -2161,4 +2074,118 @@ TokenList preprocessSource( return tokens; } +// + +// Handle a `#import` directive +static void HandleImportDirective(PreprocessorDirectiveContext* context) +{ + Token pathToken; + if(!Expect(context, TokenType::StringLiterial, Diagnostics::expectedTokenInPreprocessorDirective, &pathToken)) + return; + + String path = getFileNameTokenValue(pathToken); + + // TODO(tfoley): make this robust in presence of `#line` + String pathIncludedFrom = GetDirectiveLoc(context).FileName; + String foundPath; + String foundSource; + + IncludeHandler* includeHandler = context->preprocessor->includeHandler; + if (!includeHandler) + { + GetSink(context)->diagnose(pathToken.Position, Diagnostics::importFailed, path); + GetSink(context)->diagnose(pathToken.Position, Diagnostics::noIncludeHandlerSpecified); + return; + } + auto includeResult = includeHandler->TryToFindIncludeFile(path, pathIncludedFrom, &foundPath, &foundSource); + + switch (includeResult) + { + case IncludeResult::NotFound: + case IncludeResult::Error: + GetSink(context)->diagnose(pathToken.Position, Diagnostics::importFailed, path); + return; + + case IncludeResult::Found: + break; + } + + // Do all checking related to the end of this directive before we push a new stream, + // just to avoid complications where that check would need to deal with + // a switch of input stream + expectEndOfDirective(context); + + // TODO: may want to have some kind of canonicalization step here + String moduleName = foundPath; + + // Import code from the chosen file, if needed. We only + // need to import on the first `#import` directive, and + // after that we ignore additional `#import`s for the same file. + { + auto request = context->preprocessor->compileRequest; + + + // Have we already loaded a module matching this name? + if (request->loadedModulesMap.TryGetValue(moduleName)) + { + // The module has already been loaded, so we bail out + // and leave *nothing* in the input stream. + return; + } + else + { + // We are going to preprocess the file using the *same* preprocessor + // state that is already active. The main alternative would be + // to construct a fresh preprocessor and use that. The current + // choice is made so that macros defined in the imported file + // will be made visible to the importer, rather than disappear + // when a sub-preprocessor gets finalized. + auto preprocessor = context->preprocessor; + + // We need to save/restore the input stream, so that we can + // re-use the preprocessor + PreprocessorInputStream* savedStream = preprocessor->inputStream; + + // Create an input stream for reading from the imported file + PreprocessorInputStream* subInputStream = CreateInputStreamForSource(preprocessor, foundSource, foundPath); + + // Now preprocess that stream + preprocessor->inputStream = subInputStream; + TokenList subTokens = ReadAllTokens(preprocessor); + + // Restore the previous input stream + preprocessor->inputStream = savedStream; + + // Now we need to do something with those tokens we read + request->handlePoundImport( + moduleName, + foundPath, + subTokens); + } + } + + // Now create a dummy token stream to represent the import request, + // so that it can be manifest in the user's program + SourceTextInputStream* inputStream = new SourceTextInputStream(); + + Token token; + token.Type = TokenType::PoundImport; + token.Position = GetDirectiveLoc(context); + token.flags = 0; + token.Content = moduleName; + + inputStream->lexedTokens.mTokens.Add(token); + + token.Type = TokenType::EndOfFile; + token.flags = TokenFlag::AfterWhitespace | TokenFlag::AtStartOfLine; + inputStream->lexedTokens.mTokens.Add(token); + + inputStream->tokenReader = TokenReader(inputStream->lexedTokens); + + inputStream->parent = context->preprocessor->inputStream; + context->preprocessor->inputStream = inputStream; +} + + + } diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp index 76cb502c0..5e80dabf5 100644 --- a/source/slang/slang.cpp +++ b/source/slang/slang.cpp @@ -405,21 +405,39 @@ RefPtr CompileRequest::loadModule( } -String CompileRequest::autoImportModule( +void CompileRequest::handlePoundImport( + String const& name, String const& path, - String const& source, - CodePosition const& loc) + TokenList const& tokens) { - // TODO: may want to have some kind of canonicalization step here - String name = path; + RefPtr translationUnit = new TranslationUnitRequest(); + translationUnit->compileRequest = this; - // Have we already loaded a module matching this name? - if (loadedModulesMap.TryGetValue(name)) - return name; + // Imported code is always native Slang code + RefPtr languageScope = mSession->slangLanguageScope; + + RefPtr translationUnitSyntax = new ProgramSyntaxNode(); + translationUnit->SyntaxNode = translationUnitSyntax; + + parseSourceFile( + translationUnit.Ptr(), + tokens, + &mSink, + path, + languageScope); + + // TODO: handle errors + + checkTranslationUnit(translationUnit.Ptr()); + + // Skip code generation + + // - loadModule(name, path, source, loc); + RefPtr moduleDecl = translationUnit->SyntaxNode; - return name; + loadedModulesMap.Add(name, moduleDecl); + loadedModulesList.Add(moduleDecl); } RefPtr CompileRequest::findOrImportModule( @@ -494,15 +512,6 @@ RefPtr findOrImportModule( return request->findOrImportModule(name, loc); } -String autoImportModule( - CompileRequest* request, - String const& path, - String const& source, - CodePosition const& loc) -{ - return request->autoImportModule(path, source, loc); -} - void Session::addBuiltinSource( RefPtr const& scope, String const& path, diff --git a/tests/preprocessor/import.hlsl b/tests/preprocessor/import.hlsl new file mode 100644 index 000000000..486023678 --- /dev/null +++ b/tests/preprocessor/import.hlsl @@ -0,0 +1,18 @@ +//TEST:SIMPLE:-profile vs_5_0 + +// Confirm that `#import` interacts with preprocessor as expected + +// Here is a macro that flows from parent to child file +#define FOO float + +// Here we import the child file +#import "import.slang.h" + +// Here we use a macro that flows the other way (child->parent) +BAR g( FOO x ) { return f(x); } + +// Here we confirm that importing the file again is a no-op +#import "import.slang.h" + +void main() +{} diff --git a/tests/preprocessor/import.slang.h b/tests/preprocessor/import.slang.h new file mode 100644 index 000000000..a97a199f0 --- /dev/null +++ b/tests/preprocessor/import.slang.h @@ -0,0 +1,12 @@ +// Confirm that `#import` interacts with preprocessor as expected + +// We add a guard to ensure that this file isn't imported more than once +#ifdef BAR +#error File imported more than one! +#endif + +// Here we use a macro from the parent file +FOO f( FOO y ) { return y; } + +// Here is a macro that flows from child to parent +#define BAR float -- cgit v1.2.3