From 7d3bfe403362b294cc2a1f2607d51dfcd447aafd Mon Sep 17 00:00:00 2001 From: Tim Foley Date: Mon, 26 Jun 2017 09:32:40 -0700 Subject: Replace "auto-import" with `#import` Right now `#import` only differs from `#include` in that it takes a string literal for a file name instead of a raw identifier (to which `.slang` gets appended). The next step is to make `#import` respect preprocessor state, while `__import` doesn't. --- slang.h | 10 --- source/slang/compiler.h | 10 +-- source/slang/diagnostic-defs.h | 1 + source/slang/options.cpp | 8 --- source/slang/parser.cpp | 8 +-- source/slang/preprocessor.cpp | 116 +++++++++++++++++++----------- source/slang/preprocessor.h | 3 +- source/slang/slang.cpp | 34 +-------- source/slang/token-defs.h | 2 +- tests/render/auto-import.hlsl | 147 -------------------------------------- tests/render/auto-import.slang.h | 21 ------ tests/render/pound-import.hlsl | 147 ++++++++++++++++++++++++++++++++++++++ tests/render/pound-import.slang.h | 21 ++++++ 13 files changed, 253 insertions(+), 275 deletions(-) delete mode 100644 tests/render/auto-import.hlsl delete mode 100644 tests/render/auto-import.slang.h create mode 100644 tests/render/pound-import.hlsl create mode 100644 tests/render/pound-import.slang.h diff --git a/slang.h b/slang.h index 91347d024..91b9f5931 100644 --- a/slang.h +++ b/slang.h @@ -204,16 +204,6 @@ extern "C" const char* searchDir); /*! - @brief Add a path to use when searching for referenced files, that automatically treats `#include` as `__import` - This behaves just like `spAddSearchPath()` except that any `#include` file found through this path - will be treated as if it was referenced with `__import`. - @param ctx The compilation context. - @param searchDir The additional search directory. - */ - SLANG_API void spAddAutoImportPath( - SlangCompileRequest* request, - const char* searchDir); - /*! @brief Add a macro definition to be used during preprocessing. @param key The name of the macro to define. @param value The value of the macro to define. diff --git a/source/slang/compiler.h b/source/slang/compiler.h index 3f45ea5c7..89bb01738 100644 --- a/source/slang/compiler.h +++ b/source/slang/compiler.h @@ -150,21 +150,13 @@ namespace Slang // A directory to be searched when looking for files (e.g., `#include`) struct SearchDirectory { - enum Kind - { - Default, - AutoImport, - }; - SearchDirectory() = default; SearchDirectory(SearchDirectory const& other) = default; - SearchDirectory(String const& path, Kind kind) + SearchDirectory(String const& path) : path(path) - , kind(kind) {} String path; - Kind kind; }; class Session; diff --git a/source/slang/diagnostic-defs.h b/source/slang/diagnostic-defs.h index ff93e51fb..f92f43ddb 100644 --- a/source/slang/diagnostic-defs.h +++ b/source/slang/diagnostic-defs.h @@ -92,6 +92,7 @@ DIAGNOSTIC(-1, Note, seeOpeningToken, "see opening '$0'") // 153xx - #include DIAGNOSTIC(15300, Error, includeFailed, "failed to find include file '$0'") +DIAGNOSTIC(15301, Error, importFailed, "failed to find imported file '$0'") DIAGNOSTIC(-1, Error, noIncludeHandlerSpecified, "no `#include` handler was specified") // 154xx - macro definition diff --git a/source/slang/options.cpp b/source/slang/options.cpp index 81b893d66..62b45e025 100644 --- a/source/slang/options.cpp +++ b/source/slang/options.cpp @@ -375,14 +375,6 @@ struct OptionsParser compileRequest, String(includeDirStr).begin()); } - else if (argStr == "-auto-import-dir") - { - char const* importDirStr = tryReadCommandLineArgumentRaw(arg, &argCursor, argEnd); - - spAddAutoImportPath( - compileRequest, - String(importDirStr).begin()); - } else if (argStr == "--") { // The `--` option causes us to stop trying to parse options, diff --git a/source/slang/parser.cpp b/source/slang/parser.cpp index 0598e67a7..61d7b225d 100644 --- a/source/slang/parser.cpp +++ b/source/slang/parser.cpp @@ -805,10 +805,10 @@ namespace Slang return decl; } - static RefPtr parseAutoImportDecl( + static RefPtr parsePoundImportDecl( Parser* parser) { - Token importToken = parser->ReadToken(TokenType::AutoImport); + Token importToken = parser->ReadToken(TokenType::PoundImport); auto decl = new ImportDecl(); decl->nameToken = importToken; @@ -2186,8 +2186,8 @@ namespace Slang decl = parseModifierDecl(parser); else if(parser->LookAheadToken("__import")) decl = parseImportDecl(parser); - else if(parser->LookAheadToken(TokenType::AutoImport)) - decl = parseAutoImportDecl(parser); + else if(parser->LookAheadToken(TokenType::PoundImport)) + decl = parsePoundImportDecl(parser); else if (AdvanceIf(parser, TokenType::Semicolon)) { decl = new EmptyDecl(); diff --git a/source/slang/preprocessor.cpp b/source/slang/preprocessor.cpp index 00abb3fe5..b93e47ec2 100644 --- a/source/slang/preprocessor.cpp +++ b/source/slang/preprocessor.cpp @@ -1489,9 +1489,8 @@ static void expectEndOfDirective(PreprocessorDirectiveContext* context) AdvanceRawToken(context->preprocessor); } - -// Handle a `#include` directive -static void HandleIncludeDirective(PreprocessorDirectiveContext* context) +// Handle a `#import` directive +static void HandleImportDirective(PreprocessorDirectiveContext* context) { Token pathToken; if(!Expect(context, TokenType::StringLiterial, Diagnostics::expectedTokenInPreprocessorDirective, &pathToken)) @@ -1507,7 +1506,7 @@ static void HandleIncludeDirective(PreprocessorDirectiveContext* context) IncludeHandler* includeHandler = context->preprocessor->includeHandler; if (!includeHandler) { - GetSink(context)->diagnose(pathToken.Position, Diagnostics::includeFailed, path); + GetSink(context)->diagnose(pathToken.Position, Diagnostics::importFailed, path); GetSink(context)->diagnose(pathToken.Position, Diagnostics::noIncludeHandlerSpecified); return; } @@ -1517,10 +1516,10 @@ static void HandleIncludeDirective(PreprocessorDirectiveContext* context) { case IncludeResult::NotFound: case IncludeResult::Error: - GetSink(context)->diagnose(pathToken.Position, Diagnostics::includeFailed, path); + GetSink(context)->diagnose(pathToken.Position, Diagnostics::importFailed, path); return; - case IncludeResult::FoundIncludeFile: + case IncludeResult::Found: break; } @@ -1529,50 +1528,82 @@ static void HandleIncludeDirective(PreprocessorDirectiveContext* context) // a switch of input stream expectEndOfDirective(context); - switch( includeResult ) - { - case IncludeResult::FoundIncludeFile: - { - // Push the new file onto our stack of input streams - // TODO(tfoley): check if we have made our include stack too deep - PreprocessorInputStream* inputStream = CreateInputStreamForSource(context->preprocessor, foundSource, foundPath); - inputStream->parent = context->preprocessor->inputStream; - context->preprocessor->inputStream = inputStream; - } - break; - - case IncludeResult::FoundAutoImportFile: - { - // - - String autoImportName = autoImportModule( - context->preprocessor->compileRequest, - foundPath, - foundSource, - GetDirectiveLoc(context)); + // Import code from the chosen file + String autoImportName = autoImportModule( + context->preprocessor->compileRequest, + foundPath, + foundSource, + GetDirectiveLoc(context)); - SourceTextInputStream* inputStream = new SourceTextInputStream(); + // Now create a dummy token stream to represent the import request, + // so that it can be manifest in the user's program + SourceTextInputStream* inputStream = new SourceTextInputStream(); - Token token; - token.Type = TokenType::AutoImport; - token.Position = GetDirectiveLoc(context); - token.flags = 0; - token.Content = autoImportName; + Token token; + token.Type = TokenType::PoundImport; + token.Position = GetDirectiveLoc(context); + token.flags = 0; + token.Content = autoImportName; - inputStream->lexedTokens.mTokens.Add(token); + inputStream->lexedTokens.mTokens.Add(token); - token.Type = TokenType::EndOfFile; - token.flags = TokenFlag::AfterWhitespace | TokenFlag::AtStartOfLine; - inputStream->lexedTokens.mTokens.Add(token); + token.Type = TokenType::EndOfFile; + token.flags = TokenFlag::AfterWhitespace | TokenFlag::AtStartOfLine; + inputStream->lexedTokens.mTokens.Add(token); - inputStream->tokenReader = TokenReader(inputStream->lexedTokens); + inputStream->tokenReader = TokenReader(inputStream->lexedTokens); - inputStream->parent = context->preprocessor->inputStream; - context->preprocessor->inputStream = inputStream; - } + inputStream->parent = context->preprocessor->inputStream; + context->preprocessor->inputStream = inputStream; +} + + + +// Handle a `#include` directive +static void HandleIncludeDirective(PreprocessorDirectiveContext* context) +{ + Token pathToken; + if(!Expect(context, TokenType::StringLiterial, Diagnostics::expectedTokenInPreprocessorDirective, &pathToken)) + return; + + String path = getFileNameTokenValue(pathToken); + + // TODO(tfoley): make this robust in presence of `#line` + String pathIncludedFrom = GetDirectiveLoc(context).FileName; + String foundPath; + String foundSource; + + IncludeHandler* includeHandler = context->preprocessor->includeHandler; + if (!includeHandler) + { + GetSink(context)->diagnose(pathToken.Position, Diagnostics::includeFailed, path); + GetSink(context)->diagnose(pathToken.Position, Diagnostics::noIncludeHandlerSpecified); + return; + } + auto includeResult = includeHandler->TryToFindIncludeFile(path, pathIncludedFrom, &foundPath, &foundSource); + + switch (includeResult) + { + case IncludeResult::NotFound: + case IncludeResult::Error: + GetSink(context)->diagnose(pathToken.Position, Diagnostics::includeFailed, path); + return; + + case IncludeResult::Found: break; } - } + + // Do all checking related to the end of this directive before we push a new stream, + // just to avoid complications where that check would need to deal with + // a switch of input stream + expectEndOfDirective(context); + + // Push the new file onto our stack of input streams + // TODO(tfoley): check if we have made our include stack too deep + PreprocessorInputStream* inputStream = CreateInputStreamForSource(context->preprocessor, foundSource, foundPath); + inputStream->parent = context->preprocessor->inputStream; + context->preprocessor->inputStream = inputStream; +} // Handle a `#define` directive static void HandleDefineDirective(PreprocessorDirectiveContext* context) @@ -1878,6 +1909,7 @@ static const PreprocessorDirective kDirectives[] = { "elif", &HandleElifDirective, ProcessWhenSkipping }, { "endif", &HandleEndIfDirective, ProcessWhenSkipping }, + { "import", &HandleImportDirective, 0 }, { "include", &HandleIncludeDirective, 0 }, { "define", &HandleDefineDirective, 0 }, { "undef", &HandleUndefDirective, 0 }, diff --git a/source/slang/preprocessor.h b/source/slang/preprocessor.h index 0c7848bb4..ebeb810a0 100644 --- a/source/slang/preprocessor.h +++ b/source/slang/preprocessor.h @@ -15,8 +15,7 @@ enum class IncludeResult { Error, NotFound, - FoundIncludeFile, - FoundAutoImportFile, + Found, }; // Callback interface for the preprocessor to use when looking diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp index fc18ea1cf..76cb502c0 100644 --- a/source/slang/slang.cpp +++ b/source/slang/slang.cpp @@ -103,21 +103,7 @@ struct IncludeHandlerImpl : IncludeHandler request->mDependencyFilePaths.Add(path); - // HACK(tfoley): We might have found the file in the same directory, - // but what if this is also inside an auto-import path? - for (auto & dir : request->searchDirectories) - { - // Only consider auto-import paths - if(dir.kind != SearchDirectory::Kind::AutoImport) - continue; - - String otherPath = Path::Combine(dir.path, pathToInclude); - - if(otherPath == path) - return IncludeResult::FoundAutoImportFile; - } - - return IncludeResult::FoundIncludeFile; + return IncludeResult::Found; } for (auto & dir : request->searchDirectories) @@ -130,14 +116,7 @@ struct IncludeHandlerImpl : IncludeHandler request->mDependencyFilePaths.Add(path); - switch( dir.kind ) - { - case SearchDirectory::Kind::Default: - return IncludeResult::FoundIncludeFile; - - case SearchDirectory::Kind::AutoImport: - return IncludeResult::FoundAutoImportFile; - } + return IncludeResult::Found; } } return IncludeResult::NotFound; @@ -674,14 +653,7 @@ SLANG_API void spAddSearchPath( SlangCompileRequest* request, const char* path) { - REQ(request)->searchDirectories.Add(Slang::SearchDirectory(path, Slang::SearchDirectory::Kind::Default)); -} - -SLANG_API void spAddAutoImportPath( - SlangCompileRequest* request, - const char* path) -{ - REQ(request)->searchDirectories.Add(Slang::SearchDirectory(path, Slang::SearchDirectory::Kind::AutoImport)); + REQ(request)->searchDirectories.Add(Slang::SearchDirectory(path)); } SLANG_API void spAddPreprocessorDefine( diff --git a/source/slang/token-defs.h b/source/slang/token-defs.h index f7800de55..0fe833e36 100644 --- a/source/slang/token-defs.h +++ b/source/slang/token-defs.h @@ -30,7 +30,7 @@ TOKEN(NewLine, "newline") TOKEN(LineComment, "line comment") TOKEN(BlockComment, "block comment") -TOKEN(AutoImport, "automatic import directive") +TOKEN(PoundImport, "'#import'") #define PUNCTUATION(id, text) \ TOKEN(id, "'" text "'") diff --git a/tests/render/auto-import.hlsl b/tests/render/auto-import.hlsl deleted file mode 100644 index 588ebc612..000000000 --- a/tests/render/auto-import.hlsl +++ /dev/null @@ -1,147 +0,0 @@ -//TEST(smoke,render):COMPARE_HLSL_GLSL_RENDER: -xslang -auto-import-dir -xslang tests/render/ - -// This is a basic test case for cross-compilation behavior. -// -// We will define distinct HLSL and GLSL entry points, -// but the two will share a dependency on a file of -// pure Spire code that provides the actual shading logic. - - -// Pull in Spire code depdendency using extended syntax: -#include "auto-import.slang.h" - -#if defined(__HLSL__) - -cbuffer Uniforms -{ - float4x4 modelViewProjection; -}; - -struct AssembledVertex -{ - float3 position; - float3 color; -}; - -struct CoarseVertex -{ - float3 color; -}; - -struct Fragment -{ - float4 color; -}; - -// Vertex Shader - -struct VertexStageInput -{ - AssembledVertex assembledVertex : A; -}; - -struct VertexStageOutput -{ - CoarseVertex coarseVertex : CoarseVertex; - float4 sv_position : SV_Position; -}; - -VertexStageOutput vertexMain(VertexStageInput input) -{ - VertexStageOutput output; - - float3 position = input.assembledVertex.position; - float3 color = input.assembledVertex.color; - - output.coarseVertex.color = color; - output.sv_position = mul(modelViewProjection, float4(position, 1.0)); - - return output; - -} - -// Fragment Shader - -struct FragmentStageInput -{ - CoarseVertex coarseVertex : CoarseVertex; -}; - -struct FragmentStageOutput -{ - Fragment fragment : SV_Target; -}; - -FragmentStageOutput fragmentMain(FragmentStageInput input) -{ - FragmentStageOutput output; - - float3 color = input.coarseVertex.color; - - color = transformColor(color); - - output.fragment.color = float4(color, 1.0); - - return output; -} - -#elif defined(__GLSL__) - -#version 420 - -uniform Uniforms -{ - mat4x4 modelViewProjection; -}; - -#define ASSEMBLED_VERTEX(QUAL) \ - /* */ - -#define V2F(QUAL) \ - QUAL vec3 coarse_color; \ - /* */ - -// Vertex Shader - -#ifdef __GLSL_VERTEX__ - -layout(location = 0) -in vec3 assembled_position; - -layout(location = 1) -in vec3 assembled_color; - -V2F(out) - -void main() -{ - vec3 position = assembled_position; - vec3 color = assembled_color; - - coarse_color = color; -// gl_Position = modelViewProjection * vec4(position, 1.0); - gl_Position = vec4(position, 1.0) * modelViewProjection; -} - -#endif - -#ifdef __GLSL_FRAGMENT__ - -V2F(in) - -layout(location = 0) -out vec4 fragment_color; - -void main() -{ - vec3 color = coarse_color; - - color = transformColor(color); - - fragment_color = vec4(color, 1.0); -} - - -#endif - -#endif diff --git a/tests/render/auto-import.slang.h b/tests/render/auto-import.slang.h deleted file mode 100644 index d53005688..000000000 --- a/tests/render/auto-import.slang.h +++ /dev/null @@ -1,21 +0,0 @@ -//TEST_IGNORE_FILE: - -// This file implements the "library" code -// that both the HLSL and GLSL shaders share. -// -// This code is written in Slang (more or less -// just HLSL), and will be translated as needed -// for each of the targets. - -float3 transformColor(float3 color) -{ - float3 result; - - result.x = sin(20.0 * (color.x + color.y)); - result.y = saturate(cos(color.z * 30.0)); - result.z = sin(color.x * color.y * color.z * 100.0); - - result = 0.5 * (result + 1); - - return result; -} \ No newline at end of file diff --git a/tests/render/pound-import.hlsl b/tests/render/pound-import.hlsl new file mode 100644 index 000000000..a9b625fb6 --- /dev/null +++ b/tests/render/pound-import.hlsl @@ -0,0 +1,147 @@ +//TEST(smoke,render):COMPARE_HLSL_GLSL_RENDER: + +// This is a basic test case for cross-compilation behavior. +// +// We will define distinct HLSL and GLSL entry points, +// but the two will share a dependency on a file of +// pure Spire code that provides the actual shading logic. + + +// Pull in Spire code depdendency using extended syntax: +#import "pound-import.slang.h" + +#if defined(__HLSL__) + +cbuffer Uniforms +{ + float4x4 modelViewProjection; +}; + +struct AssembledVertex +{ + float3 position; + float3 color; +}; + +struct CoarseVertex +{ + float3 color; +}; + +struct Fragment +{ + float4 color; +}; + +// Vertex Shader + +struct VertexStageInput +{ + AssembledVertex assembledVertex : A; +}; + +struct VertexStageOutput +{ + CoarseVertex coarseVertex : CoarseVertex; + float4 sv_position : SV_Position; +}; + +VertexStageOutput vertexMain(VertexStageInput input) +{ + VertexStageOutput output; + + float3 position = input.assembledVertex.position; + float3 color = input.assembledVertex.color; + + output.coarseVertex.color = color; + output.sv_position = mul(modelViewProjection, float4(position, 1.0)); + + return output; + +} + +// Fragment Shader + +struct FragmentStageInput +{ + CoarseVertex coarseVertex : CoarseVertex; +}; + +struct FragmentStageOutput +{ + Fragment fragment : SV_Target; +}; + +FragmentStageOutput fragmentMain(FragmentStageInput input) +{ + FragmentStageOutput output; + + float3 color = input.coarseVertex.color; + + color = transformColor(color); + + output.fragment.color = float4(color, 1.0); + + return output; +} + +#elif defined(__GLSL__) + +#version 420 + +uniform Uniforms +{ + mat4x4 modelViewProjection; +}; + +#define ASSEMBLED_VERTEX(QUAL) \ + /* */ + +#define V2F(QUAL) \ + QUAL vec3 coarse_color; \ + /* */ + +// Vertex Shader + +#ifdef __GLSL_VERTEX__ + +layout(location = 0) +in vec3 assembled_position; + +layout(location = 1) +in vec3 assembled_color; + +V2F(out) + +void main() +{ + vec3 position = assembled_position; + vec3 color = assembled_color; + + coarse_color = color; +// gl_Position = modelViewProjection * vec4(position, 1.0); + gl_Position = vec4(position, 1.0) * modelViewProjection; +} + +#endif + +#ifdef __GLSL_FRAGMENT__ + +V2F(in) + +layout(location = 0) +out vec4 fragment_color; + +void main() +{ + vec3 color = coarse_color; + + color = transformColor(color); + + fragment_color = vec4(color, 1.0); +} + + +#endif + +#endif diff --git a/tests/render/pound-import.slang.h b/tests/render/pound-import.slang.h new file mode 100644 index 000000000..d53005688 --- /dev/null +++ b/tests/render/pound-import.slang.h @@ -0,0 +1,21 @@ +//TEST_IGNORE_FILE: + +// This file implements the "library" code +// that both the HLSL and GLSL shaders share. +// +// This code is written in Slang (more or less +// just HLSL), and will be translated as needed +// for each of the targets. + +float3 transformColor(float3 color) +{ + float3 result; + + result.x = sin(20.0 * (color.x + color.y)); + result.y = saturate(cos(color.z * 30.0)); + result.z = sin(color.x * color.y * color.z * 100.0); + + result = 0.5 * (result + 1); + + return result; +} \ No newline at end of file -- cgit v1.2.3 From 6e99b81c98f8c76444563d959536073befc7d8ca Mon Sep 17 00:00:00 2001 From: Tim Foley Date: Mon, 26 Jun 2017 10:11:00 -0700 Subject: Make `#import` work with preprocessor macros With this change, there is now a meaningful semantic difference between `__import` and `#import`. An `__import` compiles the target file in a fresh environment, only providing it any macro definitions passed via command line or API. Any macros defined in the imported file are not made visible at the import site. One can think of an `__import` as a bit like `using namespace` in C++. A `#import` will tokenize the input in the same preprocessor environment as the importing file, and any macros defined along the way will be visible in the parent file. It is a *bit* like a `#include` with two big differences: - The imported code is always parsed as Slang, and as its own module with default flags, etc. (so semantic checks are on even if we are in "rewriter" mode). It is pulled into the outer namespace just as for `__import`. - A given file will only get `#import`ed once for a translation unit, so it behaves a bit like there is an implicit `#pragma once` in the target file --- source/slang/compiler.h | 6 +- source/slang/preprocessor.cpp | 217 +++++++++++++++++++++----------------- source/slang/slang.cpp | 47 +++++---- tests/preprocessor/import.hlsl | 18 ++++ tests/preprocessor/import.slang.h | 12 +++ 5 files changed, 183 insertions(+), 117 deletions(-) create mode 100644 tests/preprocessor/import.hlsl create mode 100644 tests/preprocessor/import.slang.h diff --git a/source/slang/compiler.h b/source/slang/compiler.h index 89bb01738..7c4a17607 100644 --- a/source/slang/compiler.h +++ b/source/slang/compiler.h @@ -247,10 +247,10 @@ namespace Slang String const& source, CodePosition const& loc); - String autoImportModule( + void handlePoundImport( + String const& name, String const& path, - String const& source, - CodePosition const& loc); + TokenList const& tokens); RefPtr findOrImportModule( String const& name, diff --git a/source/slang/preprocessor.cpp b/source/slang/preprocessor.cpp index b93e47ec2..c3cc8cb0a 100644 --- a/source/slang/preprocessor.cpp +++ b/source/slang/preprocessor.cpp @@ -1,12 +1,11 @@ -// Preprocessor.cpp -#include "Preprocessor.h" +// preprocessor.cpp +#include "preprocessor.h" -#include "Diagnostics.h" -#include "Lexer.h" - -// Needed so that we can construct modifier syntax -// to represent GLSL directives -#include "Syntax.h" +#include "compiler.h" +#include "diagnostics.h" +#include "lexer.h" +// Needed so that we can construct modifier syntax to represent GLSL directives +#include "syntax.h" #include @@ -18,15 +17,6 @@ namespace Slang { -// Forward declaration for code provided in `slang.cpp` -// -// TODO: Need an appropriate header for this. -String autoImportModule( - CompileRequest* request, - String const& path, - String const& source, - CodePosition const& loc); - // State of a preprocessor conditional, which can change when // we encounter directives like `#elif` or `#endif` enum class PreprocessorConditionalState @@ -315,11 +305,6 @@ static Token AdvanceRawToken(Preprocessor* preprocessor) EndInputStream(preprocessor, inputStream); continue; } - else - { - // HACK(tfoley): A place to fall into debugger... - int f = 0; - } } // Everything worked, so read a token from the top-most stream @@ -351,11 +336,6 @@ static Token PeekRawToken(Preprocessor* preprocessor) inputStream = inputStream->parent; continue; } - else - { - // HACK(tfoley): A place to fall into debugger... - int f = 0; - } } // Everything worked, so the token we just peeked is fine. @@ -1490,74 +1470,7 @@ static void expectEndOfDirective(PreprocessorDirectiveContext* context) } // Handle a `#import` directive -static void HandleImportDirective(PreprocessorDirectiveContext* context) -{ - Token pathToken; - if(!Expect(context, TokenType::StringLiterial, Diagnostics::expectedTokenInPreprocessorDirective, &pathToken)) - return; - - String path = getFileNameTokenValue(pathToken); - - // TODO(tfoley): make this robust in presence of `#line` - String pathIncludedFrom = GetDirectiveLoc(context).FileName; - String foundPath; - String foundSource; - - IncludeHandler* includeHandler = context->preprocessor->includeHandler; - if (!includeHandler) - { - GetSink(context)->diagnose(pathToken.Position, Diagnostics::importFailed, path); - GetSink(context)->diagnose(pathToken.Position, Diagnostics::noIncludeHandlerSpecified); - return; - } - auto includeResult = includeHandler->TryToFindIncludeFile(path, pathIncludedFrom, &foundPath, &foundSource); - - switch (includeResult) - { - case IncludeResult::NotFound: - case IncludeResult::Error: - GetSink(context)->diagnose(pathToken.Position, Diagnostics::importFailed, path); - return; - - case IncludeResult::Found: - break; - } - - // Do all checking related to the end of this directive before we push a new stream, - // just to avoid complications where that check would need to deal with - // a switch of input stream - expectEndOfDirective(context); - - // Import code from the chosen file - String autoImportName = autoImportModule( - context->preprocessor->compileRequest, - foundPath, - foundSource, - GetDirectiveLoc(context)); - - // Now create a dummy token stream to represent the import request, - // so that it can be manifest in the user's program - SourceTextInputStream* inputStream = new SourceTextInputStream(); - - Token token; - token.Type = TokenType::PoundImport; - token.Position = GetDirectiveLoc(context); - token.flags = 0; - token.Content = autoImportName; - - inputStream->lexedTokens.mTokens.Add(token); - - token.Type = TokenType::EndOfFile; - token.flags = TokenFlag::AfterWhitespace | TokenFlag::AtStartOfLine; - inputStream->lexedTokens.mTokens.Add(token); - - inputStream->tokenReader = TokenReader(inputStream->lexedTokens); - - inputStream->parent = context->preprocessor->inputStream; - context->preprocessor->inputStream = inputStream; -} - - +static void HandleImportDirective(PreprocessorDirectiveContext* context); // Handle a `#include` directive static void HandleIncludeDirective(PreprocessorDirectiveContext* context) @@ -2161,4 +2074,118 @@ TokenList preprocessSource( return tokens; } +// + +// Handle a `#import` directive +static void HandleImportDirective(PreprocessorDirectiveContext* context) +{ + Token pathToken; + if(!Expect(context, TokenType::StringLiterial, Diagnostics::expectedTokenInPreprocessorDirective, &pathToken)) + return; + + String path = getFileNameTokenValue(pathToken); + + // TODO(tfoley): make this robust in presence of `#line` + String pathIncludedFrom = GetDirectiveLoc(context).FileName; + String foundPath; + String foundSource; + + IncludeHandler* includeHandler = context->preprocessor->includeHandler; + if (!includeHandler) + { + GetSink(context)->diagnose(pathToken.Position, Diagnostics::importFailed, path); + GetSink(context)->diagnose(pathToken.Position, Diagnostics::noIncludeHandlerSpecified); + return; + } + auto includeResult = includeHandler->TryToFindIncludeFile(path, pathIncludedFrom, &foundPath, &foundSource); + + switch (includeResult) + { + case IncludeResult::NotFound: + case IncludeResult::Error: + GetSink(context)->diagnose(pathToken.Position, Diagnostics::importFailed, path); + return; + + case IncludeResult::Found: + break; + } + + // Do all checking related to the end of this directive before we push a new stream, + // just to avoid complications where that check would need to deal with + // a switch of input stream + expectEndOfDirective(context); + + // TODO: may want to have some kind of canonicalization step here + String moduleName = foundPath; + + // Import code from the chosen file, if needed. We only + // need to import on the first `#import` directive, and + // after that we ignore additional `#import`s for the same file. + { + auto request = context->preprocessor->compileRequest; + + + // Have we already loaded a module matching this name? + if (request->loadedModulesMap.TryGetValue(moduleName)) + { + // The module has already been loaded, so we bail out + // and leave *nothing* in the input stream. + return; + } + else + { + // We are going to preprocess the file using the *same* preprocessor + // state that is already active. The main alternative would be + // to construct a fresh preprocessor and use that. The current + // choice is made so that macros defined in the imported file + // will be made visible to the importer, rather than disappear + // when a sub-preprocessor gets finalized. + auto preprocessor = context->preprocessor; + + // We need to save/restore the input stream, so that we can + // re-use the preprocessor + PreprocessorInputStream* savedStream = preprocessor->inputStream; + + // Create an input stream for reading from the imported file + PreprocessorInputStream* subInputStream = CreateInputStreamForSource(preprocessor, foundSource, foundPath); + + // Now preprocess that stream + preprocessor->inputStream = subInputStream; + TokenList subTokens = ReadAllTokens(preprocessor); + + // Restore the previous input stream + preprocessor->inputStream = savedStream; + + // Now we need to do something with those tokens we read + request->handlePoundImport( + moduleName, + foundPath, + subTokens); + } + } + + // Now create a dummy token stream to represent the import request, + // so that it can be manifest in the user's program + SourceTextInputStream* inputStream = new SourceTextInputStream(); + + Token token; + token.Type = TokenType::PoundImport; + token.Position = GetDirectiveLoc(context); + token.flags = 0; + token.Content = moduleName; + + inputStream->lexedTokens.mTokens.Add(token); + + token.Type = TokenType::EndOfFile; + token.flags = TokenFlag::AfterWhitespace | TokenFlag::AtStartOfLine; + inputStream->lexedTokens.mTokens.Add(token); + + inputStream->tokenReader = TokenReader(inputStream->lexedTokens); + + inputStream->parent = context->preprocessor->inputStream; + context->preprocessor->inputStream = inputStream; +} + + + } diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp index 76cb502c0..5e80dabf5 100644 --- a/source/slang/slang.cpp +++ b/source/slang/slang.cpp @@ -405,21 +405,39 @@ RefPtr CompileRequest::loadModule( } -String CompileRequest::autoImportModule( +void CompileRequest::handlePoundImport( + String const& name, String const& path, - String const& source, - CodePosition const& loc) + TokenList const& tokens) { - // TODO: may want to have some kind of canonicalization step here - String name = path; + RefPtr translationUnit = new TranslationUnitRequest(); + translationUnit->compileRequest = this; - // Have we already loaded a module matching this name? - if (loadedModulesMap.TryGetValue(name)) - return name; + // Imported code is always native Slang code + RefPtr languageScope = mSession->slangLanguageScope; + + RefPtr translationUnitSyntax = new ProgramSyntaxNode(); + translationUnit->SyntaxNode = translationUnitSyntax; + + parseSourceFile( + translationUnit.Ptr(), + tokens, + &mSink, + path, + languageScope); + + // TODO: handle errors + + checkTranslationUnit(translationUnit.Ptr()); + + // Skip code generation + + // - loadModule(name, path, source, loc); + RefPtr moduleDecl = translationUnit->SyntaxNode; - return name; + loadedModulesMap.Add(name, moduleDecl); + loadedModulesList.Add(moduleDecl); } RefPtr CompileRequest::findOrImportModule( @@ -494,15 +512,6 @@ RefPtr findOrImportModule( return request->findOrImportModule(name, loc); } -String autoImportModule( - CompileRequest* request, - String const& path, - String const& source, - CodePosition const& loc) -{ - return request->autoImportModule(path, source, loc); -} - void Session::addBuiltinSource( RefPtr const& scope, String const& path, diff --git a/tests/preprocessor/import.hlsl b/tests/preprocessor/import.hlsl new file mode 100644 index 000000000..486023678 --- /dev/null +++ b/tests/preprocessor/import.hlsl @@ -0,0 +1,18 @@ +//TEST:SIMPLE:-profile vs_5_0 + +// Confirm that `#import` interacts with preprocessor as expected + +// Here is a macro that flows from parent to child file +#define FOO float + +// Here we import the child file +#import "import.slang.h" + +// Here we use a macro that flows the other way (child->parent) +BAR g( FOO x ) { return f(x); } + +// Here we confirm that importing the file again is a no-op +#import "import.slang.h" + +void main() +{} diff --git a/tests/preprocessor/import.slang.h b/tests/preprocessor/import.slang.h new file mode 100644 index 000000000..a97a199f0 --- /dev/null +++ b/tests/preprocessor/import.slang.h @@ -0,0 +1,12 @@ +// Confirm that `#import` interacts with preprocessor as expected + +// We add a guard to ensure that this file isn't imported more than once +#ifdef BAR +#error File imported more than one! +#endif + +// Here we use a macro from the parent file +FOO f( FOO y ) { return y; } + +// Here is a macro that flows from child to parent +#define BAR float -- cgit v1.2.3 From f6cb66feab3439f41ca87cb307f69b49654883ab Mon Sep 17 00:00:00 2001 From: Tim Foley Date: Mon, 26 Jun 2017 10:52:31 -0700 Subject: Check for re-import at translation-unit level Previously the code checked for a duplicate `#import` using a data structure attached to the compile request, but this would fail for nested imports. It also wouldn't work for a combination of `#import` and `__import`. This change makes it so that we instead track a set of already-imported modules in the semantic checking visitor, which is instantiated once per translation unit. We also key this set on the actual module (AST) imported, rather than on path/name/whatever, so hopefully it will be robust to the same thing getting imported multiple ways. --- source/slang/check.cpp | 17 ++++++++++++++ source/slang/compiler.h | 10 +++++++-- source/slang/preprocessor.cpp | 52 ++++++++++++++++++++++++++----------------- source/slang/preprocessor.h | 7 +++--- source/slang/slang.cpp | 24 ++++++++++++++------ 5 files changed, 77 insertions(+), 33 deletions(-) diff --git a/source/slang/check.cpp b/source/slang/check.cpp index 463052bc8..582f19448 100644 --- a/source/slang/check.cpp +++ b/source/slang/check.cpp @@ -52,6 +52,14 @@ namespace Slang // lexical outer statements List outerStmts; + + // We need to track what has been `import`ed, + // to avoid importing the same thing more than once + // + // TODO: a smarter approach might be to filter + // out duplicate references during lookup. + HashSet importedModules; + public: SemanticsVisitor( DiagnosticSink * pErr, @@ -4909,6 +4917,15 @@ namespace Slang // it later during code generation. decl->importedModuleDecl = importedModuleDecl; + // If we've imported this one already, then + // skip the step where we modify the current scope. + if (importedModules.Contains(importedModuleDecl.Ptr())) + { + return; + } + importedModules.Add(importedModuleDecl.Ptr()); + + // Create a new sub-scope to wire the module // into our lookup chain. auto subScope = new Scope(); diff --git a/source/slang/compiler.h b/source/slang/compiler.h index 7c4a17607..a6f3eee0e 100644 --- a/source/slang/compiler.h +++ b/source/slang/compiler.h @@ -203,9 +203,16 @@ namespace Slang RefPtr layout; // Modules that have been dynamically loaded via `import` - Dictionary> loadedModulesMap; + // + // This is a list of unique modules loaded, in the order they were encountered. List > loadedModulesList; + // Map from the logical name of a module to its definition + Dictionary> mapPathToLoadedModule; + + // Map from the path of a module file to its definition + Dictionary> mapNameToLoadedModules; + CompileRequest(Session* session) : mSession(session) @@ -248,7 +255,6 @@ namespace Slang CodePosition const& loc); void handlePoundImport( - String const& name, String const& path, TokenList const& tokens); diff --git a/source/slang/preprocessor.cpp b/source/slang/preprocessor.cpp index c3cc8cb0a..084f518fb 100644 --- a/source/slang/preprocessor.cpp +++ b/source/slang/preprocessor.cpp @@ -167,11 +167,23 @@ struct Preprocessor // represent end-of-input situations. Token endOfFileToken; - // Syntax for the program we are trying to parse - ProgramSyntaxNode* syntax; + // The translation unit that is being parsed + TranslationUnitRequest* translationUnit; - // The over-arching compile request taht is invoking us - CompileRequest* compileRequest; + TranslationUnitRequest* getTranslationUnit() + { + return translationUnit; + } + + ProgramSyntaxNode* getSyntax() + { + return getTranslationUnit()->SyntaxNode.Ptr(); + } + + CompileRequest* getCompileRequest() + { + return getTranslationUnit()->compileRequest; + } }; // Convenience routine to access the diagnostic sink @@ -1733,7 +1745,7 @@ static void handleGLSLVersionDirective(PreprocessorDirectiveContext* context) // Attach the modifier to the program we are parsing! addModifier( - context->preprocessor->syntax, + context->preprocessor->getSyntax(), modifier); } @@ -1777,7 +1789,7 @@ static void handleGLSLExtensionDirective(PreprocessorDirectiveContext* context) // Attach the modifier to the program we are parsing! addModifier( - context->preprocessor->syntax, + context->preprocessor->getSyntax(), modifier); } @@ -2029,14 +2041,12 @@ TokenList preprocessSource( String const& fileName, DiagnosticSink* sink, IncludeHandler* includeHandler, - Dictionary defines, - ProgramSyntaxNode* syntax, - CompileRequest* compileRequest) + Dictionary defines, + TranslationUnitRequest* translationUnit) { Preprocessor preprocessor; InitializePreprocessor(&preprocessor, sink); - preprocessor.syntax = syntax; - preprocessor.compileRequest = compileRequest; + preprocessor.translationUnit = translationUnit; preprocessor.includeHandler = includeHandler; for (auto p : defines) @@ -2116,21 +2126,24 @@ static void HandleImportDirective(PreprocessorDirectiveContext* context) expectEndOfDirective(context); // TODO: may want to have some kind of canonicalization step here - String moduleName = foundPath; + String moduleKey = foundPath; // Import code from the chosen file, if needed. We only // need to import on the first `#import` directive, and // after that we ignore additional `#import`s for the same file. { - auto request = context->preprocessor->compileRequest; + auto translationUnit = context->preprocessor->translationUnit; + auto request = translationUnit->compileRequest; // Have we already loaded a module matching this name? - if (request->loadedModulesMap.TryGetValue(moduleName)) + if (request->mapPathToLoadedModule.TryGetValue(moduleKey)) { - // The module has already been loaded, so we bail out - // and leave *nothing* in the input stream. - return; + // The module has already been loaded, so we don't need to + // actually tokenize the code here. But note that we *do* + // go on to insert tokens for an `import` operation into + // the stream, so it is up to downstream code to avoid + // re-importing the same thing twice. } else { @@ -2158,8 +2171,7 @@ static void HandleImportDirective(PreprocessorDirectiveContext* context) // Now we need to do something with those tokens we read request->handlePoundImport( - moduleName, - foundPath, + moduleKey, subTokens); } } @@ -2172,7 +2184,7 @@ static void HandleImportDirective(PreprocessorDirectiveContext* context) token.Type = TokenType::PoundImport; token.Position = GetDirectiveLoc(context); token.flags = 0; - token.Content = moduleName; + token.Content = foundPath; inputStream->lexedTokens.mTokens.Add(token); diff --git a/source/slang/preprocessor.h b/source/slang/preprocessor.h index ebeb810a0..d23b19773 100644 --- a/source/slang/preprocessor.h +++ b/source/slang/preprocessor.h @@ -7,9 +7,9 @@ namespace Slang { -class CompileRequest; class DiagnosticSink; class ProgramSyntaxNode; +class TranslationUnitRequest; enum class IncludeResult { @@ -35,9 +35,8 @@ TokenList preprocessSource( String const& fileName, DiagnosticSink* sink, IncludeHandler* includeHandler, - Dictionary defines, - ProgramSyntaxNode* syntax, - CompileRequest* compileRequest); + Dictionary defines, + TranslationUnitRequest* translationUnit); } // namespace Slang diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp index 5e80dabf5..cd48a4152 100644 --- a/source/slang/slang.cpp +++ b/source/slang/slang.cpp @@ -167,8 +167,7 @@ void CompileRequest::parseTranslationUnit( &mSink, &includeHandler, combinedPreprocessorDefinitions, - translationUnitSyntax.Ptr(), - this); + translationUnit); parseSourceFile( translationUnit, @@ -398,7 +397,8 @@ RefPtr CompileRequest::loadModule( RefPtr moduleDecl = translationUnit->SyntaxNode; - loadedModulesMap.Add(name, moduleDecl); + mapPathToLoadedModule.Add(path, moduleDecl); + mapNameToLoadedModules.Add(name, moduleDecl); loadedModulesList.Add(moduleDecl); return moduleDecl; @@ -406,7 +406,6 @@ RefPtr CompileRequest::loadModule( } void CompileRequest::handlePoundImport( - String const& name, String const& path, TokenList const& tokens) { @@ -436,7 +435,13 @@ void CompileRequest::handlePoundImport( RefPtr moduleDecl = translationUnit->SyntaxNode; - loadedModulesMap.Add(name, moduleDecl); + // TODO: It is a bit broken here that we use the module path, + // as the "name" when registering things, but this saves + // us the trouble of trying to special-case things when + // checking an `import` down the road. + mapNameToLoadedModules.Add(path, moduleDecl); + + mapPathToLoadedModule.Add(path, moduleDecl); loadedModulesList.Add(moduleDecl); } @@ -447,7 +452,7 @@ RefPtr CompileRequest::findOrImportModule( // Have we already loaded a module matching this name? // If so, return it. RefPtr moduleDecl; - if (loadedModulesMap.TryGetValue(name, moduleDecl)) + if (mapNameToLoadedModules.TryGetValue(name, moduleDecl)) return moduleDecl; // Derive a file name for the module, by taking the given @@ -486,7 +491,7 @@ RefPtr CompileRequest::findOrImportModule( { this->mSink.diagnose(loc, Diagnostics::cannotFindFile, fileName); - loadedModulesMap[name] = nullptr; + mapNameToLoadedModules[name] = nullptr; return nullptr; } break; @@ -495,6 +500,11 @@ RefPtr CompileRequest::findOrImportModule( break; } + // Maybe this was loaded previously via `#import` + if (mapPathToLoadedModule.TryGetValue(foundPath, moduleDecl)) + return moduleDecl; + + // We've found a file that we can load for the given module, so // go ahead and perform the module-load action return loadModule( -- cgit v1.2.3