From 6cbc3929a54d37bd23cb5efa8e3320ba02f78b2f Mon Sep 17 00:00:00 2001 From: jsmall-nvidia Date: Fri, 31 May 2019 17:20:37 -0400 Subject: Use slang- prefix on slang compiler and core source (#973) * Prefixing source files in source/slang with slang- * Prefix source in source/slang with slang- prefix. * Rename core source files with slang- prefix. * Update project files. * Fix problems from automatic merge. --- source/slang/slang-preprocessor.cpp | 2302 +++++++++++++++++++++++++++++++++++ 1 file changed, 2302 insertions(+) create mode 100644 source/slang/slang-preprocessor.cpp (limited to 'source/slang/slang-preprocessor.cpp') diff --git a/source/slang/slang-preprocessor.cpp b/source/slang/slang-preprocessor.cpp new file mode 100644 index 000000000..16a64e571 --- /dev/null +++ b/source/slang/slang-preprocessor.cpp @@ -0,0 +1,2302 @@ +// slang-preprocessor.cpp +#include "slang-preprocessor.h" + +#include "slang-compiler.h" +#include "slang-diagnostics.h" +#include "slang-lexer.h" +// Needed so that we can construct modifier syntax to represent GLSL directives +#include "slang-syntax.h" + +#include + +// This file provides an implementation of a simple C-style preprocessor. +// It does not aim for 100% compatibility with any particular preprocessor +// specification, but the goal is to have it accept the most common +// idioms for using the preprocessor, found in shader code in the wild. + + +namespace Slang { + +// State of a preprocessor conditional, which can change when +// we encounter directives like `#elif` or `#endif` +enum class PreprocessorConditionalState +{ + Before, // We have not yet seen a branch with a `true` condition. + During, // We are inside the branch with a `true` condition. + After, // We have already seen the branch with a `true` condition. +}; + +// Represents a preprocessor conditional that we are currently +// nested inside. +struct PreprocessorConditional +{ + // The next outer conditional in the current file/stream, or NULL. + PreprocessorConditional* parent; + + // The directive token that started the conditional (an `#if` or `#ifdef`) + Token ifToken; + + // The `#else` directive token, if one has been seen (otherwise `TokenType::Unknown`) + Token elseToken; + + // The state of the conditional + PreprocessorConditionalState state; +}; + +struct PreprocessorMacro; + +struct PreprocessorEnvironment +{ + // The "outer" environment, to be used if lookup in this env fails + PreprocessorEnvironment* parent = NULL; + + // Macros defined in this environment + Dictionary macros; + + ~PreprocessorEnvironment(); +}; + +// Input tokens can either come from source text, or from macro expansion. +// In general, input streams can be nested, so we have to keep a conceptual +// stack of input. + +struct PrimaryInputStream; + +// A stream of input tokens to be consumed +struct PreprocessorInputStream +{ + // The primary input stream that is the parent to this one, + // or NULL if this stream is itself a primary stream. + PrimaryInputStream* primaryStream; + + // The next input stream up the stack, if any. + PreprocessorInputStream* parent; + + // Environment to use when looking up macros + PreprocessorEnvironment* environment; + + // Destructor is virtual so that we can clean up + // after concrete subtypes. + virtual ~PreprocessorInputStream() = default; +}; + +// A "primary" input stream represents the top-level context of a file +// being parsed, and tracks things like preprocessor conditional state +struct PrimaryInputStream : PreprocessorInputStream +{ + // The next *primary* input stream up the stack + PrimaryInputStream* parentPrimaryInputStream; + + // The deepest preprocessor conditional active for this stream. + PreprocessorConditional* conditional; + + // The lexer state that will provide input + Lexer lexer; + + // One token of lookahead + Token token; +}; + +// A "secondary" input stream represents code that is being expanded +// into the current scope, but which had already been tokenized before. +// +struct PretokenizedInputStream : PreprocessorInputStream +{ + // Reader for pre-tokenized input + TokenReader tokenReader; +}; + +// A pre-tokenized input stream that will only be used once, and which +// therefore owns the memory for its tokens. +struct SimpleTokenInputStream : PretokenizedInputStream +{ + // A list of raw tokens that will provide input + TokenList lexedTokens; +}; + +struct MacroExpansion : PretokenizedInputStream +{ + // The macro we will expand + PreprocessorMacro* macro; +}; + +struct ObjectLikeMacroExpansion : MacroExpansion +{ +}; + +struct FunctionLikeMacroExpansion : MacroExpansion +{ + // Environment for macro arguments + PreprocessorEnvironment argumentEnvironment; +}; + +// An enumeration for the diferent types of macros +enum class PreprocessorMacroFlavor +{ + ObjectLike, + FunctionArg, + FunctionLike, +}; + +// In the current design (which we may want to re-consider), +// a macro is a specialized flavor of input stream, that +// captures the token list in its expansion, and then +// can be "played back." +struct PreprocessorMacro +{ + // The name under which the macro was `#define`d + NameLoc nameAndLoc; + + // Parameters of the macro, in case of a function-like macro + List params; + + // The tokens that make up the macro body + TokenList tokens; + + // The flavor of macro + PreprocessorMacroFlavor flavor; + + // The environment in which this macro needs to be expanded. + // For ordinary macros this will be the global environment, + // while for function-like macro arguments, it will be + // the environment of the macro invocation. + PreprocessorEnvironment* environment; + + // + Name* getName() + { + return nameAndLoc.name; + } + + SourceLoc getLoc() + { + return nameAndLoc.loc; + } +}; + +// State of the preprocessor +struct Preprocessor +{ + // diagnostics sink to use when writing messages + DiagnosticSink* sink; + + // An external callback interface to use when looking + // for files in a `#include` directive + IncludeHandler* includeHandler; + + // Current input stream (top of the stack of input) + PreprocessorInputStream* inputStream; + + // Currently-defined macros + PreprocessorEnvironment globalEnv; + + // A pre-allocated token that can be returned to + // represent end-of-input situations. + Token endOfFileToken; + + /// The linkage the provides the context for preprocessing + Linkage* linkage = nullptr; + + /// The module, if any, that the preprocessed result will belong to + Module* parentModule = nullptr; + + // The unique identities of any paths that have issued `#pragma once` directives to + // stop them from being included again. + HashSet pragmaOnceUniqueIdentities; + + NamePool* getNamePool() { return linkage->getNamePool(); } + SourceManager* getSourceManager() { return linkage->getSourceManager(); } +}; + +// Convenience routine to access the diagnostic sink +static DiagnosticSink* GetSink(Preprocessor* preprocessor) +{ + return preprocessor->sink; +} + +// +// Forward declarations +// + +static void DestroyConditional(PreprocessorConditional* conditional); +static void DestroyMacro(Preprocessor* preprocessor, PreprocessorMacro* macro); +static bool IsSkipping(Preprocessor* preprocessor); + +// +// Basic Input Handling +// + +// Create a fresh input stream +static void initializeInputStream(Preprocessor* preprocessor, PreprocessorInputStream* inputStream) +{ + inputStream->parent = NULL; + inputStream->environment = &preprocessor->globalEnv; +} + +static void initializePrimaryInputStream(Preprocessor* preprocessor, PrimaryInputStream* inputStream) +{ + initializeInputStream(preprocessor, inputStream); + inputStream->primaryStream = inputStream; + inputStream->conditional = NULL; +} + +// Destroy an input stream +static void destroyInputStream(Preprocessor* /*preprocessor*/, PreprocessorInputStream* inputStream) +{ + delete inputStream; +} + +// Create an input stream to represent a pre-tokenized input file. +// TODO(tfoley): pre-tokenizing files isn't going to work in the long run. +static PreprocessorInputStream* CreateInputStreamForSource( + Preprocessor* preprocessor, + SourceView* sourceView) +{ + MemoryArena* memoryArena = sourceView->getSourceManager()->getMemoryArena(); + + PrimaryInputStream* inputStream = new PrimaryInputStream(); + initializePrimaryInputStream(preprocessor, inputStream); + + // initialize the embedded lexer so that it can generate a token stream + inputStream->lexer.initialize(sourceView, GetSink(preprocessor), preprocessor->getNamePool(), memoryArena); + inputStream->token = inputStream->lexer.lexToken(); + + return inputStream; +} + +static PrimaryInputStream* asPrimaryInputStream(PreprocessorInputStream* inputStream) +{ + auto primaryStream = inputStream->primaryStream; + if(primaryStream == inputStream) + return primaryStream; + return nullptr; +} + + +static void PushInputStream(Preprocessor* preprocessor, PreprocessorInputStream* inputStream) +{ + inputStream->parent = preprocessor->inputStream; + if(!asPrimaryInputStream(inputStream)) + inputStream->primaryStream = preprocessor->inputStream->primaryStream; + preprocessor->inputStream = inputStream; +} + +// Called when we reach the end of an input stream. +// Performs some validation and then destroys the input stream if required. +static void EndInputStream(Preprocessor* preprocessor, PreprocessorInputStream* inputStream) +{ + if(auto primaryStream = asPrimaryInputStream(inputStream)) + { + // If there are any conditionals that weren't completed, then it is an error + if (primaryStream->conditional) + { + PreprocessorConditional* conditional = primaryStream->conditional; + + GetSink(preprocessor)->diagnose(conditional->ifToken.loc, Diagnostics::endOfFileInPreprocessorConditional); + + while (conditional) + { + PreprocessorConditional* parent = conditional->parent; + DestroyConditional(conditional); + conditional = parent; + } + } + } + + destroyInputStream(preprocessor, inputStream); +} + +// Consume one token from an input stream +static Token AdvanceRawToken(PreprocessorInputStream* inputStream, LexerFlags lexerFlags = 0) +{ + if( auto primaryStream = asPrimaryInputStream(inputStream) ) + { + auto result = primaryStream->token; + primaryStream->token = primaryStream->lexer.lexToken(lexerFlags); + return result; + } + else + { + PretokenizedInputStream* pretokenized = (PretokenizedInputStream*) inputStream; + return pretokenized->tokenReader.AdvanceToken(); + } +} + +// Peek one token from an input stream +static Token PeekRawToken(PreprocessorInputStream* inputStream) +{ + if( auto primaryStream = asPrimaryInputStream(inputStream) ) + { + return primaryStream->token; + } + else + { + PretokenizedInputStream* pretokenized = (PretokenizedInputStream*) inputStream; + return pretokenized->tokenReader.PeekToken(); + } +} + +// Peek one token type from an input stream +static TokenType PeekRawTokenType(PreprocessorInputStream* inputStream) +{ + if( auto primaryStream = asPrimaryInputStream(inputStream) ) + { + return primaryStream->token.type; + } + else + { + PretokenizedInputStream* pretokenized = (PretokenizedInputStream*) inputStream; + return pretokenized->tokenReader.PeekTokenType(); + } +} + + +// Read one token in "raw" mode (meaning don't expand macros) +static Token AdvanceRawToken(Preprocessor* preprocessor, LexerFlags lexerFlags = 0) +{ + for(;;) + { + // Look at the input stream on top of the stack + PreprocessorInputStream* inputStream = preprocessor->inputStream; + + // If there isn't one, then there is no more input left to read. + if(!inputStream) + { + return preprocessor->endOfFileToken; + } + + // The top-most input stream may be at its end + if(PeekRawTokenType(inputStream) == TokenType::EndOfFile) + { + // If there is another stream remaining, switch to it + if(inputStream->parent) + { + preprocessor->inputStream = inputStream->parent; + EndInputStream(preprocessor, inputStream); + continue; + } + } + + // Everything worked, so read a token from the top-most stream + return AdvanceRawToken( + inputStream, + lexerFlags | (IsSkipping(preprocessor) ? kLexerFlag_IgnoreInvalid : 0)); + } +} + +// Return the next token in "raw" mode, but don't advance the +// current token state. +static Token PeekRawToken(Preprocessor* preprocessor) +{ + // We need to find the stream that `advanceRawToken` would read from. + PreprocessorInputStream* inputStream = preprocessor->inputStream; + for (;;) + { + if (!inputStream) + { + // No more input streams left to read + return preprocessor->endOfFileToken; + } + + // The top-most input stream may be at its end, so + // look one entry up the stack (don't actually pop + // here, since we are just peeking) + if (PeekRawTokenType(inputStream) == TokenType::EndOfFile) + { + if (inputStream->parent) + { + inputStream = inputStream->parent; + continue; + } + } + + // Everything worked, so the token we just peeked is fine. + return PeekRawToken(inputStream); + } +} + +// Get the location of the current (raw) token +static SourceLoc PeekLoc(Preprocessor* preprocessor) +{ + return PeekRawToken(preprocessor).loc; +} + +// Get the `TokenType` of the current (raw) token +static TokenType PeekRawTokenType(Preprocessor* preprocessor) +{ + return PeekRawToken(preprocessor).type; +} + +// +// Macros +// + +// Create a macro +static PreprocessorMacro* CreateMacro(Preprocessor* preprocessor) +{ + // TODO(tfoley): Allocate these more intelligently. + // For example, consider pooling them on the preprocessor. + + PreprocessorMacro* macro = new PreprocessorMacro(); + macro->flavor = PreprocessorMacroFlavor::ObjectLike; + macro->environment = &preprocessor->globalEnv; + return macro; +} + +// Destroy a macro +static void DestroyMacro(Preprocessor* /*preprocessor*/, PreprocessorMacro* macro) +{ + delete macro; +} + + +// Find the currently-defined macro of the given name, or return NULL +static PreprocessorMacro* LookupMacro(PreprocessorEnvironment* environment, Name* name) +{ + for(PreprocessorEnvironment* e = environment; e; e = e->parent) + { + PreprocessorMacro* macro = NULL; + if (e->macros.TryGetValue(name, macro)) + return macro; + } + + return NULL; +} + +static PreprocessorEnvironment* GetCurrentEnvironment(Preprocessor* preprocessor) +{ + // The environment we will use for looking up a macro is associated + // with the current input stream (because it may include entries + // for macro arguments). + // + // We need to be careful, though, when we are at the end of an + // input stream (e.g., representing one argument), so that we + // don't use its environment. + + PreprocessorInputStream* inputStream = preprocessor->inputStream; + + for(;;) + { + // If there is no input stream that isn't at its end, + // then fall back to the global environment. + if (!inputStream) + return &preprocessor->globalEnv; + + // If the current input stream is at its end, then + // fall back to its parent stream. + if (PeekRawTokenType(inputStream) == TokenType::EndOfFile) + { + inputStream = inputStream->parent; + continue; + } + + // If we've found an active stream that isn't at its end, + // then use that for lookup. + return inputStream->environment; + } +} + +static PreprocessorMacro* LookupMacro(Preprocessor* preprocessor, Name* name) +{ + return LookupMacro(GetCurrentEnvironment(preprocessor), name); +} + +// A macro is "busy" if it is currently being used for expansion. +// A macro cannot be expanded again while busy, to avoid infinite recursion. +static bool IsMacroBusy(PreprocessorMacro* /*macro*/) +{ + // TODO: need to implement this correctly + // + // The challenge here is that we are implementing expansion + // for argumenst to function-like macros in a "lazy" fashion. + // + // The letter of the spec is that we should macro expand + // each argument *before* substitution, and then go and + // macro-expand the substituted body. This means that we + // can invoke a macro as part of an argument to an + // invocation of the same macro: + // + // FOO( 1, FOO(22), 333 ); + // + // In our implementation, the "inner" invocation of `FOO` + // gets expanded at the point where it gets referenced + // in the body of the "outer" invocation of `FOO`. + // Doing things this way leads to greatly simplified + // code for handling expansion. + // + // A proper implementation of `IsMacroBusy` needs to + // take context into account, so that it bans recursive + // use of a macro when it occurs (indirectly) through + // the *body* of the expansion, but not when it occcurs + // only through an *argument*. + return false; +} + +// +// Reading Tokens With Expansion +// + +static void InitializeMacroExpansion( + Preprocessor* preprocessor, + MacroExpansion* expansion, + PreprocessorMacro* macro) +{ + initializeInputStream(preprocessor, expansion); + + expansion->parent = preprocessor->inputStream; + expansion->primaryStream = preprocessor->inputStream->primaryStream; + + expansion->environment = macro->environment; + expansion->macro = macro; + expansion->tokenReader = TokenReader(macro->tokens); +} + +static void PushMacroExpansion( + Preprocessor* preprocessor, + MacroExpansion* expansion) +{ + PushInputStream(preprocessor, expansion); +} + +static void AddEndOfStreamToken( + Preprocessor* preprocessor, + PreprocessorMacro* macro) +{ + Token token = PeekRawToken(preprocessor); + token.type = TokenType::EndOfFile; + macro->tokens.mTokens.add(token); +} + +static SimpleTokenInputStream* createSimpleInputStream( + Preprocessor* preprocessor, + Token const& token) +{ + SimpleTokenInputStream* inputStream = new SimpleTokenInputStream(); + initializeInputStream(preprocessor, inputStream); + + inputStream->lexedTokens.mTokens.add(token); + + Token eofToken; + eofToken.type = TokenType::EndOfFile; + eofToken.loc = token.loc; + eofToken.flags = TokenFlag::AfterWhitespace | TokenFlag::AtStartOfLine; + inputStream->lexedTokens.mTokens.add(eofToken); + + inputStream->tokenReader = TokenReader(inputStream->lexedTokens); + + return inputStream; +} + +// Check whether the current token on the given input stream should be +// treated as a macro invocation, and if so set up state for expanding +// that macro. +static void MaybeBeginMacroExpansion( + Preprocessor* preprocessor ) +{ + // We iterate because the first token in the expansion of one + // macro may be another macro invocation. + for (;;) + { + // Look at the next token ahead of us + Token token = PeekRawToken(preprocessor); + + // Not an identifier? Can't be a macro. + if (token.type != TokenType::Identifier) + return; + + // Look for a macro with the given name. + Name* name = token.getName(); + PreprocessorMacro* macro = LookupMacro(preprocessor, name); + + // Not a macro? Can't be an invocation. + if (!macro) + return; + + // If the macro is busy (already being expanded), + // don't try to trigger recursive expansion + if (IsMacroBusy(macro)) + return; + + // We might already have looked at this token, + // and need to suppress expansion + if (token.flags & TokenFlag::SuppressMacroExpansion) + return; + + // A function-style macro invocation should only match + // if the token *after* the identifier is `(`. This + // requires more lookahead than we usually have/need + if (macro->flavor == PreprocessorMacroFlavor::FunctionLike) + { + // Consume the token that (possibly) triggered macro expansion + AdvanceRawToken(preprocessor); + + // Look at the next token, and see if it is an opening `(` + // that indicates we should actually expand a macro. + if(PeekRawTokenType(preprocessor) != TokenType::LParent) + { + // In this case, we are in a bit of a mess, because we have + // consumed the token that named the macro, but we need to + // make sure that token (and not whatever came after it) + // gets returned to the user. + // + // To work around this we will construct a short-lived input + // stream just to handle that one token, and also set + // a flag on the token to keep us from doing this logic again. + + token.flags |= TokenFlag::SuppressMacroExpansion; + + SimpleTokenInputStream* simpleStream = createSimpleInputStream(preprocessor, token); + PushInputStream(preprocessor, simpleStream); + return; + } + + // Consume the opening `(` + Token leftParen = AdvanceRawToken(preprocessor); + + FunctionLikeMacroExpansion* expansion = new FunctionLikeMacroExpansion(); + InitializeMacroExpansion(preprocessor, expansion, macro); + expansion->argumentEnvironment.parent = &preprocessor->globalEnv; + expansion->environment = &expansion->argumentEnvironment; + + // Try to read any arguments present. + UInt paramCount = macro->params.getCount(); + UInt argIndex = 0; + + switch (PeekRawTokenType(preprocessor)) + { + case TokenType::EndOfFile: + case TokenType::RParent: + // No arguments. + break; + + default: + // At least one argument + while(argIndex < paramCount) + { + // Read an argument + + // Create the argument, represented as a special flavor of macro + PreprocessorMacro* arg = CreateMacro(preprocessor); + arg->flavor = PreprocessorMacroFlavor::FunctionArg; + arg->environment = GetCurrentEnvironment(preprocessor); + + // Associate the new macro with its parameter name + NameLoc paramNameAndLoc = macro->params[argIndex]; + Name* paramName = paramNameAndLoc.name; + arg->nameAndLoc = paramNameAndLoc; + expansion->argumentEnvironment.macros[paramName] = arg; + argIndex++; + + // Read tokens for the argument + + // We track the nesting depth, since we don't break + // arguments on a `,` nested in balanced parentheses + // + int nesting = 0; + for (;;) + { + switch (PeekRawTokenType(preprocessor)) + { + case TokenType::EndOfFile: + // if we reach the end of the file, + // then we have an error, and need to + // bail out + AddEndOfStreamToken(preprocessor, arg); + goto doneWithAllArguments; + + case TokenType::RParent: + // If we see a right paren when we aren't nested + // then we are at the end of an argument + if (nesting == 0) + { + AddEndOfStreamToken(preprocessor, arg); + goto doneWithAllArguments; + } + // Otherwise we decrease our nesting depth, add + // the token, and keep going + nesting--; + break; + + case TokenType::Comma: + // If we see a comma when we aren't nested + // then we are at the end of an argument + if (nesting == 0) + { + AddEndOfStreamToken(preprocessor, arg); + AdvanceRawToken(preprocessor); + goto doneWithArgument; + } + // Otherwise we add it as a normal token + break; + + case TokenType::LParent: + // If we see a left paren then we need to + // increase our tracking of nesting + nesting++; + break; + + default: + break; + } + + // Add the token and continue parsing. + arg->tokens.mTokens.add(AdvanceRawToken(preprocessor)); + } + doneWithArgument: {} + // We've parsed an argument and should move onto + // the next one. + } + break; + } + doneWithAllArguments: + // TODO: handle possible varargs + + // Expect closing right paren + if (PeekRawTokenType(preprocessor) == TokenType::RParent) + { + AdvanceRawToken(preprocessor); + } + else + { + GetSink(preprocessor)->diagnose(PeekLoc(preprocessor), Diagnostics::expectedTokenInMacroArguments, TokenType::RParent, PeekRawTokenType(preprocessor)); + } + + UInt argCount = argIndex; + if (argCount != paramCount) + { + GetSink(preprocessor)->diagnose(PeekLoc(preprocessor), Diagnostics::wrongNumberOfArgumentsToMacro, paramCount, argCount); + } + + // We are ready to expand. + PushMacroExpansion(preprocessor, expansion); + } + else + { + // Consume the token that triggered macro expansion + AdvanceRawToken(preprocessor); + + // Object-like macros are the easy case. + ObjectLikeMacroExpansion* expansion = new ObjectLikeMacroExpansion(); + InitializeMacroExpansion(preprocessor, expansion, macro); + PushMacroExpansion(preprocessor, expansion); + } + } +} + +// Read one token with macro-expansion enabled. +static Token AdvanceToken(Preprocessor* preprocessor) +{ +top: + // Check whether we need to macro expand at the cursor. + MaybeBeginMacroExpansion(preprocessor); + + // Read a raw token (now that expansion has been triggered) + Token token = AdvanceRawToken(preprocessor); + + // Check if we need to perform token pasting + if (PeekRawTokenType(preprocessor) != TokenType::PoundPound) + { + // If we aren't token pasting, then we are done + return token; + } + else + { + // We are pasting tokens, which could get messy + + StringBuilder sb; + sb << token.Content; + + while (PeekRawTokenType(preprocessor) == TokenType::PoundPound) + { + // Consume the `##` + AdvanceRawToken(preprocessor); + + // Possibly macro-expand the next token + MaybeBeginMacroExpansion(preprocessor); + + // Read the next raw token (now that expansion has been triggered) + Token nextToken = AdvanceRawToken(preprocessor); + + sb << nextToken.Content; + } + + // Now re-lex the input + + SourceManager* sourceManager = preprocessor->getSourceManager(); + + // We create a dummy file to represent the token-paste operation + PathInfo pathInfo = PathInfo::makeTokenPaste(); + + SourceFile* sourceFile = sourceManager->createSourceFileWithString(pathInfo, sb.ProduceString()); + SourceView* sourceView = sourceManager->createSourceView(sourceFile, nullptr); + + Lexer lexer; + lexer.initialize(sourceView, GetSink(preprocessor), preprocessor->getNamePool(), sourceManager->getMemoryArena()); + + SimpleTokenInputStream* inputStream = new SimpleTokenInputStream(); + initializeInputStream(preprocessor, inputStream); + + inputStream->lexedTokens = lexer.lexAllTokens(); + inputStream->tokenReader = TokenReader(inputStream->lexedTokens); + + // We expect the reuslt of lexing to be two tokens: one for the actual value, + // and one for the end-of-input marker. + if (inputStream->tokenReader.GetCount() != 2) + { + // We expect a token paste to produce a single token + // TODO(tfoley): emit a diagnostic here + } + + PushInputStream(preprocessor, inputStream); + goto top; + } +} + +// Read one token with macro-expansion enabled. +// +// Note that because triggering macro expansion may +// involve changing the input-stream state, this +// operation *can* have side effects. +static Token PeekToken(Preprocessor* preprocessor) +{ + // Check whether we need to macro expand at the cursor. + MaybeBeginMacroExpansion(preprocessor); + + // Peek a raw token (now that expansion has been triggered) + return PeekRawToken(preprocessor); + + // TODO: need a plan for how to handle token pasting + // here without it being onerous. Would be nice if we + // didn't have to re-do pasting on a "peek"... +} + +// Peek the type of the next token, including macro expansion. +static TokenType PeekTokenType(Preprocessor* preprocessor) +{ + return PeekToken(preprocessor).type; +} + +// +// Preprocessor Directives +// + +// When reading a preprocessor directive, we use a context +// to wrap the direct preprocessor routines defines so far. +// +// One of the most important things the directive context +// does is give us a convenient way to read tokens with +// a guarantee that we won't read past the end of a line. +struct PreprocessorDirectiveContext +{ + // The preprocessor that is parsing the directive. + Preprocessor* preprocessor; + + // The directive token (e.g., the `if` in `#if`). + // Useful for reference in diagnostic messages. + Token directiveToken; + + // Has any kind of parse error been encountered in + // the directive so far? + bool parseError; + + // Have we done the necessary checks at the end + // of the directive already? + bool haveDoneEndOfDirectiveChecks; +}; + +// Get the token for the preprocessor directive being parsed. +inline Token const& GetDirective(PreprocessorDirectiveContext* context) +{ + return context->directiveToken; +} + +// Get the name of the directive being parsed. +inline UnownedStringSlice const& GetDirectiveName(PreprocessorDirectiveContext* context) +{ + return context->directiveToken.Content; +} + +// Get the location of the directive being parsed. +inline SourceLoc const& GetDirectiveLoc(PreprocessorDirectiveContext* context) +{ + return context->directiveToken.loc; +} + +// Wrapper to get the diagnostic sink in the context of a directive. +static inline DiagnosticSink* GetSink(PreprocessorDirectiveContext* context) +{ + return GetSink(context->preprocessor); +} + +// Wrapper to get a "current" location when parsing a directive +static SourceLoc PeekLoc(PreprocessorDirectiveContext* context) +{ + return PeekLoc(context->preprocessor); +} + +// Wrapper to look up a macro in the context of a directive. +static PreprocessorMacro* LookupMacro(PreprocessorDirectiveContext* context, Name* name) +{ + return LookupMacro(context->preprocessor, name); +} + +// Determine if we have read everything on the directive's line. +static bool IsEndOfLine(PreprocessorDirectiveContext* context) +{ + return PeekRawToken(context->preprocessor).type == TokenType::EndOfDirective; +} + +// Peek one raw token in a directive, without going past the end of the line. +static Token PeekRawToken(PreprocessorDirectiveContext* context) +{ + return PeekRawToken(context->preprocessor); +} + +// Read one raw token in a directive, without going past the end of the line. +static Token AdvanceRawToken(PreprocessorDirectiveContext* context, LexerFlags lexerFlags = 0) +{ + if (IsEndOfLine(context)) + return PeekRawToken(context); + return AdvanceRawToken(context->preprocessor, lexerFlags); +} + +// Peek next raw token type, without going past the end of the line. +static TokenType PeekRawTokenType(PreprocessorDirectiveContext* context) +{ + return PeekRawTokenType(context->preprocessor); +} + +// Read one token, with macro-expansion, without going past the end of the line. +static Token AdvanceToken(PreprocessorDirectiveContext* context) +{ + if (IsEndOfLine(context)) + return PeekRawToken(context); + return AdvanceToken(context->preprocessor); +} + +// Peek one token, with macro-expansion, without going past the end of the line. +static Token PeekToken(PreprocessorDirectiveContext* context) +{ + if (IsEndOfLine(context)) + return context->preprocessor->endOfFileToken; + return PeekToken(context->preprocessor); +} + +// Peek next token type, with macro-expansion, without going past the end of the line. +static TokenType PeekTokenType(PreprocessorDirectiveContext* context) +{ + if (IsEndOfLine(context)) + return TokenType::EndOfDirective; + return PeekTokenType(context->preprocessor); +} + +// Skip to the end of the line (useful for recovering from errors in a directive) +static void SkipToEndOfLine(PreprocessorDirectiveContext* context) +{ + while(!IsEndOfLine(context)) + { + AdvanceRawToken(context); + } +} + +static bool ExpectRaw(PreprocessorDirectiveContext* context, TokenType tokenType, DiagnosticInfo const& diagnostic, Token* outToken = NULL) +{ + if (PeekRawTokenType(context) != tokenType) + { + // Only report the first parse error within a directive + if (!context->parseError) + { + GetSink(context)->diagnose(PeekLoc(context), diagnostic, tokenType, GetDirectiveName(context)); + } + context->parseError = true; + return false; + } + Token const& token = AdvanceRawToken(context); + if (outToken) + *outToken = token; + return true; +} + +static bool Expect(PreprocessorDirectiveContext* context, TokenType tokenType, DiagnosticInfo const& diagnostic, Token* outToken = NULL) +{ + if (PeekTokenType(context) != tokenType) + { + // Only report the first parse error within a directive + if (!context->parseError) + { + GetSink(context)->diagnose(PeekLoc(context), diagnostic, tokenType, GetDirectiveName(context)); + context->parseError = true; + } + return false; + } + Token const& token = AdvanceToken(context); + if (outToken) + *outToken = token; + return true; +} + + + +// +// Preprocessor Conditionals +// + +// Determine whether the current preprocessor state means we +// should be skipping tokens. +static bool IsSkipping(Preprocessor* preprocessor) +{ + PreprocessorInputStream* inputStream = preprocessor->inputStream; + if (!inputStream) return false; + + PrimaryInputStream* primaryStream = inputStream->primaryStream; + if(!primaryStream) return false; + + // If we are not inside a preprocessor conditional, then don't skip + PreprocessorConditional* conditional = primaryStream->conditional; + if (!conditional) return false; + + // skip tokens unless the conditional is inside its `true` case + return conditional->state != PreprocessorConditionalState::During; +} + +// Wrapper for use inside directives +static inline bool IsSkipping(PreprocessorDirectiveContext* context) +{ + return IsSkipping(context->preprocessor); +} + +// Create a preprocessor conditional +static PreprocessorConditional* CreateConditional(Preprocessor* /*preprocessor*/) +{ + // TODO(tfoley): allocate these more intelligently (for example, + // pool them on the `Preprocessor`. + return new PreprocessorConditional(); +} + +// Destroy a preprocessor conditional. +static void DestroyConditional(PreprocessorConditional* conditional) +{ + delete conditional; +} + +// Start a preprocessor conditional, with an initial enable/disable state. +static void beginConditional( + PreprocessorDirectiveContext* context, + PreprocessorInputStream* inputStream, + bool enable) +{ + Preprocessor* preprocessor = context->preprocessor; + SLANG_ASSERT(inputStream); + + PreprocessorConditional* conditional = CreateConditional(preprocessor); + + conditional->ifToken = context->directiveToken; + + // Set state of this condition appropriately. + // + // Default to the "haven't yet seen a `true` branch" state. + PreprocessorConditionalState state = PreprocessorConditionalState::Before; + // + // If we are nested inside a `false` branch of another condition, then + // we never want to enable, so we act as if we already *saw* the `true` branch. + // + if (IsSkipping(preprocessor)) state = PreprocessorConditionalState::After; + // + // Similarly, if we ran into any parse errors when dealing with the + // opening directive, then things are probably screwy and we should just + // skip all the branches. + if (IsSkipping(preprocessor)) state = PreprocessorConditionalState::After; + // + // Otherwise, if our condition was true, then set us to be inside the `true` branch + else if (enable) state = PreprocessorConditionalState::During; + + conditional->state = state; + + // Push conditional onto the stack + auto primaryStream = inputStream->primaryStream; + conditional->parent = primaryStream->conditional; + primaryStream->conditional = conditional; +} + +// Start a preprocessor conditional, with an initial enable/disable state. +static void beginConditional( + PreprocessorDirectiveContext* context, + bool enable) +{ + beginConditional(context, context->preprocessor->inputStream, enable); +} + +// +// Preprocessor Conditional Expressions +// + +// Conditional expressions are always of type `int` +typedef int PreprocessorExpressionValue; + +// Forward-declaretion +static PreprocessorExpressionValue ParseAndEvaluateExpression(PreprocessorDirectiveContext* context); + +// Parse a unary (prefix) expression inside of a preprocessor directive. +static PreprocessorExpressionValue ParseAndEvaluateUnaryExpression(PreprocessorDirectiveContext* context) +{ + switch (PeekTokenType(context)) + { + // handle prefix unary ops + case TokenType::OpSub: + AdvanceToken(context); + return -ParseAndEvaluateUnaryExpression(context); + case TokenType::OpNot: + AdvanceToken(context); + return !ParseAndEvaluateUnaryExpression(context); + case TokenType::OpBitNot: + AdvanceToken(context); + return ~ParseAndEvaluateUnaryExpression(context); + + // handle parenthized sub-expression + case TokenType::LParent: + { + Token leftParen = AdvanceToken(context); + PreprocessorExpressionValue value = ParseAndEvaluateExpression(context); + if (!Expect(context, TokenType::RParent, Diagnostics::expectedTokenInPreprocessorExpression)) + { + GetSink(context)->diagnose(leftParen.loc, Diagnostics::seeOpeningToken, leftParen); + } + return value; + } + + case TokenType::IntegerLiteral: + return StringToInt(AdvanceToken(context).Content); + + case TokenType::Identifier: + { + Token token = AdvanceToken(context); + if (token.Content == "defined") + { + // handle `defined(someName)` + + // Possibly parse a `(` + Token leftParen; + if (PeekRawTokenType(context) == TokenType::LParent) + { + leftParen = AdvanceRawToken(context); + } + + // Expect an identifier + Token nameToken; + if (!ExpectRaw(context, TokenType::Identifier, Diagnostics::expectedTokenInDefinedExpression, &nameToken)) + { + return 0; + } + Name* name = nameToken.getName(); + + // If we saw an opening `(`, then expect one to close + if (leftParen.type != TokenType::Unknown) + { + if(!ExpectRaw(context, TokenType::RParent, Diagnostics::expectedTokenInDefinedExpression)) + { + GetSink(context)->diagnose(leftParen.loc, Diagnostics::seeOpeningToken, leftParen); + return 0; + } + } + + return LookupMacro(context, name) != NULL; + } + + // An identifier here means it was not defined as a macro (or + // it is defined, but as a function-like macro. These should + // just evaluate to zero (possibly with a warning) + GetSink(context)->diagnose(token.loc, Diagnostics::undefinedIdentifierInPreprocessorExpression, token.getName()); + return 0; + } + + default: + GetSink(context)->diagnose(PeekLoc(context), Diagnostics::syntaxErrorInPreprocessorExpression); + return 0; + } +} + +// Determine the precedence level of an infix operator +// for use in parsing preprocessor conditionals. +static int GetInfixOpPrecedence(Token const& opToken) +{ + // If token is on another line, it is not part of the + // expression + if (opToken.flags & TokenFlag::AtStartOfLine) + return -1; + + // otherwise we look at the token type to figure + // out what precedence it should be parse with + switch (opToken.type) + { + default: + // tokens that aren't infix operators should + // cause us to stop parsing an expression + return -1; + + case TokenType::OpMul: return 10; + case TokenType::OpDiv: return 10; + case TokenType::OpMod: return 10; + + case TokenType::OpAdd: return 9; + case TokenType::OpSub: return 9; + + case TokenType::OpLsh: return 8; + case TokenType::OpRsh: return 8; + + case TokenType::OpLess: return 7; + case TokenType::OpGreater: return 7; + case TokenType::OpLeq: return 7; + case TokenType::OpGeq: return 7; + + case TokenType::OpEql: return 6; + case TokenType::OpNeq: return 6; + + case TokenType::OpBitAnd: return 5; + case TokenType::OpBitOr: return 4; + case TokenType::OpBitXor: return 3; + case TokenType::OpAnd: return 2; + case TokenType::OpOr: return 1; + } +}; + +// Evaluate one infix operation in a preprocessor +// conditional expression +static PreprocessorExpressionValue EvaluateInfixOp( + PreprocessorDirectiveContext* context, + Token const& opToken, + PreprocessorExpressionValue left, + PreprocessorExpressionValue right) +{ + switch (opToken.type) + { + default: +// SLANG_INTERNAL_ERROR(getSink(preprocessor), opToken); + return 0; + break; + + case TokenType::OpMul: return left * right; + case TokenType::OpDiv: + { + if (right == 0) + { + if (!context->parseError) + { + GetSink(context)->diagnose(opToken.loc, Diagnostics::divideByZeroInPreprocessorExpression); + } + return 0; + } + return left / right; + } + case TokenType::OpMod: + { + if (right == 0) + { + if (!context->parseError) + { + GetSink(context)->diagnose(opToken.loc, Diagnostics::divideByZeroInPreprocessorExpression); + } + return 0; + } + return left % right; + } + case TokenType::OpAdd: return left + right; + case TokenType::OpSub: return left - right; + case TokenType::OpLsh: return left << right; + case TokenType::OpRsh: return left >> right; + case TokenType::OpLess: return left < right ? 1 : 0; + case TokenType::OpGreater: return left > right ? 1 : 0; + case TokenType::OpLeq: return left <= right ? 1 : 0; + case TokenType::OpGeq: return left >= right ? 1 : 0; + case TokenType::OpEql: return left == right ? 1 : 0; + case TokenType::OpNeq: return left != right ? 1 : 0; + case TokenType::OpBitAnd: return left & right; + case TokenType::OpBitOr: return left | right; + case TokenType::OpBitXor: return left ^ right; + case TokenType::OpAnd: return left && right; + case TokenType::OpOr: return left || right; + } +} + +// Parse the rest of an infix preprocessor expression with +// precedence greater than or equal to the given `precedence` argument. +// The value of the left-hand-side expression is provided as +// an argument. +// This is used to form a simple recursive-descent expression parser. +static PreprocessorExpressionValue ParseAndEvaluateInfixExpressionWithPrecedence( + PreprocessorDirectiveContext* context, + PreprocessorExpressionValue left, + int precedence) +{ + for (;;) + { + // Look at the next token, and see if it is an operator of + // high enough precedence to be included in our expression + Token opToken = PeekToken(context); + int opPrecedence = GetInfixOpPrecedence(opToken); + + // If it isn't an operator of high enough precedence, we are done. + if(opPrecedence < precedence) + break; + + // Otherwise we need to consume the operator token. + AdvanceToken(context); + + // Next we parse a right-hand-side expression by starting with + // a unary expression and absorbing and many infix operators + // as possible with strictly higher precedence than the operator + // we found above. + PreprocessorExpressionValue right = ParseAndEvaluateUnaryExpression(context); + for (;;) + { + // Look for an operator token + Token rightOpToken = PeekToken(context); + int rightOpPrecedence = GetInfixOpPrecedence(rightOpToken); + + // If no operator was found, or the operator wasn't high + // enough precedence to fold into the right-hand-side, + // exit this loop. + if (rightOpPrecedence <= opPrecedence) + break; + + // Now invoke the parser recursively, passing in our + // existing right-hand side to form an even larger one. + right = ParseAndEvaluateInfixExpressionWithPrecedence( + context, + right, + rightOpPrecedence); + } + + // Now combine the left- and right-hand sides using + // the operator we found above. + left = EvaluateInfixOp(context, opToken, left, right); + } + return left; +} + +// Parse a complete (infix) preprocessor expression, and return its value +static PreprocessorExpressionValue ParseAndEvaluateExpression(PreprocessorDirectiveContext* context) +{ + // First read in the left-hand side (or the whole expression in the unary case) + PreprocessorExpressionValue value = ParseAndEvaluateUnaryExpression(context); + + // Try to read in trailing infix operators with correct precedence + return ParseAndEvaluateInfixExpressionWithPrecedence(context, value, 0); +} + +// Handle a `#if` directive +static void HandleIfDirective(PreprocessorDirectiveContext* context) +{ + // Record current input stream in case preprocessor expression + // changes the input stream to a macro expansion while we + // are parsing. + auto inputStream = context->preprocessor->inputStream; + + // If we are skipping, we can just consume the expression, and assume true + if (IsSkipping(context->preprocessor)) + { + // Consume everything until the end of the line + SkipToEndOfLine(context); + // Begin a preprocessor block, assume true based on the expression + // (contents will all be ignored because skipping). + beginConditional(context, inputStream, true); + } + else + { + // Parse a preprocessor expression. + PreprocessorExpressionValue value = ParseAndEvaluateExpression(context); + + // Begin a preprocessor block, enabled based on the expression. + beginConditional(context, inputStream, value != 0); + } +} + +// Handle a `#ifdef` directive +static void HandleIfDefDirective(PreprocessorDirectiveContext* context) +{ + // Expect a raw identifier, so we can check if it is defined + Token nameToken; + if(!ExpectRaw(context, TokenType::Identifier, Diagnostics::expectedTokenInPreprocessorDirective, &nameToken)) + return; + Name* name = nameToken.getName(); + + // Check if the name is defined. + beginConditional(context, LookupMacro(context, name) != NULL); +} + +// Handle a `#ifndef` directive +static void HandleIfNDefDirective(PreprocessorDirectiveContext* context) +{ + // Expect a raw identifier, so we can check if it is defined + Token nameToken; + if(!ExpectRaw(context, TokenType::Identifier, Diagnostics::expectedTokenInPreprocessorDirective, &nameToken)) + return; + Name* name = nameToken.getName(); + + // Check if the name is defined. + beginConditional(context, LookupMacro(context, name) == NULL); +} + +// Handle a `#else` directive +static void HandleElseDirective(PreprocessorDirectiveContext* context) +{ + PreprocessorInputStream* inputStream = context->preprocessor->inputStream; + SLANG_ASSERT(inputStream); + + // if we aren't inside a conditional, then error + PreprocessorConditional* conditional = inputStream->primaryStream->conditional; + if (!conditional) + { + GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::directiveWithoutIf, GetDirectiveName(context)); + return; + } + + // if we've already seen a `#else`, then it is an error + if (conditional->elseToken.type != TokenType::Unknown) + { + GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::directiveAfterElse, GetDirectiveName(context)); + GetSink(context)->diagnose(conditional->elseToken.loc, Diagnostics::seeDirective); + return; + } + conditional->elseToken = context->directiveToken; + + switch (conditional->state) + { + case PreprocessorConditionalState::Before: + conditional->state = PreprocessorConditionalState::During; + break; + + case PreprocessorConditionalState::During: + conditional->state = PreprocessorConditionalState::After; + break; + + default: + break; + } +} + +// Handle a `#elif` directive +static void HandleElifDirective(PreprocessorDirectiveContext* context) +{ + // Need to grab current input stream *before* we try to parse + // the conditional expression. + PreprocessorInputStream* inputStream = context->preprocessor->inputStream; + SLANG_ASSERT(inputStream); + + // HACK(tfoley): handle an empty `elif` like an `else` directive + // + // This is the behavior expected by at least one input program. + // We will eventually want to be pedantic about this. + // even if t + if (PeekRawTokenType(context) == TokenType::EndOfDirective) + { + GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::directiveExpectsExpression, GetDirectiveName(context)); + HandleElseDirective(context); + return; + } + + PreprocessorExpressionValue value = ParseAndEvaluateExpression(context); + + // if we aren't inside a conditional, then error + PreprocessorConditional* conditional = inputStream->primaryStream->conditional; + if (!conditional) + { + GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::directiveWithoutIf, GetDirectiveName(context)); + return; + } + + // if we've already seen a `#else`, then it is an error + if (conditional->elseToken.type != TokenType::Unknown) + { + GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::directiveAfterElse, GetDirectiveName(context)); + GetSink(context)->diagnose(conditional->elseToken.loc, Diagnostics::seeDirective); + return; + } + + switch (conditional->state) + { + case PreprocessorConditionalState::Before: + if(value) + conditional->state = PreprocessorConditionalState::During; + break; + + case PreprocessorConditionalState::During: + conditional->state = PreprocessorConditionalState::After; + break; + + default: + break; + } +} + +// Handle a `#endif` directive +static void HandleEndIfDirective(PreprocessorDirectiveContext* context) +{ + PreprocessorInputStream* inputStream = context->preprocessor->inputStream; + SLANG_ASSERT(inputStream); + + // if we aren't inside a conditional, then error + PreprocessorConditional* conditional = inputStream->primaryStream->conditional; + if (!conditional) + { + GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::directiveWithoutIf, GetDirectiveName(context)); + return; + } + + inputStream->primaryStream->conditional = conditional->parent; + DestroyConditional(conditional); +} + +// Helper routine to check that we find the end of a directive where +// we expect it. +// +// Most directives do not need to call this directly, since we have +// a catch-all case in the main `HandleDirective()` function. +// The `#include` case will call it directly to avoid complications +// when it switches the input stream. +static void expectEndOfDirective(PreprocessorDirectiveContext* context) +{ + if(context->haveDoneEndOfDirectiveChecks) + return; + + context->haveDoneEndOfDirectiveChecks = true; + + if (!IsEndOfLine(context)) + { + // If we already saw a previous parse error, then don't + // emit another one for the same directive. + if (!context->parseError) + { + GetSink(context)->diagnose(PeekLoc(context), Diagnostics::unexpectedTokensAfterDirective, GetDirectiveName(context)); + } + SkipToEndOfLine(context); + } + + // Clear out the end-of-directive token + AdvanceRawToken(context->preprocessor); +} + + /// Read a file in the context of handling a preprocessor directive +static SlangResult readFile( + PreprocessorDirectiveContext* context, + String const& path, + ISlangBlob** outBlob) +{ + // The actual file loading will be handled by the file system + // associated with the parent linkage. + // + auto linkage = context->preprocessor->linkage; + auto fileSystemExt = linkage->getFileSystemExt(); + SLANG_RETURN_ON_FAIL(fileSystemExt->loadFile(path.getBuffer(), outBlob)); + + // If we are running the preprocessor as part of compiling a + // specific module, then we must keep track of the file we've + // read as yet another file that the module will depend on. + // + if(auto module = context->preprocessor->parentModule) + { + module->addFilePathDependency(path); + } + + return SLANG_OK; +} + +// Handle a `#include` directive +static void HandleIncludeDirective(PreprocessorDirectiveContext* context) +{ + // Consume the directive, and inform the lexer to process the remainder of the line as a file path. + AdvanceRawToken(context, kLexerFlag_ExpectFileName); + + Token pathToken; + if(!Expect(context, TokenType::StringLiteral, Diagnostics::expectedTokenInPreprocessorDirective, &pathToken)) + return; + + String path = getFileNameTokenValue(pathToken); + + auto directiveLoc = GetDirectiveLoc(context); + + PathInfo includedFromPathInfo = context->preprocessor->getSourceManager()->getPathInfo(directiveLoc, SourceLocType::Actual); + + IncludeHandler* includeHandler = context->preprocessor->includeHandler; + if (!includeHandler) + { + GetSink(context)->diagnose(pathToken.loc, Diagnostics::includeFailed, path); + GetSink(context)->diagnose(pathToken.loc, Diagnostics::noIncludeHandlerSpecified); + return; + } + + /* Find the path relative to the foundPath */ + PathInfo filePathInfo; + if (SLANG_FAILED(includeHandler->findFile(path, includedFromPathInfo.foundPath, filePathInfo))) + { + GetSink(context)->diagnose(pathToken.loc, Diagnostics::includeFailed, path); + return; + } + + // We must have a uniqueIdentity to be compare + if (!filePathInfo.hasUniqueIdentity()) + { + GetSink(context)->diagnose(pathToken.loc, Diagnostics::noUniqueIdentity, path); + return; + } + + // Do all checking related to the end of this directive before we push a new stream, + // just to avoid complications where that check would need to deal with + // a switch of input stream + expectEndOfDirective(context); + + // Check whether we've previously included this file and seen a `#pragma once` directive + if(context->preprocessor->pragmaOnceUniqueIdentities.Contains(filePathInfo.uniqueIdentity)) + { + return; + } + + // Simplify the path + filePathInfo.foundPath = includeHandler->simplifyPath(filePathInfo.foundPath); + + // Push the new file onto our stack of input streams + // TODO(tfoley): check if we have made our include stack too deep + auto sourceManager = context->preprocessor->getSourceManager(); + + // See if this an already loaded source file + SourceFile* sourceFile = sourceManager->findSourceFileRecursively(filePathInfo.uniqueIdentity); + // If not create a new one, and add to the list of known source files + if (!sourceFile) + { + ComPtr foundSourceBlob; + if (SLANG_FAILED(readFile(context, filePathInfo.foundPath, foundSourceBlob.writeRef()))) + { + GetSink(context)->diagnose(pathToken.loc, Diagnostics::includeFailed, path); + return; + } + + + sourceFile = sourceManager->createSourceFileWithBlob(filePathInfo, foundSourceBlob); + sourceManager->addSourceFile(filePathInfo.uniqueIdentity, sourceFile); + } + + // This is a new parse (even if it's a pre-existing source file), so create a new SourceUnit + SourceView* sourceView = sourceManager->createSourceView(sourceFile, &filePathInfo); + + PreprocessorInputStream* inputStream = CreateInputStreamForSource(context->preprocessor, sourceView); + inputStream->parent = context->preprocessor->inputStream; + context->preprocessor->inputStream = inputStream; +} + +// Handle a `#define` directive +static void HandleDefineDirective(PreprocessorDirectiveContext* context) +{ + Token nameToken; + if (!ExpectRaw(context, TokenType::Identifier, Diagnostics::expectedTokenInPreprocessorDirective, &nameToken)) + return; + Name* name = nameToken.getName(); + + PreprocessorMacro* macro = CreateMacro(context->preprocessor); + macro->nameAndLoc = NameLoc(nameToken); + + PreprocessorMacro* oldMacro = LookupMacro(&context->preprocessor->globalEnv, name); + if (oldMacro) + { + GetSink(context)->diagnose(nameToken.loc, Diagnostics::macroRedefinition, name); + GetSink(context)->diagnose(oldMacro->getLoc(), Diagnostics::seePreviousDefinitionOf, name); + + DestroyMacro(context->preprocessor, oldMacro); + } + context->preprocessor->globalEnv.macros[name] = macro; + + // If macro name is immediately followed (with no space) by `(`, + // then we have a function-like macro + if (PeekRawTokenType(context) == TokenType::LParent) + { + if (!(PeekRawToken(context).flags & TokenFlag::AfterWhitespace)) + { + // This is a function-like macro, so we need to remember that + // and start capturing parameters + macro->flavor = PreprocessorMacroFlavor::FunctionLike; + + AdvanceRawToken(context); + + // If there are any parameters, parse them + if (PeekRawTokenType(context) != TokenType::RParent) + { + for (;;) + { + // TODO: handle elipsis (`...`) for varags + + // A macro parameter name should be a raw identifier + Token paramToken; + if (!ExpectRaw(context, TokenType::Identifier, Diagnostics::expectedTokenInMacroParameters, ¶mToken)) + break; + + // TODO(tfoley): some validation on parameter name. + // Certain names (e.g., `defined` and `__VA_ARGS__` + // are not allowed to be used as macros or parameters). + + // Add the parameter to the macro being deifned + macro->params.add(paramToken); + + // If we see `)` then we are done with arguments + if (PeekRawTokenType(context) == TokenType::RParent) + break; + + ExpectRaw(context, TokenType::Comma, Diagnostics::expectedTokenInMacroParameters); + } + } + + ExpectRaw(context, TokenType::RParent, Diagnostics::expectedTokenInMacroParameters); + } + } + + // consume tokens until end-of-line + for(;;) + { + Token token = AdvanceRawToken(context); + if( token.type == TokenType::EndOfDirective ) + { + // Last token on line will be turned into a conceptual end-of-file + // token for the sub-stream that the macro expands into. + token.type = TokenType::EndOfFile; + macro->tokens.mTokens.add(token); + break; + } + + // In the ordinary case, we just add the token to the definition + macro->tokens.mTokens.add(token); + } +} + +// Handle a `#undef` directive +static void HandleUndefDirective(PreprocessorDirectiveContext* context) +{ + Token nameToken; + if (!ExpectRaw(context, TokenType::Identifier, Diagnostics::expectedTokenInPreprocessorDirective, &nameToken)) + return; + Name* name = nameToken.getName(); + + PreprocessorEnvironment* env = &context->preprocessor->globalEnv; + PreprocessorMacro* macro = LookupMacro(env, name); + if (macro != NULL) + { + // name was defined, so remove it + env->macros.Remove(name); + + DestroyMacro(context->preprocessor, macro); + } + else + { + // name wasn't defined + GetSink(context)->diagnose(nameToken.loc, Diagnostics::macroNotDefined, name); + } +} + +// Handle a `#warning` directive +static void HandleWarningDirective(PreprocessorDirectiveContext* context) +{ + // Consume the directive, and inform the lexer to process the remainder of the line as a custom message. + AdvanceRawToken(context, kLexerFlag_ExpectDirectiveMessage); + + // Read the message token. + Token messageToken; + Expect(context, TokenType::DirectiveMessage, Diagnostics::expectedTokenInPreprocessorDirective, &messageToken); + + // Report the custom error. + GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::userDefinedWarning, messageToken.Content); +} + +// Handle a `#error` directive +static void HandleErrorDirective(PreprocessorDirectiveContext* context) +{ + // Consume the directive, and inform the lexer to process the remainder of the line as a custom message. + AdvanceRawToken(context, kLexerFlag_ExpectDirectiveMessage); + + // Read the message token. + Token messageToken; + Expect(context, TokenType::DirectiveMessage, Diagnostics::expectedTokenInPreprocessorDirective, &messageToken); + + // Report the custom error. + GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::userDefinedError, messageToken.Content); +} + +// Handle a `#line` directive +static void HandleLineDirective(PreprocessorDirectiveContext* context) +{ + auto inputStream = context->preprocessor->inputStream; + + int line = 0; + + SourceLoc directiveLoc = GetDirectiveLoc(context); + + // `#line ...` + if (PeekTokenType(context) == TokenType::IntegerLiteral) + { + line = StringToInt(AdvanceToken(context).Content); + } + // `#line` + // `#line default` + else if ( + PeekTokenType(context) == TokenType::EndOfDirective + || (PeekTokenType(context) == TokenType::Identifier + && PeekToken(context).Content == "default")) + { + AdvanceToken(context); + + // Stop overriding source locations. + auto sourceView = inputStream->primaryStream->lexer.sourceView; + sourceView->addDefaultLineDirective(directiveLoc); + return; + } + else + { + GetSink(context)->diagnose(PeekLoc(context), Diagnostics::expected2TokensInPreprocessorDirective, + TokenType::IntegerLiteral, + "default", + GetDirectiveName(context)); + context->parseError = true; + return; + } + + auto sourceManager = context->preprocessor->getSourceManager(); + + String file; + if (PeekTokenType(context) == TokenType::EndOfDirective) + { + file = sourceManager->getPathInfo(directiveLoc).foundPath; + } + else if (PeekTokenType(context) == TokenType::StringLiteral) + { + file = getStringLiteralTokenValue(AdvanceToken(context)); + } + else if (PeekTokenType(context) == TokenType::IntegerLiteral) + { + // Note(tfoley): GLSL allows the "source string" to be indicated by an integer + // TODO(tfoley): Figure out a better way to handle this, if it matters + file = AdvanceToken(context).Content; + } + else + { + Expect(context, TokenType::StringLiteral, Diagnostics::expectedTokenInPreprocessorDirective); + return; + } + + auto sourceView = inputStream->primaryStream->lexer.sourceView; + sourceView->addLineDirective(directiveLoc, file, line); +} + +#define SLANG_PRAGMA_DIRECTIVE_CALLBACK(NAME) \ + void NAME(PreprocessorDirectiveContext* context, Token subDirectiveToken) + +// Callback interface used by `#pragma` directives +typedef SLANG_PRAGMA_DIRECTIVE_CALLBACK((*PragmaDirectiveCallback)); + +SLANG_PRAGMA_DIRECTIVE_CALLBACK(handleUnknownPragmaDirective) +{ + GetSink(context)->diagnose(subDirectiveToken, Diagnostics::unknownPragmaDirectiveIgnored, subDirectiveToken.getName()); + SkipToEndOfLine(context); + return; +} + +SLANG_PRAGMA_DIRECTIVE_CALLBACK(handlePragmaOnceDirective) +{ + // We need to identify the path of the file we are preprocessing, + // so that we can avoid including it again. + // + // We are using the 'uniqueIdentity' as determined by the ISlangFileSystemEx interface to determine file identities. + + auto directiveLoc = GetDirectiveLoc(context); + auto issuedFromPathInfo = context->preprocessor->getSourceManager()->getPathInfo(directiveLoc, SourceLocType::Actual); + + // Must have uniqueIdentity for a #pragma once to work + if (!issuedFromPathInfo.hasUniqueIdentity()) + { + GetSink(context)->diagnose(subDirectiveToken, Diagnostics::pragmaOnceIgnored); + return; + } + + context->preprocessor->pragmaOnceUniqueIdentities.Add(issuedFromPathInfo.uniqueIdentity); +} + +// Information about a specific `#pragma` directive +struct PragmaDirective +{ + // name of the directive + char const* name; + + // Callback to handle the directive + PragmaDirectiveCallback callback; +}; + +// A simple array of all the `#pragma` directives we know how to handle. +static const PragmaDirective kPragmaDirectives[] = +{ + { "once", &handlePragmaOnceDirective }, + + { NULL, NULL }, +}; + +static const PragmaDirective kUnknownPragmaDirective = { + NULL, &handleUnknownPragmaDirective, +}; + +// Look up the `#pragma` directive with the given name. +static PragmaDirective const* findPragmaDirective(String const& name) +{ + char const* nameStr = name.getBuffer(); + for (int ii = 0; kPragmaDirectives[ii].name; ++ii) + { + if (strcmp(kPragmaDirectives[ii].name, nameStr) != 0) + continue; + + return &kPragmaDirectives[ii]; + } + + return &kUnknownPragmaDirective; +} + +// Handle a `#pragma` directive +static void HandlePragmaDirective(PreprocessorDirectiveContext* context) +{ + // Try to read the sub-directive name. + Token subDirectiveToken = PeekRawToken(context); + + // The sub-directive had better be an identifier + if (subDirectiveToken.type != TokenType::Identifier) + { + GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::expectedPragmaDirectiveName); + SkipToEndOfLine(context); + return; + } + AdvanceRawToken(context); + + // Look up the handler for the sub-directive. + PragmaDirective const* subDirective = findPragmaDirective(subDirectiveToken.getName()->text); + + // Apply the sub-directive-specific callback + (subDirective->callback)(context, subDirectiveToken); +} + +// Handle an invalid directive +static void HandleInvalidDirective(PreprocessorDirectiveContext* context) +{ + GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::unknownPreprocessorDirective, GetDirectiveName(context)); + SkipToEndOfLine(context); +} + +// Callback interface used by preprocessor directives +typedef void (*PreprocessorDirectiveCallback)(PreprocessorDirectiveContext* context); + +enum PreprocessorDirectiveFlag : unsigned int +{ + // Should this directive be handled even when skipping disbaled code? + ProcessWhenSkipping = 1 << 0, + + /// Allow the handler for this directive to advance past the + /// directive token itself, so that it can control lexer behavior + /// more closely. + DontConsumeDirectiveAutomatically = 1 << 1, +}; + +// Information about a specific directive +struct PreprocessorDirective +{ + // name of the directive + char const* name; + + // Callback to handle the directive + PreprocessorDirectiveCallback callback; + + unsigned int flags; +}; + +// A simple array of all the directives we know how to handle. +// TODO(tfoley): considering making this into a real hash map, +// and then make it easy-ish for users of the codebase to add +// their own directives as desired. +static const PreprocessorDirective kDirectives[] = +{ + { "if", &HandleIfDirective, ProcessWhenSkipping }, + { "ifdef", &HandleIfDefDirective, ProcessWhenSkipping }, + { "ifndef", &HandleIfNDefDirective, ProcessWhenSkipping }, + { "else", &HandleElseDirective, ProcessWhenSkipping }, + { "elif", &HandleElifDirective, ProcessWhenSkipping }, + { "endif", &HandleEndIfDirective, ProcessWhenSkipping }, + + { "include", &HandleIncludeDirective, DontConsumeDirectiveAutomatically }, + { "define", &HandleDefineDirective, 0 }, + { "undef", &HandleUndefDirective, 0 }, + { "warning", &HandleWarningDirective, DontConsumeDirectiveAutomatically }, + { "error", &HandleErrorDirective, DontConsumeDirectiveAutomatically }, + { "line", &HandleLineDirective, 0 }, + { "pragma", &HandlePragmaDirective, 0 }, + + { nullptr, nullptr, 0 }, +}; + +static const PreprocessorDirective kInvalidDirective = { + nullptr, &HandleInvalidDirective, 0, +}; + +// Look up the directive with the given name. +static PreprocessorDirective const* FindDirective(String const& name) +{ + char const* nameStr = name.getBuffer(); + for (int ii = 0; kDirectives[ii].name; ++ii) + { + if (strcmp(kDirectives[ii].name, nameStr) != 0) + continue; + + return &kDirectives[ii]; + } + + return &kInvalidDirective; +} + +// Process a directive, where the preprocessor has already consumed the +// `#` token that started the directive line. +static void HandleDirective(PreprocessorDirectiveContext* context) +{ + // Try to read the directive name. + context->directiveToken = PeekRawToken(context); + + TokenType directiveTokenType = GetDirective(context).type; + + // An empty directive is allowed, and ignored. + if (directiveTokenType == TokenType::EndOfDirective) + { + return; + } + // Otherwise the directive name had better be an identifier + else if (directiveTokenType != TokenType::Identifier) + { + GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::expectedPreprocessorDirectiveName); + SkipToEndOfLine(context); + return; + } + + // Look up the handler for the directive. + PreprocessorDirective const* directive = FindDirective(GetDirectiveName(context)); + + // If we are skipping disabled code, and the directive is not one + // of the small number that need to run even in that case, skip it. + if (IsSkipping(context) && !(directive->flags & PreprocessorDirectiveFlag::ProcessWhenSkipping)) + { + SkipToEndOfLine(context); + return; + } + + if(!(directive->flags & PreprocessorDirectiveFlag::DontConsumeDirectiveAutomatically)) + { + // Consume the directive name token. + AdvanceRawToken(context); + } + + // Apply the directive-specific callback + (directive->callback)(context); + + // We expect the directive callback to consume the entire line, so if + // it hasn't that is a parse error. + expectEndOfDirective(context); +} + +// Read one token using the full preprocessor, with all its behaviors. +static Token ReadToken(Preprocessor* preprocessor) +{ + for (;;) + { + // Depending on what the lookahead token is, we + // might need to start expanding it. + // + // Note: doing this at the start of this loop + // is important, in case a macro has an empty + // expansion, and we end up looking at a different + // token after applying the expansion. + if(!IsSkipping(preprocessor)) + { + MaybeBeginMacroExpansion(preprocessor); + } + + // Look at the next raw token in the input. + Token const& token = PeekRawToken(preprocessor); + if (token.type == TokenType::EndOfFile) + return token; + + // If we have a directive (`#` at start of line) then handle it + if ((token.type == TokenType::Pound) && (token.flags & TokenFlag::AtStartOfLine)) + { + // Skip the `#` + AdvanceRawToken(preprocessor); + + // Create a context for parsing the directive + PreprocessorDirectiveContext directiveContext; + directiveContext.preprocessor = preprocessor; + directiveContext.parseError = false; + directiveContext.haveDoneEndOfDirectiveChecks = false; + + // Parse and handle the directive + HandleDirective(&directiveContext); + continue; + } + + // otherwise, if we are currently in a skipping mode, then skip tokens + if (IsSkipping(preprocessor)) + { + AdvanceRawToken(preprocessor); + continue; + } + + // otherwise read a token, which may involve macro expansion + return AdvanceToken(preprocessor); + } +} + +// intialize a preprocessor context, using the given sink for errros +static void InitializePreprocessor( + Preprocessor* preprocessor, + DiagnosticSink* sink) +{ + preprocessor->sink = sink; + preprocessor->includeHandler = NULL; + preprocessor->endOfFileToken.type = TokenType::EndOfFile; + preprocessor->endOfFileToken.flags = TokenFlag::AtStartOfLine; +} + +// clean up after an environment +PreprocessorEnvironment::~PreprocessorEnvironment() +{ + for (auto pair : this->macros) + { + DestroyMacro(NULL, pair.Value); + } +} + +// finalize a preprocessor and free any memory still in use +static void FinalizePreprocessor( + Preprocessor* preprocessor) +{ + // Clear out any waiting input streams + PreprocessorInputStream* input = preprocessor->inputStream; + while (input) + { + PreprocessorInputStream* parent = input->parent; + EndInputStream(preprocessor, input); + input = parent; + } + +#if 0 + // clean up any macros that were allocated + for (auto pair : preprocessor->globalEnv.macros) + { + DestroyMacro(preprocessor, pair.Value); + } +#endif +} + +// Add a simple macro definition from a string (e.g., for a +// `-D` option passed on the command line +static void DefineMacro( + Preprocessor* preprocessor, + String const& key, + String const& value) +{ + PathInfo pathInfo = PathInfo::makeCommandLine(); + + PreprocessorMacro* macro = CreateMacro(preprocessor); + + auto sourceManager = preprocessor->getSourceManager(); + + SourceFile* keyFile = sourceManager->createSourceFileWithString(pathInfo, key); + SourceFile* valueFile = sourceManager->createSourceFileWithString(pathInfo, value); + + SourceView* keyView = sourceManager->createSourceView(keyFile, nullptr); + SourceView* valueView = sourceManager->createSourceView(valueFile, nullptr); + + // Use existing `Lexer` to generate a token stream. + Lexer lexer; + lexer.initialize(valueView, GetSink(preprocessor), preprocessor->getNamePool(), sourceManager->getMemoryArena()); + macro->tokens = lexer.lexAllTokens(); + + Name* keyName = preprocessor->getNamePool()->getName(key); + + macro->nameAndLoc.name = keyName; + macro->nameAndLoc.loc = keyView->getRange().begin; + + PreprocessorMacro* oldMacro = NULL; + if (preprocessor->globalEnv.macros.TryGetValue(keyName, oldMacro)) + { + DestroyMacro(preprocessor, oldMacro); + } + + preprocessor->globalEnv.macros[keyName] = macro; +} + +// read the entire input into tokens +static TokenList ReadAllTokens( + Preprocessor* preprocessor) +{ + TokenList tokens; + for (;;) + { + Token token = ReadToken(preprocessor); + + tokens.mTokens.add(token); + + // Note: we include the EOF token in the list, + // since that is expected by the `TokenList` type. + if (token.type == TokenType::EndOfFile) + break; + } + return tokens; +} + +TokenList preprocessSource( + SourceFile* file, + DiagnosticSink* sink, + IncludeHandler* includeHandler, + Dictionary defines, + Linkage* linkage, + Module* parentModule) +{ + Preprocessor preprocessor; + InitializePreprocessor(&preprocessor, sink); + preprocessor.linkage = linkage; + preprocessor.parentModule = parentModule; + + preprocessor.includeHandler = includeHandler; + for (auto p : defines) + { + DefineMacro(&preprocessor, p.Key, p.Value); + } + + SourceManager* sourceManager = linkage->getSourceManager(); + + SourceView* sourceView = sourceManager->createSourceView(file, nullptr); + + // create an initial input stream based on the provided buffer + preprocessor.inputStream = CreateInputStreamForSource(&preprocessor, sourceView); + + TokenList tokens = ReadAllTokens(&preprocessor); + + FinalizePreprocessor(&preprocessor); + + // debugging: build the pre-processed source back together +#if 0 + StringBuilder sb; + for (auto t : tokens) + { + if (t.flags & TokenFlag::AtStartOfLine) + { + sb << "\n"; + } + else if (t.flags & TokenFlag::AfterWhitespace) + { + sb << " "; + } + + sb << t.Content; + } + + String s = sb.ProduceString(); +#endif + + return tokens; +} + +} -- cgit v1.2.3