diff options
Diffstat (limited to 'source/slang/preprocessor.cpp')
| -rw-r--r-- | source/slang/preprocessor.cpp | 2032 |
1 files changed, 2032 insertions, 0 deletions
diff --git a/source/slang/preprocessor.cpp b/source/slang/preprocessor.cpp new file mode 100644 index 000000000..cdde2591d --- /dev/null +++ b/source/slang/preprocessor.cpp @@ -0,0 +1,2032 @@ +// Preprocessor.cpp +#include "Preprocessor.h" + +#include "Diagnostics.h" +#include "Lexer.h" + +// Needed so that we can construct modifier syntax +// to represent GLSL directives +#include "Syntax.h" + +#include <assert.h> + +using namespace CoreLib; + +// This file provides an implementation of a simple C-style preprocessor. +// It does not aim for 100% compatibility with any particular preprocessor +// specification, but the goal is to have it accept the most common +// idioms for using the preprocessor, found in shader code in the wild. + + +namespace Slang{ namespace Compiler { + +// State of a preprocessor conditional, which can change when +// we encounter directives like `#elif` or `#endif` +enum class PreprocessorConditionalState +{ + Before, // We have not yet seen a branch with a `true` condition. + During, // We are inside the branch with a `true` condition. + After, // We have already seen the branch with a `true` condition. +}; + +// Represents a preprocessor conditional that we are currently +// nested inside. +struct PreprocessorConditional +{ + // The next outer conditional in the current file/stream, or NULL. + PreprocessorConditional* parent; + + // The directive token that started the conditional (an `#if` or `#ifdef`) + Token ifToken; + + // The `#else` directive token, if one has been seen (otherwise `TokenType::Unknown`) + Token elseToken; + + // The state of the conditional + PreprocessorConditionalState state; +}; + +struct PreprocessorMacro; + +struct PreprocessorEnvironment +{ + // The "outer" environment, to be used if lookup in this env fails + PreprocessorEnvironment* parent = NULL; + + // Macros defined in this environment + Dictionary<String, PreprocessorMacro*> macros; + + ~PreprocessorEnvironment(); +}; + +// Input tokens can either come from source text, or from macro expansion. +// In general, input streams can be nested, so we have to keep a conceptual +// stack of input. + +// A stream of input tokens to be consumed +struct PreprocessorInputStream +{ + // The next input stream up the stack, if any. + PreprocessorInputStream* parent; + + // The deepest preprocessor conditional active for this stream. + PreprocessorConditional* conditional; + + // Environment to use when looking up macros + PreprocessorEnvironment* environment; + + // Reader for pre-tokenized input + TokenReader tokenReader; + + // If we are clobbering source locations with `#line`, then + // the state is tracked here: + + // Are we overriding source locations? + bool isOverridingSourceLoc; + + // What is the file name we are overriding to? + String overrideFileName; + + // What is the relative offset to apply to any line numbers? + int overrideLineOffset; + + // Destructor is virtual so that we can clean up + // after concrete subtypes. + virtual ~PreprocessorInputStream() = default; +}; + +struct SourceTextInputStream : PreprocessorInputStream +{ + // The pre-tokenized input + TokenList lexedTokens; +}; + +struct MacroExpansion : PreprocessorInputStream +{ + // The macro we will expand + PreprocessorMacro* macro; +}; + +struct ObjectLikeMacroExpansion : MacroExpansion +{ +}; + +struct FunctionLikeMacroExpansion : MacroExpansion +{ + // Environment for macro arguments + PreprocessorEnvironment argumentEnvironment; +}; + +// An enumeration for the diferent types of macros +enum class PreprocessorMacroFlavor +{ + ObjectLike, + FunctionArg, + FunctionLike, +}; + +// In the current design (which we may want to re-consider), +// a macro is a specialized flavor of input stream, that +// captures the token list in its expansion, and then +// can be "played back." +struct PreprocessorMacro +{ + // The name under which the macro was `#define`d + Token nameToken; + + // Parameters of the macro, in case of a function-like macro + List<Token> params; + + // The tokens that make up the macro body + TokenList tokens; + + // The flavor of macro + PreprocessorMacroFlavor flavor; + + // The environment in which this macro needs to be expanded. + // For ordinary macros this will be the global environment, + // while for function-like macro arguments, it will be + // the environment of the macro invocation. + PreprocessorEnvironment* environment; +}; + +// State of the preprocessor +struct Preprocessor +{ + // diagnostics sink to use when writing messages + DiagnosticSink* sink; + + // An external callback interface to use when looking + // for files in a `#include` directive + IncludeHandler* includeHandler; + + // Current input stream (top of the stack of input) + PreprocessorInputStream* inputStream; + + // Currently-defined macros + PreprocessorEnvironment globalEnv; + + // A pre-allocated token that can be returned to + // represent end-of-input situations. + Token endOfFileToken; + + // Syntax for the program we are trying to parse + ProgramSyntaxNode* syntax; +}; + +// Convenience routine to access the diagnostic sink +static DiagnosticSink* GetSink(Preprocessor* preprocessor) +{ + return preprocessor->sink; +} + +// +// Forward declarations +// + +static void DestroyConditional(PreprocessorConditional* conditional); +static void DestroyMacro(Preprocessor* preprocessor, PreprocessorMacro* macro); + +// +// Basic Input Handling +// + +// Create a fresh input stream +static void InitializeInputStream(Preprocessor* preprocessor, PreprocessorInputStream* inputStream) +{ + inputStream->parent = NULL; + inputStream->conditional = NULL; + inputStream->environment = &preprocessor->globalEnv; +} + +// Destroy an input stream +static void DestroyInputStream(Preprocessor* /*preprocessor*/, PreprocessorInputStream* inputStream) +{ + delete inputStream; +} + +// Create an input stream to represent a pre-tokenized input file. +// TODO(tfoley): pre-tokenizing files isn't going to work in the long run. +static PreprocessorInputStream* CreateInputStreamForSource(Preprocessor* preprocessor, CoreLib::String const& source, CoreLib::String const& fileName) +{ + SourceTextInputStream* inputStream = new SourceTextInputStream(); + InitializeInputStream(preprocessor, inputStream); + + // Use existing `Lexer` to generate a token stream. + Lexer lexer(fileName, source, GetSink(preprocessor)); + inputStream->lexedTokens = lexer.lexAllTokens(); + inputStream->tokenReader = TokenReader(inputStream->lexedTokens); + + return inputStream; +} + + + +static void PushInputStream(Preprocessor* preprocessor, PreprocessorInputStream* inputStream) +{ + inputStream->parent = preprocessor->inputStream; + preprocessor->inputStream = inputStream; +} + +// Called when we reach the end of an input stream. +// Performs some validation and then destroys the input stream if required. +static void EndInputStream(Preprocessor* preprocessor, PreprocessorInputStream* inputStream) +{ + // If there are any conditionals that weren't completed, then it is an error + if (inputStream->conditional) + { + PreprocessorConditional* conditional = inputStream->conditional; + + GetSink(preprocessor)->diagnose(conditional->ifToken.Position, Diagnostics::endOfFileInPreprocessorConditional); + + while (conditional) + { + PreprocessorConditional* parent = conditional->parent; + DestroyConditional(conditional); + conditional = parent; + } + } + + DestroyInputStream(preprocessor, inputStream); +} + +// Potentially clobber source location information based on `#line` +static Token PossiblyOverrideSourceLoc(PreprocessorInputStream* inputStream, Token const& token) +{ + Token result = token; + if( inputStream->isOverridingSourceLoc ) + { + result.Position.FileName = inputStream->overrideFileName; + result.Position.Line += inputStream->overrideLineOffset; + } + return result; +} + +// Consume one token from an input stream +static Token AdvanceRawToken(PreprocessorInputStream* inputStream) +{ + return PossiblyOverrideSourceLoc(inputStream, inputStream->tokenReader.AdvanceToken()); +} + +// Peek one token from an input stream +static Token PeekRawToken(PreprocessorInputStream* inputStream) +{ + return PossiblyOverrideSourceLoc(inputStream, inputStream->tokenReader.PeekToken()); +} + +// Peek one token type from an input stream +static TokenType PeekRawTokenType(PreprocessorInputStream* inputStream) +{ + return inputStream->tokenReader.PeekTokenType(); +} + + +// Read one token in "raw" mode (meaning don't expand macros) +static Token AdvanceRawToken(Preprocessor* preprocessor) +{ + for (;;) + { + // Look at the input stream on top of the stack + PreprocessorInputStream* inputStream = preprocessor->inputStream; + + // If there isn't one, then there is no more input left to read. + if (!inputStream) + { + return preprocessor->endOfFileToken; + } + + // The top-most input stream may be at its end + if (PeekRawTokenType(inputStream) == TokenType::EndOfFile) + { + // If there is another stream remaining, switch to it + if (inputStream->parent) + { + preprocessor->inputStream = inputStream->parent; + EndInputStream(preprocessor, inputStream); + continue; + } + else + { + // HACK(tfoley): A place to fall into debugger... + int f = 0; + } + } + + // Everything worked, so read a token from the top-most stream + return AdvanceRawToken(inputStream); + } +} + +// Return the next token in "raw" mode, but don't advance the +// current token state. +static Token PeekRawToken(Preprocessor* preprocessor) +{ + // We need to find the strema that `advanceRawToken` would read from. + PreprocessorInputStream* inputStream = preprocessor->inputStream; + for (;;) + { + if (!inputStream) + { + // No more input streams left to read + return preprocessor->endOfFileToken; + } + + // The top-most input stream may be at its end, so + // look one entry up the stack (don't actually pop + // here, since we are just peeking) + if (PeekRawTokenType(inputStream) == TokenType::EndOfFile) + { + if (inputStream->parent) + { + inputStream = inputStream->parent; + continue; + } + else + { + // HACK(tfoley): A place to fall into debugger... + int f = 0; + } + } + + // Everything worked, so the token we just peeked is fine. + return PeekRawToken(inputStream); + } +} + +// Without advancing preprocessor state, look *two* raw tokens ahead +// (This is only needed in order to determine when we are possibly +// expanding a function-style macro) +TokenType PeekSecondRawTokenType(Preprocessor* preprocessor) +{ + // We need to find the strema that `advanceRawToken` would read from. + PreprocessorInputStream* inputStream = preprocessor->inputStream; + int count = 1; + for (;;) + { + if (!inputStream) + { + // No more input streams left to read + return TokenType::EndOfFile; + } + + // The top-most input stream may be at its end, so + // look one entry up the stack (don't actually pop + // here, since we are just peeking) + + TokenReader reader = inputStream->tokenReader; + if (reader.PeekTokenType() == TokenType::EndOfFile) + { + inputStream = inputStream->parent; + continue; + } + + if (count) + { + count--; + + // Note: we are advancing our temporary + // copy of the token reader + reader.AdvanceToken(); + if (reader.PeekTokenType() == TokenType::EndOfFile) + { + inputStream = inputStream->parent; + continue; + } + } + + // Everything worked, so peek a token from the top-most stream + return reader.PeekTokenType(); + } +} + + +// Get the location of the current (raw) token +static CodePosition PeekLoc(Preprocessor* preprocessor) +{ + return PeekRawToken(preprocessor).Position; +} + +// Get the `TokenType` of the current (raw) token +static TokenType PeekRawTokenType(Preprocessor* preprocessor) +{ + return PeekRawToken(preprocessor).Type; +} + +// +// Macros +// + +// Create a macro +static PreprocessorMacro* CreateMacro(Preprocessor* preprocessor) +{ + // TODO(tfoley): Allocate these more intelligently. + // For example, consider pooling them on the preprocessor. + + PreprocessorMacro* macro = new PreprocessorMacro(); + macro->flavor = PreprocessorMacroFlavor::ObjectLike; + macro->environment = &preprocessor->globalEnv; + return macro; +} + +// Destroy a macro +static void DestroyMacro(Preprocessor* /*preprocessor*/, PreprocessorMacro* macro) +{ + delete macro; +} + + +// Find the currently-defined macro of the given name, or return NULL +static PreprocessorMacro* LookupMacro(PreprocessorEnvironment* environment, String const& name) +{ + for(PreprocessorEnvironment* e = environment; e; e = e->parent) + { + PreprocessorMacro* macro = NULL; + if (e->macros.TryGetValue(name, macro)) + return macro; + } + + return NULL; +} + +static PreprocessorEnvironment* GetCurrentEnvironment(Preprocessor* preprocessor) +{ + PreprocessorInputStream* inputStream = preprocessor->inputStream; + return inputStream ? inputStream->environment : &preprocessor->globalEnv; +} + +static PreprocessorMacro* LookupMacro(Preprocessor* preprocessor, String const& name) +{ + return LookupMacro(GetCurrentEnvironment(preprocessor), name); +} + +// A macro is "busy" if it is currently being used for expansion. +// A macro cannot be expanded again while busy, to avoid infinite recursion. +static bool IsMacroBusy(PreprocessorMacro* /*macro*/) +{ + // TODO: need to implement this correctly + // + // The challenge here is that we are implementing expansion + // for argumenst to function-like macros in a "lazy" fashion. + // + // The letter of the spec is that we should macro expand + // each argument *before* substitution, and then go and + // macro-expand the substituted body. This means that we + // can invoke a macro as part of an argument to an + // invocation of the same macro: + // + // FOO( 1, FOO(22), 333 ); + // + // In our implementation, the "inner" invocation of `FOO` + // gets expanded at the point where it gets referenced + // in the body of the "outer" invocation of `FOO`. + // Doing things this way leads to greatly simplified + // code for handling expansion. + // + // A proper implementation of `IsMacroBusy` needs to + // take context into account, so that it bans recursive + // use of a macro when it occurs (indirectly) through + // the *body* of the expansion, but not when it occcurs + // only through an *argument*. + return false; +} + +// +// Reading Tokens With Expansion +// + +static void InitializeMacroExpansion( + Preprocessor* preprocessor, + MacroExpansion* expansion, + PreprocessorMacro* macro) +{ + InitializeInputStream(preprocessor, expansion); + expansion->environment = macro->environment; + expansion->macro = macro; + expansion->tokenReader = TokenReader(macro->tokens); +} + +static void PushMacroExpansion( + Preprocessor* preprocessor, + MacroExpansion* expansion) +{ + PushInputStream(preprocessor, expansion); +} + +static void AddEndOfStreamToken( + Preprocessor* preprocessor, + PreprocessorMacro* macro) +{ + Token token = PeekRawToken(preprocessor); + token.Type = TokenType::EndOfFile; + macro->tokens.mTokens.Add(token); +} + +// Check whether the current token on the given input stream should be +// treated as a macro invocation, and if so set up state for expanding +// that macro. +static void MaybeBeginMacroExpansion( + Preprocessor* preprocessor ) +{ + // We iterate because the first token in the expansion of one + // macro may be another macro invocation. + for (;;) + { + // Look at the next token ahead of us + Token const& token = PeekRawToken(preprocessor); + + // Not an identifier? Can't be a macro. + if (token.Type != TokenType::Identifier) + return; + + // Look for a macro with the given name. + String name = token.Content; + PreprocessorMacro* macro = LookupMacro(preprocessor, name); + + // Not a macro? Can't be an invocation. + if (!macro) + return; + + // If the macro is busy (already being expanded), + // don't try to trigger recursive expansion + if (IsMacroBusy(macro)) + return; + + // A function-style macro invocation should only match + // if the token *after* the identifier is `(`. This + // requires more lookahead than we usually have/need + if (macro->flavor == PreprocessorMacroFlavor::FunctionLike) + { + if(PeekSecondRawTokenType(preprocessor) != TokenType::LParent) + return; + + // Consume the token that triggered macro expansion + AdvanceRawToken(preprocessor); + + // Consume the opening `(` + Token leftParen = AdvanceRawToken(preprocessor); + + FunctionLikeMacroExpansion* expansion = new FunctionLikeMacroExpansion(); + InitializeMacroExpansion(preprocessor, expansion, macro); + expansion->argumentEnvironment.parent = &preprocessor->globalEnv; + expansion->environment = &expansion->argumentEnvironment; + + // Try to read any arguments present. + int paramCount = macro->params.Count(); + int argIndex = 0; + + switch (PeekRawTokenType(preprocessor)) + { + case TokenType::EndOfFile: + case TokenType::RParent: + // No arguments. + break; + + default: + // At least one argument + while(argIndex < paramCount) + { + // Read an argument + + // Create the argument, represented as a special flavor of macro + PreprocessorMacro* arg = CreateMacro(preprocessor); + arg->flavor = PreprocessorMacroFlavor::FunctionArg; + arg->environment = GetCurrentEnvironment(preprocessor); + + // Associate the new macro with its parameter name + Token paramToken = macro->params[argIndex]; + String const& paramName = paramToken.Content; + arg->nameToken = paramToken; + expansion->argumentEnvironment.macros[paramName] = arg; + argIndex++; + + // Read tokens for the argument + + // We track the nesting depth, since we don't break + // arguments on a `,` nested in balanced parentheses + // + int nesting = 0; + for (;;) + { + switch (PeekRawTokenType(preprocessor)) + { + case TokenType::EndOfFile: + // if we reach the end of the file, + // then we have an error, and need to + // bail out + AddEndOfStreamToken(preprocessor, arg); + goto doneWithAllArguments; + + case TokenType::RParent: + // If we see a right paren when we aren't nested + // then we are at the end of an argument + if (nesting == 0) + { + AddEndOfStreamToken(preprocessor, arg); + goto doneWithAllArguments; + } + // Otherwise we decrease our nesting depth, add + // the token, and keep going + nesting--; + break; + + case TokenType::Comma: + // If we see a comma when we aren't nested + // then we are at the end of an argument + if (nesting == 0) + { + AddEndOfStreamToken(preprocessor, arg); + AdvanceRawToken(preprocessor); + goto doneWithArgument; + } + // Otherwise we add it as a normal token + break; + + case TokenType::LParent: + // If we see a left paren then we need to + // increase our tracking of nesting + nesting++; + break; + + default: + break; + } + + // Add the token and continue parsing. + arg->tokens.mTokens.Add(AdvanceRawToken(preprocessor)); + } + doneWithArgument: {} + // We've parsed an argument and should move onto + // the next one. + } + break; + } + doneWithAllArguments: + // TODO: handle possible varargs + + // Expect closing right paren + if (PeekRawTokenType(preprocessor) == TokenType::RParent) + { + AdvanceRawToken(preprocessor); + } + else + { + GetSink(preprocessor)->diagnose(PeekLoc(preprocessor), Diagnostics::expectedTokenInMacroArguments, TokenType::RParent, PeekRawTokenType(preprocessor)); + } + + int argCount = argIndex; + if (argCount != paramCount) + { + // TODO: diagnose + throw 99; + } + + // We are ready to expand. + PushMacroExpansion(preprocessor, expansion); + } + else + { + // Consume the token that triggered macro expansion + AdvanceRawToken(preprocessor); + + // Object-like macros are the easy case. + ObjectLikeMacroExpansion* expansion = new ObjectLikeMacroExpansion(); + InitializeMacroExpansion(preprocessor, expansion, macro); + PushMacroExpansion(preprocessor, expansion); + } + } +} + +// Read one token with macro-expansion enabled. +static Token AdvanceToken(Preprocessor* preprocessor) +{ +top: + // Check whether we need to macro expand at the cursor. + MaybeBeginMacroExpansion(preprocessor); + + // Read a raw token (now that expansion has been triggered) + Token token = AdvanceRawToken(preprocessor); + + // Check if we need to perform token pasting + if (PeekRawTokenType(preprocessor) != TokenType::PoundPound) + { + // If we aren't token pasting, then we are done + return token; + } + else + { + // We are pasting tokens, which could get messy + + StringBuilder sb; + sb << token.Content; + + while (PeekRawTokenType(preprocessor) == TokenType::PoundPound) + { + // Consume the `##` + AdvanceRawToken(preprocessor); + + // Possibly macro-expand the next token + MaybeBeginMacroExpansion(preprocessor); + + // Read the next raw token (now that expansion has been triggered) + Token nextToken = AdvanceRawToken(preprocessor); + + sb << nextToken.Content; + } + + // Now re-lex the input + PreprocessorInputStream* inputStream = CreateInputStreamForSource(preprocessor, sb.ProduceString(), "token paste"); + if (inputStream->tokenReader.GetCount() != 1) + { + // We expect a token paste to produce a single token + // TODO(tfoley): emit a diagnostic here + } + + PushInputStream(preprocessor, inputStream); + goto top; + } +} + +// Read one token with macro-expansion enabled. +// +// Note that because triggering macro expansion may +// involve changing the input-stream state, this +// operation *can* have side effects. +static Token PeekToken(Preprocessor* preprocessor) +{ + // Check whether we need to macro expand at the cursor. + MaybeBeginMacroExpansion(preprocessor); + + // Peek a raw token (now that expansion has been triggered) + return PeekRawToken(preprocessor); + + // TODO: need a plan for how to handle token pasting + // here without it being onerous. Would be nice if we + // didn't have to re-do pasting on a "peek"... +} + +// Peek the type of the next token, including macro expansion. +static TokenType PeekTokenType(Preprocessor* preprocessor) +{ + return PeekToken(preprocessor).Type; +} + +// +// Preprocessor Directives +// + +// When reading a preprocessor directive, we use a context +// to wrap the direct preprocessor routines defines so far. +// +// One of the most important things the directive context +// does is give us a convenient way to read tokens with +// a guarantee that we won't read past the end of a line. +struct PreprocessorDirectiveContext +{ + // The preprocessor that is parsing the directive. + Preprocessor* preprocessor; + + // The directive token (e.g., the `if` in `#if`). + // Useful for reference in diagnostic messages. + Token directiveToken; + + // Has any kind of parse error been encountered in + // the directive so far? + bool parseError; + + // Have we done the necessary checks at the end + // of the directive already? + bool haveDoneEndOfDirectiveChecks; +}; + +// Get the token for the preprocessor directive being parsed. +inline Token const& GetDirective(PreprocessorDirectiveContext* context) +{ + return context->directiveToken; +} + +// Get the name of the directive being parsed. +inline String const& GetDirectiveName(PreprocessorDirectiveContext* context) +{ + return context->directiveToken.Content; +} + +// Get the location of the directive being parsed. +inline CodePosition const& GetDirectiveLoc(PreprocessorDirectiveContext* context) +{ + return context->directiveToken.Position; +} + +// Wrapper to get the diagnostic sink in the context of a directive. +static inline DiagnosticSink* GetSink(PreprocessorDirectiveContext* context) +{ + return GetSink(context->preprocessor); +} + +// Wrapper to get a "current" location when parsing a directive +static CodePosition PeekLoc(PreprocessorDirectiveContext* context) +{ + return PeekLoc(context->preprocessor); +} + +// Wrapper to look up a macro in the context of a directive. +static PreprocessorMacro* LookupMacro(PreprocessorDirectiveContext* context, String const& name) +{ + return LookupMacro(context->preprocessor, name); +} + +// Determine if we have read everthing on the directive's line. +static bool IsEndOfLine(PreprocessorDirectiveContext* context) +{ + return PeekRawToken(context->preprocessor).Type == TokenType::EndOfDirective; +} + +// Peek one raw token in a directive, without going past the end of the line. +static Token PeekRawToken(PreprocessorDirectiveContext* context) +{ + return PeekRawToken(context->preprocessor); +} + +// Read one raw token in a directive, without going past the end of the line. +static Token AdvanceRawToken(PreprocessorDirectiveContext* context) +{ + if (IsEndOfLine(context)) + return PeekRawToken(context); + return AdvanceRawToken(context->preprocessor); +} + +// Peek next raw token type, without going past the end of the line. +static TokenType PeekRawTokenType(PreprocessorDirectiveContext* context) +{ + return PeekRawTokenType(context->preprocessor); +} + +// Read one token, with macro-expansion, without going past the end of the line. +static Token AdvanceToken(PreprocessorDirectiveContext* context) +{ + if (IsEndOfLine(context)) + return PeekRawToken(context); + return AdvanceToken(context->preprocessor); +} + +// Peek one token, with macro-expansion, without going past the end of the line. +static Token PeekToken(PreprocessorDirectiveContext* context) +{ + if (IsEndOfLine(context)) + context->preprocessor->endOfFileToken; + return PeekToken(context->preprocessor); +} + +// Peek next token type, with macro-expansion, without going past the end of the line. +static TokenType PeekTokenType(PreprocessorDirectiveContext* context) +{ + if (IsEndOfLine(context)) + return TokenType::EndOfDirective; + return PeekTokenType(context->preprocessor); +} + +// Skip to the end of the line (useful for recovering from errors in a directive) +static void SkipToEndOfLine(PreprocessorDirectiveContext* context) +{ + while(!IsEndOfLine(context)) + { + AdvanceRawToken(context); + } +} + +static bool ExpectRaw(PreprocessorDirectiveContext* context, TokenType tokenType, DiagnosticInfo const& diagnostic, Token* outToken = NULL) +{ + if (PeekRawTokenType(context) != tokenType) + { + // Only report the first parse error within a directive + if (!context->parseError) + { + GetSink(context)->diagnose(PeekLoc(context), diagnostic, tokenType, GetDirectiveName(context)); + } + context->parseError = true; + return false; + } + Token const& token = AdvanceRawToken(context); + if (outToken) + *outToken = token; + return true; +} + +static bool Expect(PreprocessorDirectiveContext* context, TokenType tokenType, DiagnosticInfo const& diagnostic, Token* outToken = NULL) +{ + if (PeekTokenType(context) != tokenType) + { + // Only report the first parse error within a directive + if (!context->parseError) + { + GetSink(context)->diagnose(PeekLoc(context), diagnostic, tokenType, GetDirectiveName(context)); + context->parseError = true; + } + return false; + } + Token const& token = AdvanceToken(context); + if (outToken) + *outToken = token; + return true; +} + + + +// +// Preprocessor Conditionals +// + +// Determine whether the current preprocessor state means we +// should be skipping tokens. +static bool IsSkipping(Preprocessor* preprocessor) +{ + PreprocessorInputStream* inputStream = preprocessor->inputStream; + if (!inputStream) return false; + + // If we are not inside a preprocessor conditional, then don't skip + PreprocessorConditional* conditional = inputStream->conditional; + if (!conditional) return false; + + // skip tokens unless the conditional is inside its `true` case + return conditional->state != PreprocessorConditionalState::During; +} + +// Wrapper for use inside directives +static inline bool IsSkipping(PreprocessorDirectiveContext* context) +{ + return IsSkipping(context->preprocessor); +} + +// Create a preprocessor conditional +static PreprocessorConditional* CreateConditional(Preprocessor* /*preprocessor*/) +{ + // TODO(tfoley): allocate these more intelligently (for example, + // pool them on the `Preprocessor`. + return new PreprocessorConditional(); +} + +// Destroy a preprocessor conditional. +static void DestroyConditional(PreprocessorConditional* conditional) +{ + delete conditional; +} + +// Start a preprocessor conditional, with an initial enable/disable state. +static void BeginConditional(PreprocessorDirectiveContext* context, bool enable) +{ + Preprocessor* preprocessor = context->preprocessor; + PreprocessorInputStream* inputStream = preprocessor->inputStream; + assert(inputStream); + + PreprocessorConditional* conditional = CreateConditional(preprocessor); + + conditional->ifToken = context->directiveToken; + + // Set state of this condition appropriately. + // + // Default to the "haven't yet seen a `true` branch" state. + PreprocessorConditionalState state = PreprocessorConditionalState::Before; + // + // If we are nested inside a `false` branch of another condition, then + // we never want to enable, so we act as if we already *saw* the `true` branch. + // + if (IsSkipping(preprocessor)) state = PreprocessorConditionalState::After; + // + // Similarly, if we ran into any parse errors when dealing with the + // opening directive, then things are probably screwy and we should just + // skip all the branches. + if (IsSkipping(preprocessor)) state = PreprocessorConditionalState::After; + // + // Otherwise, if our condition was true, then set us to be inside the `true` branch + else if (enable) state = PreprocessorConditionalState::During; + + conditional->state = state; + + // Push conditional onto the stack + conditional->parent = inputStream->conditional; + inputStream->conditional = conditional; +} + +// +// Preprocessor Conditional Expressions +// + +// Conditional expressions are always of type `int` +typedef int PreprocessorExpressionValue; + +// Forward-declaretion +static PreprocessorExpressionValue ParseAndEvaluateExpression(PreprocessorDirectiveContext* context); + +// Parse a unary (prefix) expression inside of a preprocessor directive. +static PreprocessorExpressionValue ParseAndEvaluateUnaryExpression(PreprocessorDirectiveContext* context) +{ + switch (PeekTokenType(context)) + { + // handle prefix unary ops + case TokenType::OpSub: + AdvanceToken(context); + return -ParseAndEvaluateUnaryExpression(context); + case TokenType::OpNot: + AdvanceToken(context); + return !ParseAndEvaluateUnaryExpression(context); + case TokenType::OpBitNot: + AdvanceToken(context); + return ~ParseAndEvaluateUnaryExpression(context); + + // handle parenthized sub-expression + case TokenType::LParent: + { + Token leftParen = AdvanceToken(context); + PreprocessorExpressionValue value = ParseAndEvaluateExpression(context); + if (!Expect(context, TokenType::RParent, Diagnostics::expectedTokenInPreprocessorExpression)) + { + GetSink(context)->diagnose(leftParen.Position, Diagnostics::seeOpeningToken, leftParen); + } + return value; + } + + case TokenType::IntLiterial: + return StringToInt(AdvanceToken(context).Content); + + case TokenType::Identifier: + { + Token token = AdvanceToken(context); + if (token.Content == "defined") + { + // handle `defined(someName)` + + // Possibly parse a `(` + Token leftParen; + if (PeekRawTokenType(context) == TokenType::LParent) + { + leftParen = AdvanceRawToken(context); + } + + // Expect an identifier + Token nameToken; + if (!ExpectRaw(context, TokenType::Identifier, Diagnostics::expectedTokenInDefinedExpression, &nameToken)) + { + return 0; + } + String name = nameToken.Content; + + // If we saw an opening `(`, then expect one to close + if (leftParen.Type != TokenType::Unknown) + { + if(!ExpectRaw(context, TokenType::RParent, Diagnostics::expectedTokenInDefinedExpression)) + { + GetSink(context)->diagnose(leftParen.Position, Diagnostics::seeOpeningToken, leftParen); + return 0; + } + } + + return LookupMacro(context, name) != NULL; + } + + // An identifier here means it was not defined as a macro (or + // it is defined, but as a function-like macro. These should + // just evaluate to zero (possibly with a warning) + return 0; + } + + default: + GetSink(context)->diagnose(PeekLoc(context), Diagnostics::syntaxErrorInPreprocessorExpression); + return 0; + } +} + +// Determine the precedence level of an infix operator +// for use in parsing preprocessor conditionals. +static int GetInfixOpPrecedence(Token const& opToken) +{ + // If token is on another line, it is not part of the + // expression + if (opToken.flags & TokenFlag::AtStartOfLine) + return -1; + + // otherwise we look at the token type to figure + // out what precednece it should be parse with + switch (opToken.Type) + { + default: + // tokens that aren't infix operators should + // cause us to stop parsing an expression + return -1; + + case TokenType::OpMul: return 10; + case TokenType::OpDiv: return 10; + case TokenType::OpMod: return 10; + + case TokenType::OpAdd: return 9; + case TokenType::OpSub: return 9; + + case TokenType::OpLsh: return 8; + case TokenType::OpRsh: return 8; + + case TokenType::OpLess: return 7; + case TokenType::OpGreater: return 7; + case TokenType::OpLeq: return 7; + case TokenType::OpGeq: return 7; + + case TokenType::OpEql: return 6; + case TokenType::OpNeq: return 6; + + case TokenType::OpBitAnd: return 5; + case TokenType::OpBitOr: return 4; + case TokenType::OpBitXor: return 3; + case TokenType::OpAnd: return 2; + case TokenType::OpOr: return 1; + } +}; + +// Evaluate one infix operation in a preprocessor +// conditional expression +static PreprocessorExpressionValue EvaluateInfixOp( + PreprocessorDirectiveContext* context, + Token const& opToken, + PreprocessorExpressionValue left, + PreprocessorExpressionValue right) +{ + switch (opToken.Type) + { + default: +// SLANG_INTERNAL_ERROR(getSink(preprocessor), opToken); + return 0; + break; + + case TokenType::OpMul: return left * right; + case TokenType::OpDiv: + { + if (right == 0) + { + if (!context->parseError) + { + GetSink(context)->diagnose(opToken.Position, Diagnostics::divideByZeroInPreprocessorExpression); + } + return 0; + } + return left / right; + } + case TokenType::OpMod: + { + if (right == 0) + { + if (!context->parseError) + { + GetSink(context)->diagnose(opToken.Position, Diagnostics::divideByZeroInPreprocessorExpression); + } + return 0; + } + return left % right; + } + case TokenType::OpAdd: return left + right; + case TokenType::OpSub: return left - right; + case TokenType::OpLsh: return left << right; + case TokenType::OpRsh: return left >> right; + case TokenType::OpLess: return left < right ? 1 : 0; + case TokenType::OpGreater: return left > right ? 1 : 0; + case TokenType::OpLeq: return left <= right ? 1 : 0; + case TokenType::OpGeq: return left <= right ? 1 : 0; + case TokenType::OpEql: return left == right ? 1 : 0; + case TokenType::OpNeq: return left != right ? 1 : 0; + case TokenType::OpBitAnd: return left & right; + case TokenType::OpBitOr: return left | right; + case TokenType::OpBitXor: return left ^ right; + case TokenType::OpAnd: return left && right; + case TokenType::OpOr: return left || right; + } +} + +// Parse the rest of an infix preprocessor expression with +// precedence greater than or equal to the given `precedence` argument. +// The value of the left-hand-side expression is provided as +// an argument. +// This is used to form a simple recursive-descent expression parser. +static PreprocessorExpressionValue ParseAndEvaluateInfixExpressionWithPrecedence( + PreprocessorDirectiveContext* context, + PreprocessorExpressionValue left, + int precedence) +{ + for (;;) + { + // Look at the next token, and see if it is an operator of + // high enough precedence to be included in our expression + Token opToken = PeekToken(context); + int opPrecedence = GetInfixOpPrecedence(opToken); + + // If it isn't an operator of high enough precendece, we are done. + if(opPrecedence < precedence) + break; + + // Otherwise we need to consume the operator token. + AdvanceToken(context); + + // Next we parse a right-hand-side expression by starting with + // a unary expression and absorbing and many infix operators + // as possible with strictly higher precedence than the operator + // we found above. + PreprocessorExpressionValue right = ParseAndEvaluateUnaryExpression(context); + for (;;) + { + // Look for an operator token + Token rightOpToken = PeekToken(context); + int rightOpPrecedence = GetInfixOpPrecedence(rightOpToken); + + // If no operator was found, or the operator wasn't high + // enough precedence to fold into the right-hand-side, + // exit this loop. + if (rightOpPrecedence <= opPrecedence) + break; + + // Now invoke the parser recursively, passing in our + // existing right-hand side to form an even larger one. + right = ParseAndEvaluateInfixExpressionWithPrecedence( + context, + right, + rightOpPrecedence); + } + + // Now combine the left- and right-hand sides using + // the operator we found above. + left = EvaluateInfixOp(context, opToken, left, right); + } + return left; +} + +// Parse a complete (infix) preprocessor expression, and return its value +static PreprocessorExpressionValue ParseAndEvaluateExpression(PreprocessorDirectiveContext* context) +{ + // First read in the left-hand side (or the whole expression in the unary case) + PreprocessorExpressionValue value = ParseAndEvaluateUnaryExpression(context); + + // Try to read in trailing infix operators with correct precedence + return ParseAndEvaluateInfixExpressionWithPrecedence(context, value, 0); +} + +// Handle a `#if` directive +static void HandleIfDirective(PreprocessorDirectiveContext* context) +{ + // Parse a preprocessor expression. + PreprocessorExpressionValue value = ParseAndEvaluateExpression(context); + + // Begin a preprocessor block, enabled based on the expression. + BeginConditional(context, value != 0); +} + +// Handle a `#ifdef` directive +static void HandleIfDefDirective(PreprocessorDirectiveContext* context) +{ + // Expect a raw identifier, so we can check if it is defined + Token nameToken; + if(!ExpectRaw(context, TokenType::Identifier, Diagnostics::expectedTokenInPreprocessorDirective, &nameToken)) + return; + String name = nameToken.Content; + + // Check if the name is defined. + BeginConditional(context, LookupMacro(context, name) != NULL); +} + +// Handle a `#ifndef` directive +static void HandleIfNDefDirective(PreprocessorDirectiveContext* context) +{ + // Expect a raw identifier, so we can check if it is defined + Token nameToken; + if(!ExpectRaw(context, TokenType::Identifier, Diagnostics::expectedTokenInPreprocessorDirective, &nameToken)) + return; + String name = nameToken.Content; + + // Check if the name is defined. + BeginConditional(context, LookupMacro(context, name) == NULL); +} + +// Handle a `#else` directive +static void HandleElseDirective(PreprocessorDirectiveContext* context) +{ + PreprocessorInputStream* inputStream = context->preprocessor->inputStream; + assert(inputStream); + + // if we aren't inside a conditional, then error + PreprocessorConditional* conditional = inputStream->conditional; + if (!conditional) + { + GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::directiveWithoutIf, GetDirectiveName(context)); + return; + } + + // if we've already seen a `#else`, then it is an error + if (conditional->elseToken.Type != TokenType::Unknown) + { + GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::directiveAfterElse, GetDirectiveName(context)); + GetSink(context)->diagnose(conditional->elseToken.Position, Diagnostics::seeDirective); + return; + } + conditional->elseToken = context->directiveToken; + + switch (conditional->state) + { + case PreprocessorConditionalState::Before: + conditional->state = PreprocessorConditionalState::During; + break; + + case PreprocessorConditionalState::During: + conditional->state = PreprocessorConditionalState::After; + break; + + default: + break; + } +} + +// Handle a `#elif` directive +static void HandleElifDirective(PreprocessorDirectiveContext* context) +{ + // HACK(tfoley): handle an empty `elif` like an `else` directive + // + // This is the behavior expected by at least one input program. + // We will eventually want to be pedantic about this. + // even if t + if (PeekRawTokenType(context) == TokenType::EndOfDirective) + { + GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::directiveExpectsExpression, GetDirectiveName(context)); + HandleElseDirective(context); + return; + } + + PreprocessorExpressionValue value = ParseAndEvaluateExpression(context); + + PreprocessorInputStream* inputStream = context->preprocessor->inputStream; + assert(inputStream); + + // if we aren't inside a conditional, then error + PreprocessorConditional* conditional = inputStream->conditional; + if (!conditional) + { + GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::directiveWithoutIf, GetDirectiveName(context)); + return; + } + + // if we've already seen a `#else`, then it is an error + if (conditional->elseToken.Type != TokenType::Unknown) + { + GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::directiveAfterElse, GetDirectiveName(context)); + GetSink(context)->diagnose(conditional->elseToken.Position, Diagnostics::seeDirective); + return; + } + + switch (conditional->state) + { + case PreprocessorConditionalState::Before: + if(value) + conditional->state = PreprocessorConditionalState::During; + break; + + case PreprocessorConditionalState::During: + conditional->state = PreprocessorConditionalState::After; + break; + + default: + break; + } +} + +// Handle a `#endif` directive +static void HandleEndIfDirective(PreprocessorDirectiveContext* context) +{ + PreprocessorInputStream* inputStream = context->preprocessor->inputStream; + assert(inputStream); + + // if we aren't inside a conditional, then error + PreprocessorConditional* conditional = inputStream->conditional; + if (!conditional) + { + GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::directiveWithoutIf, GetDirectiveName(context)); + return; + } + + inputStream->conditional = conditional->parent; + DestroyConditional(conditional); +} + +// Helper routine to check that we find the end of a directive where +// we expect it. +// +// Most directives do not need to call this directly, since we have +// a catch-all case in the main `HandleDirective()` funciton. +// The `#include` case will call it directly to avoid complications +// when it switches the input stream. +static void expectEndOfDirective(PreprocessorDirectiveContext* context) +{ + if(context->haveDoneEndOfDirectiveChecks) + return; + + context->haveDoneEndOfDirectiveChecks = true; + + if (!IsEndOfLine(context)) + { + // If we already saw a previous parse error, then don't + // emit another one for the same directive. + if (!context->parseError) + { + GetSink(context)->diagnose(PeekLoc(context), Diagnostics::unexpectedTokensAfterDirective, GetDirectiveName(context)); + } + SkipToEndOfLine(context); + } + + // Clear out the end-of-directive token + AdvanceRawToken(context->preprocessor); +} + + +// Handle a `#include` directive +static void HandleIncludeDirective(PreprocessorDirectiveContext* context) +{ + Token pathToken; + if(!Expect(context, TokenType::StringLiterial, Diagnostics::expectedTokenInPreprocessorDirective, &pathToken)) + return; + + String path = getFileNameTokenValue(pathToken); + + // TODO(tfoley): make this robust in presence of `#line` + String pathIncludedFrom = GetDirectiveLoc(context).FileName; + String foundPath; + String foundSource; + + + IncludeHandler* includeHandler = context->preprocessor->includeHandler; + if (!includeHandler) + { + GetSink(context)->diagnose(pathToken.Position, Diagnostics::includeFailed, path); + GetSink(context)->diagnose(pathToken.Position, Diagnostics::noIncludeHandlerSpecified); + return; + } + if (!includeHandler->TryToFindIncludeFile(path, pathIncludedFrom, &foundPath, &foundSource)) + { + GetSink(context)->diagnose(pathToken.Position, Diagnostics::includeFailed, path); + return; + } + + // Do all checking related to the end of this directive before we push a new stream, + // just to avoid complications where that check would need to deal with + // a switch of input stream + expectEndOfDirective(context); + + // Push the new file onto our stack of input streams + // TODO(tfoley): check if we have made our include stack too deep + PreprocessorInputStream* inputStream = CreateInputStreamForSource(context->preprocessor, foundSource, foundPath); + inputStream->parent = context->preprocessor->inputStream; + context->preprocessor->inputStream = inputStream; +} + +// Handle a `#define` directive +static void HandleDefineDirective(PreprocessorDirectiveContext* context) +{ + Token nameToken; + if (!Expect(context, TokenType::Identifier, Diagnostics::expectedTokenInPreprocessorDirective, &nameToken)) + return; + String name = nameToken.Content; + + PreprocessorMacro* macro = CreateMacro(context->preprocessor); + macro->nameToken = nameToken; + + PreprocessorMacro* oldMacro = LookupMacro(&context->preprocessor->globalEnv, name); + if (oldMacro) + { + GetSink(context)->diagnose(nameToken.Position, Diagnostics::macroRedefinition, name); + GetSink(context)->diagnose(oldMacro->nameToken.Position, Diagnostics::seePreviousDefinitionOf, name); + + DestroyMacro(context->preprocessor, oldMacro); + } + context->preprocessor->globalEnv.macros[name] = macro; + + // If macro name is immediately followed (with no space) by `(`, + // then we have a function-like macro + if (PeekRawTokenType(context) == TokenType::LParent) + { + if (!(PeekRawToken(context).flags & TokenFlag::AfterWhitespace)) + { + // This is a function-like macro, so we need to remember that + // and start capturing parameters + macro->flavor = PreprocessorMacroFlavor::FunctionLike; + + AdvanceRawToken(context); + + // If there are any parameters, parse them + if (PeekRawTokenType(context) != TokenType::RParent) + { + for (;;) + { + // TODO: handle elipsis (`...`) for varags + + // A macro parameter name should be a raw identifier + Token paramToken; + if (!ExpectRaw(context, TokenType::Identifier, Diagnostics::expectedTokenInMacroParameters, ¶mToken)) + break; + + // TODO(tfoley): some validation on parameter name. + // Certain names (e.g., `defined` and `__VA_ARGS__` + // are not allowed to be used as macros or parameters). + + // Add the parameter to the macro being deifned + macro->params.Add(paramToken); + + // If we see `)` then we are done with arguments + if (PeekRawTokenType(context) == TokenType::RParent) + break; + + ExpectRaw(context, TokenType::Comma, Diagnostics::expectedTokenInMacroParameters); + } + } + + ExpectRaw(context, TokenType::RParent, Diagnostics::expectedTokenInMacroParameters); + } + } + + // consume tokens until end-of-line + for(;;) + { + Token token = AdvanceRawToken(context); + if( token.Type == TokenType::EndOfDirective ) + { + // Last token on line will be turned into a conceptual end-of-file + // token for the sub-stream that the macro expands into. + token.Type = TokenType::EndOfFile; + macro->tokens.mTokens.Add(token); + break; + } + + // In the ordinary case, we just add the token to the definition + macro->tokens.mTokens.Add(token); + } +} + +// Handle a `#undef` directive +static void HandleUndefDirective(PreprocessorDirectiveContext* context) +{ + Token nameToken; + if (!Expect(context, TokenType::Identifier, Diagnostics::expectedTokenInPreprocessorDirective, &nameToken)) + return; + String name = nameToken.Content; + + PreprocessorEnvironment* env = &context->preprocessor->globalEnv; + PreprocessorMacro* macro = LookupMacro(env, name); + if (macro != NULL) + { + // name was defined, so remove it + env->macros.Remove(name); + + DestroyMacro(context->preprocessor, macro); + } + else + { + // name wasn't defined + GetSink(context)->diagnose(nameToken.Position, Diagnostics::macroNotDefined, name); + } +} + +// Handle a `#warning` directive +static void HandleWarningDirective(PreprocessorDirectiveContext* context) +{ + // TODO: read rest of line without actual tokenization + GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::userDefinedWarning, "user-defined warning"); + SkipToEndOfLine(context); +} + +// Handle a `#error` directive +static void HandleErrorDirective(PreprocessorDirectiveContext* context) +{ + // TODO: read rest of line without actual tokenization + GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::userDefinedError, "user-defined warning"); + SkipToEndOfLine(context); +} + +// Handle a `#line` directive +static void HandleLineDirective(PreprocessorDirectiveContext* context) +{ + int line = 0; + if (PeekTokenType(context) == TokenType::IntLiterial) + { + line = StringToInt(AdvanceToken(context).Content); + } + else if (PeekTokenType(context) == TokenType::Identifier + && PeekToken(context).Content == "default") + { + AdvanceToken(context); + + // Stop overiding soure locations. + context->preprocessor->inputStream->isOverridingSourceLoc = false; + return; + } + else + { + GetSink(context)->diagnose(PeekLoc(context), Diagnostics::expected2TokensInPreprocessorDirective, + TokenType::IntLiterial, + "default", + GetDirectiveName(context)); + context->parseError = true; + return; + } + + CodePosition directiveLoc = GetDirectiveLoc(context); + + String file; + if (PeekTokenType(context) == TokenType::EndOfDirective) + { + file = directiveLoc.FileName; + } + else if (PeekTokenType(context) == TokenType::StringLiterial) + { + file = AdvanceToken(context).Content; + } + else if (PeekTokenType(context) == TokenType::IntLiterial) + { + // Note(tfoley): GLSL allows the "source string" to be indicated by an integer + // TODO(tfoley): Figure out a better way to handle this, if it matters + file = AdvanceToken(context).Content; + } + else + { + Expect(context, TokenType::StringLiterial, Diagnostics::expectedTokenInPreprocessorDirective); + return; + } + + PreprocessorInputStream* inputStream = context->preprocessor->inputStream; + + inputStream->isOverridingSourceLoc = true; + inputStream->overrideFileName = file; + inputStream->overrideLineOffset = line - (directiveLoc.Line + 1); +} + +// Handle a `#pragma` directive +static void HandlePragmaDirective(PreprocessorDirectiveContext* context) +{ + // TODO(tfoley): figure out which pragmas to parse, + // and which to pass along + SkipToEndOfLine(context); +} + +// Handle a `#version` directive +static void handleGLSLVersionDirective(PreprocessorDirectiveContext* context) +{ + Token versionNumberToken; + if(!ExpectRaw( + context, + TokenType::IntLiterial, + Diagnostics::expectedTokenInPreprocessorDirective, + &versionNumberToken)) + { + return; + } + + Token glslProfileToken; + if(PeekTokenType(context) == TokenType::Identifier) + { + glslProfileToken = AdvanceToken(context); + } + + // Need to construct a representation taht we can hook into our compilation result + + auto modifier = new GLSLVersionDirective(); + modifier->versionNumberToken = versionNumberToken; + modifier->glslProfileToken = glslProfileToken; + + // Attach the modifier to the program we are parsing! + + addModifier( + context->preprocessor->syntax, + modifier); +} + +// Handle a `#extension` directive, e.g., +// +// #extension some_extension_name : enable +// +static void handleGLSLExtensionDirective(PreprocessorDirectiveContext* context) +{ + Token extensionNameToken; + if(!ExpectRaw( + context, + TokenType::Identifier, + Diagnostics::expectedTokenInPreprocessorDirective, + &extensionNameToken)) + { + return; + } + + if( !ExpectRaw(context, TokenType::Colon, Diagnostics::expectedTokenInPreprocessorDirective) ) + { + return; + } + + Token dispositionToken; + if(!ExpectRaw( + context, + TokenType::Identifier, + Diagnostics::expectedTokenInPreprocessorDirective, + &dispositionToken)) + { + return; + } + + // Need to construct a representation taht we can hook into our compilation result + + auto modifier = new GLSLExtensionDirective(); + modifier->extensionNameToken = extensionNameToken; + modifier->dispositionToken = dispositionToken; + + // Attach the modifier to the program we are parsing! + + addModifier( + context->preprocessor->syntax, + modifier); +} + +// Handle an invalid directive +static void HandleInvalidDirective(PreprocessorDirectiveContext* context) +{ + GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::unknownPreprocessorDirective, GetDirectiveName(context)); + SkipToEndOfLine(context); +} + +// Callback interface used by preprocessor directives +typedef void (*PreprocessorDirectiveCallback)(PreprocessorDirectiveContext* context); + +enum PreprocessorDirectiveFlag : unsigned int +{ + // Should this directive be handled even when skipping disbaled code? + ProcessWhenSkipping = 1 << 0, +}; + +// Information about a specific directive +struct PreprocessorDirective +{ + // Name of the directive + char const* name; + + // Callback to handle the directive + PreprocessorDirectiveCallback callback; + + unsigned int flags; +}; + +// A simple array of all the directives we know how to handle. +// TODO(tfoley): considering making this into a real hash map, +// and then make it easy-ish for users of the codebase to add +// their own directives as desired. +static const PreprocessorDirective kDirectives[] = +{ + { "if", &HandleIfDirective, ProcessWhenSkipping }, + { "ifdef", &HandleIfDefDirective, ProcessWhenSkipping }, + { "ifndef", &HandleIfNDefDirective, ProcessWhenSkipping }, + { "else", &HandleElseDirective, ProcessWhenSkipping }, + { "elif", &HandleElifDirective, ProcessWhenSkipping }, + { "endif", &HandleEndIfDirective, ProcessWhenSkipping }, + + { "include", &HandleIncludeDirective, 0 }, + { "define", &HandleDefineDirective, 0 }, + { "undef", &HandleUndefDirective, 0 }, + { "warning", &HandleWarningDirective, 0 }, + { "error", &HandleErrorDirective, 0 }, + { "line", &HandleLineDirective, 0 }, + { "pragma", &HandlePragmaDirective, 0 }, + + // TODO(tfoley): These are specific to GLSL, and probably + // shouldn't be enabled for HLSL or Slang + { "version", &handleGLSLVersionDirective, 0 }, + { "extension", &handleGLSLExtensionDirective, 0 }, + + { NULL, NULL }, +}; + +static const PreprocessorDirective kInvalidDirective = { + NULL, &HandleInvalidDirective, 0, +}; + +// Look up the directive with the given name. +static PreprocessorDirective const* FindDirective(String const& name) +{ + char const* nameStr = name.Buffer(); + for (int ii = 0; kDirectives[ii].name; ++ii) + { + if (strcmp(kDirectives[ii].name, nameStr) != 0) + continue; + + return &kDirectives[ii]; + } + + return &kInvalidDirective; +} + +// Process a directive, where the preprocessor has already consumed the +// `#` token that started the directive line. +static void HandleDirective(PreprocessorDirectiveContext* context) +{ + // Try to read the directive name. + context->directiveToken = PeekRawToken(context); + + TokenType directiveTokenType = GetDirective(context).Type; + + // An empty directive is allowed, and ignored. + if (directiveTokenType == TokenType::EndOfDirective) + { + return; + } + // Otherwise the directive name had better be an identifier + else if (directiveTokenType != TokenType::Identifier) + { + GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::expectedPreprocessorDirectiveName); + SkipToEndOfLine(context); + return; + } + + // Consume the directive name token. + AdvanceRawToken(context); + + // Look up the handler for the directive. + PreprocessorDirective const* directive = FindDirective(GetDirectiveName(context)); + + // If we are skipping disabled code, and the directive is not one + // of the small number that need to run even in that case, skip it. + if (IsSkipping(context) && !(directive->flags & PreprocessorDirectiveFlag::ProcessWhenSkipping)) + { + SkipToEndOfLine(context); + return; + } + + // Apply the directive-specific callback + (directive->callback)(context); + + // We expect the directive callback to consume the entire line, so if + // it hasn't that is a parse error. + expectEndOfDirective(context); +} + +// Read one token using the full preprocessor, with all its behaviors. +static Token ReadToken(Preprocessor* preprocessor) +{ + for (;;) + { + // Look at the next raw token in the input. + Token const& token = PeekRawToken(preprocessor); + + // If we have a directive (`#` at start of line) then handle it + if ((token.Type == TokenType::Pound) && (token.flags & TokenFlag::AtStartOfLine)) + { + // Skip the `#` + AdvanceRawToken(preprocessor); + + // Create a context for parsing the directive + PreprocessorDirectiveContext directiveContext; + directiveContext.preprocessor = preprocessor; + directiveContext.parseError = false; + directiveContext.haveDoneEndOfDirectiveChecks = false; + + // Parse and handle the directive + HandleDirective(&directiveContext); + continue; + } + + // otherwise, if we are currently in a skipping mode, then skip tokens + if (IsSkipping(preprocessor)) + { + AdvanceRawToken(preprocessor); + continue; + } + + // otherwise read a token, which may involve macro expansion + return AdvanceToken(preprocessor); + } +} + +// intialize a preprocessor context, using the given sink for errros +static void InitializePreprocessor( + Preprocessor* preprocessor, + DiagnosticSink* sink) +{ + preprocessor->sink = sink; + preprocessor->includeHandler = NULL; + preprocessor->endOfFileToken.Type = TokenType::EndOfFile; + preprocessor->endOfFileToken.flags = TokenFlag::AtStartOfLine; +} + +// clean up after an environment +PreprocessorEnvironment::~PreprocessorEnvironment() +{ + for (auto pair : this->macros) + { + DestroyMacro(NULL, pair.Value); + } +} + +// finalize a preprocessor and free any memory still in use +static void FinalizePreprocessor( + Preprocessor* preprocessor) +{ + // Clear out any waiting input streams + PreprocessorInputStream* input = preprocessor->inputStream; + while (input) + { + PreprocessorInputStream* parent = input->parent; + DestroyInputStream(preprocessor, input); + input = parent; + } + +#if 0 + // clean up any macros that were allocated + for (auto pair : preprocessor->globalEnv.macros) + { + DestroyMacro(preprocessor, pair.Value); + } +#endif +} + +// Add a simple macro definition from a string (e.g., for a +// `-D` option passed on the command line +static void DefineMacro( + Preprocessor* preprocessor, + String const& key, + String const& value) +{ + String fileName = "command line"; + PreprocessorMacro* macro = CreateMacro(preprocessor); + + // Use existing `Lexer` to generate a token stream. + Lexer lexer(fileName, value, GetSink(preprocessor)); + macro->tokens = lexer.lexAllTokens(); + macro->nameToken = Token(TokenType::Identifier, key, 0, 0, 0, fileName); + + PreprocessorMacro* oldMacro = NULL; + if (preprocessor->globalEnv.macros.TryGetValue(key, oldMacro)) + { + DestroyMacro(preprocessor, oldMacro); + } + + preprocessor->globalEnv.macros[key] = macro; +} + +// read the entire input into tokens +static TokenList ReadAllTokens( + Preprocessor* preprocessor) +{ + TokenList tokens; + for (;;) + { + Token token = ReadToken(preprocessor); + + tokens.mTokens.Add(token); + + // Note: we include the EOF token in the list, + // since that is expected by the `TokenList` type. + if (token.Type == TokenType::EndOfFile) + break; + } + return tokens; +} + +TokenList preprocessSource( + CoreLib::String const& source, + CoreLib::String const& fileName, + DiagnosticSink* sink, + IncludeHandler* includeHandler, + CoreLib::Dictionary<CoreLib::String, CoreLib::String> defines, + ProgramSyntaxNode* syntax) +{ + Preprocessor preprocessor; + InitializePreprocessor(&preprocessor, sink); + preprocessor.syntax = syntax; + + preprocessor.includeHandler = includeHandler; + for (auto p : defines) + { + DefineMacro(&preprocessor, p.Key, p.Value); + } + + // create an initial input stream based on the provided buffer + preprocessor.inputStream = CreateInputStreamForSource(&preprocessor, source, fileName); + + TokenList tokens = ReadAllTokens(&preprocessor); + + FinalizePreprocessor(&preprocessor); + + // debugging: build the pre-processed source back together +#if 0 + StringBuilder sb; + for (auto t : tokens) + { + if (t.flags & TokenFlag::AtStartOfLine) + { + sb << "\n"; + } + else if (t.flags & TokenFlag::AfterWhitespace) + { + sb << " "; + } + + sb << t.Content; + } + + String s = sb.ProduceString(); +#endif + + return tokens; +} + +}} |