Initial import of code.

1 files changed, 2032 insertions, 0 deletions

diff --git a/source/slang/preprocessor.cpp b/source/slang/preprocessor.cpp
new file mode 100644
index 000000000..cdde2591d
--- /dev/null
+++ b/source/slang/preprocessor.cpp
@@ -0,0 +1,2032 @@
+// Preprocessor.cpp
+#include "Preprocessor.h"
+
+#include "Diagnostics.h"
+#include "Lexer.h"
+
+// Needed so that we can construct modifier syntax
+// to represent GLSL directives
+#include "Syntax.h"
+
+#include <assert.h>
+
+using namespace CoreLib;
+
+// This file provides an implementation of a simple C-style preprocessor.
+// It does not aim for 100% compatibility with any particular preprocessor
+// specification, but the goal is to have it accept the most common
+// idioms for using the preprocessor, found in shader code in the wild.
+
+
+namespace Slang{ namespace Compiler {
+
+// State of a preprocessor conditional, which can change when
+// we encounter directives like `#elif` or `#endif`
+enum class PreprocessorConditionalState
+{
+    Before, // We have not yet seen a branch with a `true` condition.
+    During, // We are inside the branch with a `true` condition.
+    After,  // We have already seen the branch with a `true` condition.
+};
+
+// Represents a preprocessor conditional that we are currently
+// nested inside.
+struct PreprocessorConditional
+{
+    // The next outer conditional in the current file/stream, or NULL.
+    PreprocessorConditional*        parent;
+
+    // The directive token that started the conditional (an `#if` or `#ifdef`)
+    Token                           ifToken;
+
+    // The `#else` directive token, if one has been seen (otherwise `TokenType::Unknown`)
+    Token                           elseToken;
+
+    // The state of the conditional
+    PreprocessorConditionalState    state;
+};
+
+struct PreprocessorMacro;
+
+struct PreprocessorEnvironment
+{
+    // The "outer" environment, to be used if lookup in this env fails
+    PreprocessorEnvironment*                parent = NULL;
+
+    // Macros defined in this environment
+    Dictionary<String, PreprocessorMacro*>  macros;
+
+    ~PreprocessorEnvironment();
+};
+
+// Input tokens can either come from source text, or from macro expansion.
+// In general, input streams can be nested, so we have to keep a conceptual
+// stack of input.
+
+// A stream of input tokens to be consumed
+struct PreprocessorInputStream
+{
+    // The next input stream up the stack, if any.
+    PreprocessorInputStream*        parent;
+
+    // The deepest preprocessor conditional active for this stream.
+    PreprocessorConditional*        conditional;
+
+    // Environment to use when looking up macros
+    PreprocessorEnvironment*        environment;
+
+    // Reader for pre-tokenized input
+    TokenReader                     tokenReader;
+
+    // If we are clobbering source locations with `#line`, then
+    // the state is tracked here:
+
+    // Are we overriding source locations?
+    bool                            isOverridingSourceLoc;
+
+    // What is the file name we are overriding to?
+    String                          overrideFileName;
+
+    // What is the relative offset to apply to any line numbers?
+    int                             overrideLineOffset;
+
+    // Destructor is virtual so that we can clean up
+    // after concrete subtypes.
+    virtual ~PreprocessorInputStream() = default;
+};
+
+struct SourceTextInputStream : PreprocessorInputStream
+{
+    // The pre-tokenized input
+    TokenList           lexedTokens;
+};
+
+struct MacroExpansion : PreprocessorInputStream
+{
+    // The macro we will expand
+    PreprocessorMacro*  macro;
+};
+
+struct ObjectLikeMacroExpansion : MacroExpansion
+{
+};
+
+struct FunctionLikeMacroExpansion : MacroExpansion
+{
+    // Environment for macro arguments
+    PreprocessorEnvironment     argumentEnvironment;
+};
+
+// An enumeration for the diferent types of macros
+enum class PreprocessorMacroFlavor
+{
+    ObjectLike,
+    FunctionArg,
+    FunctionLike,
+};
+
+// In the current design (which we may want to re-consider),
+// a macro is a specialized flavor of input stream, that
+// captures the token list in its expansion, and then
+// can be "played back."
+struct PreprocessorMacro
+{
+    // The name under which the macro was `#define`d
+    Token                       nameToken;
+
+    // Parameters of the macro, in case of a function-like macro
+    List<Token>                 params;
+
+    // The tokens that make up the macro body
+    TokenList                   tokens;
+
+    // The flavor of macro
+    PreprocessorMacroFlavor     flavor;
+
+    // The environment in which this macro needs to be expanded.
+    // For ordinary macros this will be the global environment,
+    // while for function-like macro arguments, it will be
+    // the environment of the macro invocation.
+    PreprocessorEnvironment*    environment;
+};
+
+// State of the preprocessor
+struct Preprocessor
+{
+    // diagnostics sink to use when writing messages
+    DiagnosticSink*                         sink;
+
+    // An external callback interface to use when looking
+    // for files in a `#include` directive
+    IncludeHandler*                         includeHandler;
+
+    // Current input stream (top of the stack of input)
+    PreprocessorInputStream*                inputStream;
+
+    // Currently-defined macros
+    PreprocessorEnvironment                 globalEnv;
+
+    // A pre-allocated token that can be returned to
+    // represent end-of-input situations.
+    Token                                   endOfFileToken;
+
+    // Syntax for the program we are trying to parse
+    ProgramSyntaxNode*                      syntax;
+};
+
+// Convenience routine to access the diagnostic sink
+static DiagnosticSink* GetSink(Preprocessor* preprocessor)
+{
+    return preprocessor->sink;
+}
+
+//
+// Forward declarations
+//
+
+static void DestroyConditional(PreprocessorConditional* conditional);
+static void DestroyMacro(Preprocessor* preprocessor, PreprocessorMacro* macro);
+
+//
+// Basic Input Handling
+//
+
+// Create a fresh input stream
+static void  InitializeInputStream(Preprocessor* preprocessor, PreprocessorInputStream* inputStream)
+{
+    inputStream->parent = NULL;
+    inputStream->conditional = NULL;
+    inputStream->environment = &preprocessor->globalEnv;
+}
+
+// Destroy an input stream
+static void DestroyInputStream(Preprocessor* /*preprocessor*/, PreprocessorInputStream* inputStream)
+{
+    delete inputStream;
+}
+
+// Create an input stream to represent a pre-tokenized input file.
+// TODO(tfoley): pre-tokenizing files isn't going to work in the long run.
+static PreprocessorInputStream* CreateInputStreamForSource(Preprocessor* preprocessor, CoreLib::String const& source, CoreLib::String const& fileName)
+{
+    SourceTextInputStream* inputStream = new SourceTextInputStream();
+    InitializeInputStream(preprocessor, inputStream);
+
+    // Use existing `Lexer` to generate a token stream.
+    Lexer lexer(fileName, source, GetSink(preprocessor));
+    inputStream->lexedTokens = lexer.lexAllTokens();
+    inputStream->tokenReader = TokenReader(inputStream->lexedTokens);
+
+    return inputStream;
+}
+
+
+
+static void PushInputStream(Preprocessor* preprocessor, PreprocessorInputStream* inputStream)
+{
+    inputStream->parent = preprocessor->inputStream;
+    preprocessor->inputStream = inputStream;
+}
+
+// Called when we reach the end of an input stream.
+// Performs some validation and then destroys the input stream if required.
+static void EndInputStream(Preprocessor* preprocessor, PreprocessorInputStream* inputStream)
+{
+    // If there are any conditionals that weren't completed, then it is an error
+    if (inputStream->conditional)
+    {
+        PreprocessorConditional* conditional = inputStream->conditional;
+
+        GetSink(preprocessor)->diagnose(conditional->ifToken.Position, Diagnostics::endOfFileInPreprocessorConditional);
+
+        while (conditional)
+        {
+            PreprocessorConditional* parent = conditional->parent;
+            DestroyConditional(conditional);
+            conditional = parent;
+        }
+    }
+
+    DestroyInputStream(preprocessor, inputStream);
+}
+
+// Potentially clobber source location information based on `#line`
+static Token PossiblyOverrideSourceLoc(PreprocessorInputStream* inputStream, Token const& token)
+{
+    Token result = token;
+    if( inputStream->isOverridingSourceLoc )
+    {
+        result.Position.FileName = inputStream->overrideFileName;
+        result.Position.Line += inputStream->overrideLineOffset;
+    }
+    return result;
+}
+
+// Consume one token from an input stream
+static Token AdvanceRawToken(PreprocessorInputStream* inputStream)
+{
+    return PossiblyOverrideSourceLoc(inputStream, inputStream->tokenReader.AdvanceToken());
+}
+
+// Peek one token from an input stream
+static Token PeekRawToken(PreprocessorInputStream* inputStream)
+{
+    return PossiblyOverrideSourceLoc(inputStream, inputStream->tokenReader.PeekToken());
+}
+
+// Peek one token type from an input stream
+static TokenType PeekRawTokenType(PreprocessorInputStream* inputStream)
+{
+    return inputStream->tokenReader.PeekTokenType();
+}
+
+
+// Read one token in "raw" mode (meaning don't expand macros)
+static Token AdvanceRawToken(Preprocessor* preprocessor)
+{
+    for (;;)
+    {
+        // Look at the input stream on top of the stack
+        PreprocessorInputStream* inputStream = preprocessor->inputStream;
+
+        // If there isn't one, then there is no more input left to read.
+        if (!inputStream)
+        {
+            return preprocessor->endOfFileToken;
+        }
+
+        // The top-most input stream may be at its end
+        if (PeekRawTokenType(inputStream) == TokenType::EndOfFile)
+        {
+            // If there is another stream remaining, switch to it
+            if (inputStream->parent)
+            {
+                preprocessor->inputStream = inputStream->parent;
+                EndInputStream(preprocessor, inputStream);
+                continue;
+            }
+            else
+            {
+                // HACK(tfoley): A place to fall into debugger...
+                int f = 0;
+            }
+        }
+
+        // Everything worked, so read a token from the top-most stream
+        return AdvanceRawToken(inputStream);
+    }
+}
+
+// Return the next token in "raw" mode, but don't advance the
+// current token state.
+static Token PeekRawToken(Preprocessor* preprocessor)
+{
+    // We need to find the strema that `advanceRawToken` would read from.
+    PreprocessorInputStream* inputStream = preprocessor->inputStream;
+    for (;;)
+    {
+        if (!inputStream)
+        {
+            // No more input streams left to read
+            return preprocessor->endOfFileToken;
+        }
+
+        // The top-most input stream may be at its end, so
+        // look one entry up the stack (don't actually pop
+        // here, since we are just peeking)
+        if (PeekRawTokenType(inputStream) == TokenType::EndOfFile)
+        {
+            if (inputStream->parent)
+            {
+                inputStream = inputStream->parent;
+                continue;
+            }
+            else
+            {
+                // HACK(tfoley): A place to fall into debugger...
+                int f = 0;
+            }
+        }
+
+        // Everything worked, so the token we just peeked is fine.
+        return PeekRawToken(inputStream);
+    }
+}
+
+// Without advancing preprocessor state, look *two* raw tokens ahead
+// (This is only needed in order to determine when we are possibly
+// expanding a function-style macro)
+TokenType PeekSecondRawTokenType(Preprocessor* preprocessor)
+{
+    // We need to find the strema that `advanceRawToken` would read from.
+    PreprocessorInputStream* inputStream = preprocessor->inputStream;
+    int count = 1;
+    for (;;)
+    {
+        if (!inputStream)
+        {
+            // No more input streams left to read
+            return TokenType::EndOfFile;
+        }
+
+        // The top-most input stream may be at its end, so
+        // look one entry up the stack (don't actually pop
+        // here, since we are just peeking)
+
+        TokenReader reader = inputStream->tokenReader;
+        if (reader.PeekTokenType() == TokenType::EndOfFile)
+        {
+            inputStream = inputStream->parent;
+            continue;
+        }
+
+        if (count)
+        {
+            count--;
+
+            // Note: we are advancing our temporary
+            // copy of the token reader
+            reader.AdvanceToken();
+            if (reader.PeekTokenType() == TokenType::EndOfFile)
+            {
+                inputStream = inputStream->parent;
+                continue;
+            }
+        }
+
+        // Everything worked, so peek a token from the top-most stream
+        return reader.PeekTokenType();
+    }
+}
+
+
+// Get the location of the current (raw) token
+static CodePosition PeekLoc(Preprocessor* preprocessor)
+{
+    return PeekRawToken(preprocessor).Position;
+}
+
+// Get the `TokenType` of the current (raw) token
+static TokenType PeekRawTokenType(Preprocessor* preprocessor)
+{
+    return PeekRawToken(preprocessor).Type;
+}
+
+//
+// Macros
+//
+
+// Create a macro
+static PreprocessorMacro* CreateMacro(Preprocessor* preprocessor)
+{
+    // TODO(tfoley): Allocate these more intelligently.
+    // For example, consider pooling them on the preprocessor.
+
+    PreprocessorMacro* macro = new PreprocessorMacro();
+    macro->flavor = PreprocessorMacroFlavor::ObjectLike;
+    macro->environment = &preprocessor->globalEnv;
+    return macro;
+}
+
+// Destroy a macro
+static void DestroyMacro(Preprocessor* /*preprocessor*/, PreprocessorMacro* macro)
+{
+    delete macro;
+}
+
+
+// Find the currently-defined macro of the given name, or return NULL
+static PreprocessorMacro* LookupMacro(PreprocessorEnvironment* environment, String const& name)
+{
+    for(PreprocessorEnvironment* e = environment; e; e = e->parent)
+    {
+        PreprocessorMacro* macro = NULL;
+        if (e->macros.TryGetValue(name, macro))
+            return macro;
+    }
+
+    return NULL;
+}
+
+static PreprocessorEnvironment* GetCurrentEnvironment(Preprocessor* preprocessor)
+{
+    PreprocessorInputStream* inputStream = preprocessor->inputStream;
+    return inputStream ? inputStream->environment : &preprocessor->globalEnv;
+}
+
+static PreprocessorMacro* LookupMacro(Preprocessor* preprocessor, String const& name)
+{
+    return LookupMacro(GetCurrentEnvironment(preprocessor), name);
+}
+
+// A macro is "busy" if it is currently being used for expansion.
+// A macro cannot be expanded again while busy, to avoid infinite recursion.
+static bool IsMacroBusy(PreprocessorMacro* /*macro*/)
+{
+    // TODO: need to implement this correctly
+    //
+    // The challenge here is that we are implementing expansion
+    // for argumenst to function-like macros in a "lazy" fashion.
+    //
+    // The letter of the spec is that we should macro expand
+    // each argument *before* substitution, and then go and
+    // macro-expand the substituted body. This means that we
+    // can invoke a macro as part of an argument to an
+    // invocation of the same macro:
+    //
+    //     FOO( 1, FOO(22), 333 );
+    //
+    // In our implementation, the "inner" invocation of `FOO`
+    // gets expanded at the point where it gets referenced
+    // in the body of the "outer" invocation of `FOO`.
+    // Doing things this way leads to greatly simplified
+    // code for handling expansion.
+    //
+    // A proper implementation of `IsMacroBusy` needs to
+    // take context into account, so that it bans recursive
+    // use of a macro when it occurs (indirectly) through
+    // the *body* of the expansion, but not when it occcurs
+    // only through an *argument*.
+    return false;
+}
+
+//
+// Reading Tokens With Expansion
+//
+
+static void InitializeMacroExpansion(
+    Preprocessor*       preprocessor,
+    MacroExpansion*     expansion,
+    PreprocessorMacro*  macro)
+{
+    InitializeInputStream(preprocessor, expansion);
+    expansion->environment = macro->environment;
+    expansion->macro = macro;
+    expansion->tokenReader = TokenReader(macro->tokens);
+}
+
+static void PushMacroExpansion(
+    Preprocessor*   preprocessor,
+    MacroExpansion* expansion)
+{
+    PushInputStream(preprocessor, expansion);
+}
+
+static void AddEndOfStreamToken(
+    Preprocessor*       preprocessor,
+    PreprocessorMacro*  macro)
+{
+    Token token = PeekRawToken(preprocessor);
+    token.Type = TokenType::EndOfFile;
+    macro->tokens.mTokens.Add(token);
+}
+
+// Check whether the current token on the given input stream should be
+// treated as a macro invocation, and if so set up state for expanding
+// that macro.
+static void MaybeBeginMacroExpansion(
+    Preprocessor*               preprocessor )
+{
+    // We iterate because the first token in the expansion of one
+    // macro may be another macro invocation.
+    for (;;)
+    {
+        // Look at the next token ahead of us
+        Token const& token = PeekRawToken(preprocessor);
+
+        // Not an identifier? Can't be a macro.
+        if (token.Type != TokenType::Identifier)
+            return;
+
+        // Look for a macro with the given name.
+        String name = token.Content;
+        PreprocessorMacro* macro = LookupMacro(preprocessor, name);
+
+        // Not a macro? Can't be an invocation.
+        if (!macro)
+            return;
+
+        // If the macro is busy (already being expanded),
+        // don't try to trigger recursive expansion
+        if (IsMacroBusy(macro))
+            return;
+
+        // A function-style macro invocation should only match
+        // if the token *after* the identifier is `(`. This
+        // requires more lookahead than we usually have/need
+        if (macro->flavor == PreprocessorMacroFlavor::FunctionLike)
+        {
+            if(PeekSecondRawTokenType(preprocessor) != TokenType::LParent)
+                return;
+
+            // Consume the token that triggered macro expansion
+            AdvanceRawToken(preprocessor);
+
+            // Consume the opening `(`
+            Token leftParen = AdvanceRawToken(preprocessor);
+
+            FunctionLikeMacroExpansion* expansion = new FunctionLikeMacroExpansion();
+            InitializeMacroExpansion(preprocessor, expansion, macro);
+            expansion->argumentEnvironment.parent = &preprocessor->globalEnv;
+            expansion->environment = &expansion->argumentEnvironment;
+
+            // Try to read any arguments present.
+            int paramCount = macro->params.Count();
+            int argIndex = 0;
+
+            switch (PeekRawTokenType(preprocessor))
+            {
+            case TokenType::EndOfFile:
+            case TokenType::RParent:
+                // No arguments.
+                break;
+
+            default:
+                // At least one argument
+                while(argIndex < paramCount)
+                {
+                    // Read an argument
+
+                    // Create the argument, represented as a special flavor of macro
+                    PreprocessorMacro* arg = CreateMacro(preprocessor);
+                    arg->flavor = PreprocessorMacroFlavor::FunctionArg;
+                    arg->environment = GetCurrentEnvironment(preprocessor);
+
+                    // Associate the new macro with its parameter name
+                    Token paramToken = macro->params[argIndex];
+                    String const& paramName = paramToken.Content;
+                    arg->nameToken = paramToken;
+                    expansion->argumentEnvironment.macros[paramName] = arg;
+                    argIndex++;
+
+                    // Read tokens for the argument
+
+                    // We track the nesting depth, since we don't break
+                    // arguments on a `,` nested in balanced parentheses
+                    //
+                    int nesting = 0;
+                    for (;;)
+                    {
+                        switch (PeekRawTokenType(preprocessor))
+                        {
+                        case TokenType::EndOfFile:
+                            // if we reach the end of the file,
+                            // then we have an error, and need to
+                            // bail out
+                            AddEndOfStreamToken(preprocessor, arg);
+                            goto doneWithAllArguments;
+
+                        case TokenType::RParent:
+                            // If we see a right paren when we aren't nested
+                            // then we are at the end of an argument
+                            if (nesting == 0)
+                            {
+                                AddEndOfStreamToken(preprocessor, arg);
+                                goto doneWithAllArguments;
+                            }
+                            // Otherwise we decrease our nesting depth, add
+                            // the token, and keep going
+                            nesting--;
+                            break;
+
+                        case TokenType::Comma:
+                            // If we see a comma when we aren't nested
+                            // then we are at the end of an argument
+                            if (nesting == 0)
+                            {
+                                AddEndOfStreamToken(preprocessor, arg);
+                                AdvanceRawToken(preprocessor);
+                                goto doneWithArgument;
+                            }
+                            // Otherwise we add it as a normal token
+                            break;
+
+                        case TokenType::LParent:
+                            // If we see a left paren then we need to
+                            // increase our tracking of nesting
+                            nesting++;
+                            break;
+
+                        default:
+                            break;
+                        }
+
+                        // Add the token and continue parsing.
+                        arg->tokens.mTokens.Add(AdvanceRawToken(preprocessor));
+                    }
+                doneWithArgument: {}
+                    // We've parsed an argument and should move onto
+                    // the next one.
+                }
+                break;
+            }
+        doneWithAllArguments:
+            // TODO: handle possible varargs
+
+            // Expect closing right paren
+            if (PeekRawTokenType(preprocessor) == TokenType::RParent)
+            {
+                AdvanceRawToken(preprocessor);
+            }
+            else
+            {
+                GetSink(preprocessor)->diagnose(PeekLoc(preprocessor), Diagnostics::expectedTokenInMacroArguments, TokenType::RParent, PeekRawTokenType(preprocessor));
+            }
+
+            int argCount = argIndex;
+            if (argCount != paramCount)
+            {
+                // TODO: diagnose
+                throw 99;
+            }
+
+            // We are ready to expand.
+            PushMacroExpansion(preprocessor, expansion);
+        }
+        else
+        {
+            // Consume the token that triggered macro expansion
+            AdvanceRawToken(preprocessor);
+
+            // Object-like macros are the easy case.
+            ObjectLikeMacroExpansion* expansion = new ObjectLikeMacroExpansion();
+            InitializeMacroExpansion(preprocessor, expansion, macro);
+            PushMacroExpansion(preprocessor, expansion);
+        }
+    }
+}
+
+// Read one token with macro-expansion enabled.
+static Token AdvanceToken(Preprocessor* preprocessor)
+{
+top:
+    // Check whether we need to macro expand at the cursor.
+    MaybeBeginMacroExpansion(preprocessor);
+
+    // Read a raw token (now that expansion has been triggered)
+    Token token = AdvanceRawToken(preprocessor);
+
+    // Check if we need to perform token pasting
+    if (PeekRawTokenType(preprocessor) != TokenType::PoundPound)
+    {
+        // If we aren't token pasting, then we are done
+        return token;
+    }
+    else
+    {
+        // We are pasting tokens, which could get messy
+
+        StringBuilder sb;
+        sb << token.Content;
+
+        while (PeekRawTokenType(preprocessor) == TokenType::PoundPound)
+        {
+            // Consume the `##`
+            AdvanceRawToken(preprocessor);
+
+            // Possibly macro-expand the next token
+            MaybeBeginMacroExpansion(preprocessor);
+
+            // Read the next raw token (now that expansion has been triggered)
+            Token nextToken = AdvanceRawToken(preprocessor);
+
+            sb << nextToken.Content;
+        }
+
+        // Now re-lex the input
+        PreprocessorInputStream* inputStream = CreateInputStreamForSource(preprocessor, sb.ProduceString(), "token paste");
+        if (inputStream->tokenReader.GetCount() != 1)
+        {
+            // We expect a token paste to produce a single token
+            // TODO(tfoley): emit a diagnostic here
+        }
+
+        PushInputStream(preprocessor, inputStream);
+        goto top;
+    }
+}
+
+// Read one token with macro-expansion enabled.
+//
+// Note that because triggering macro expansion may
+// involve changing the input-stream state, this
+// operation *can* have side effects.
+static Token PeekToken(Preprocessor* preprocessor)
+{
+    // Check whether we need to macro expand at the cursor.
+    MaybeBeginMacroExpansion(preprocessor);
+
+    // Peek a raw token (now that expansion has been triggered)
+    return PeekRawToken(preprocessor);
+
+    // TODO: need a plan for how to handle token pasting
+    // here without it being onerous. Would be nice if we
+    // didn't have to re-do pasting on a "peek"...
+}
+
+// Peek the type of the next token, including macro expansion.
+static TokenType PeekTokenType(Preprocessor* preprocessor)
+{
+    return PeekToken(preprocessor).Type;
+}
+
+//
+// Preprocessor Directives
+//
+
+// When reading a preprocessor directive, we use a context
+// to wrap the direct preprocessor routines defines so far.
+//
+// One of the most important things the directive context
+// does is give us a convenient way to read tokens with
+// a guarantee that we won't read past the end of a line.
+struct PreprocessorDirectiveContext
+{
+    // The preprocessor that is parsing the directive.
+    Preprocessor*   preprocessor;
+
+    // The directive token (e.g., the `if` in `#if`).
+    // Useful for reference in diagnostic messages.
+    Token           directiveToken;
+
+    // Has any kind of parse error been encountered in
+    // the directive so far?
+    bool            parseError;
+
+    // Have we done the necessary checks at the end
+    // of the directive already?
+    bool            haveDoneEndOfDirectiveChecks;
+};
+
+// Get the token for  the preprocessor directive being parsed.
+inline Token const& GetDirective(PreprocessorDirectiveContext* context)
+{
+    return context->directiveToken;
+}
+
+// Get the name of the directive being parsed.
+inline String const& GetDirectiveName(PreprocessorDirectiveContext* context)
+{
+    return context->directiveToken.Content;
+}
+
+// Get the location of the directive being parsed.
+inline CodePosition const& GetDirectiveLoc(PreprocessorDirectiveContext* context)
+{
+    return context->directiveToken.Position;
+}
+
+// Wrapper to get the diagnostic sink in the context of a directive.
+static inline DiagnosticSink* GetSink(PreprocessorDirectiveContext* context)
+{
+    return GetSink(context->preprocessor);
+}
+
+// Wrapper to get a "current" location when parsing a directive
+static CodePosition PeekLoc(PreprocessorDirectiveContext* context)
+{
+    return PeekLoc(context->preprocessor);
+}
+
+// Wrapper to look up a macro in the context of a directive.
+static PreprocessorMacro* LookupMacro(PreprocessorDirectiveContext* context, String const& name)
+{
+    return LookupMacro(context->preprocessor, name);
+}
+
+// Determine if we have read everthing on the directive's line.
+static bool IsEndOfLine(PreprocessorDirectiveContext* context)
+{
+    return PeekRawToken(context->preprocessor).Type == TokenType::EndOfDirective;
+}
+
+// Peek one raw token in a directive, without going past the end of the line.
+static Token PeekRawToken(PreprocessorDirectiveContext* context)
+{
+    return PeekRawToken(context->preprocessor);
+}
+
+// Read one raw token in a directive, without going past the end of the line.
+static Token AdvanceRawToken(PreprocessorDirectiveContext* context)
+{
+    if (IsEndOfLine(context))
+        return PeekRawToken(context);
+    return AdvanceRawToken(context->preprocessor);
+}
+
+// Peek next raw token type, without going past the end of the line.
+static TokenType PeekRawTokenType(PreprocessorDirectiveContext* context)
+{
+    return PeekRawTokenType(context->preprocessor);
+}
+
+// Read one token, with macro-expansion, without going past the end of the line.
+static Token AdvanceToken(PreprocessorDirectiveContext* context)
+{
+    if (IsEndOfLine(context))
+        return PeekRawToken(context);
+    return AdvanceToken(context->preprocessor);
+}
+
+// Peek one token, with macro-expansion, without going past the end of the line.
+static Token PeekToken(PreprocessorDirectiveContext* context)
+{
+    if (IsEndOfLine(context))
+        context->preprocessor->endOfFileToken;
+    return PeekToken(context->preprocessor);
+}
+
+// Peek next token type, with macro-expansion, without going past the end of the line.
+static TokenType PeekTokenType(PreprocessorDirectiveContext* context)
+{
+    if (IsEndOfLine(context))
+        return TokenType::EndOfDirective;
+    return PeekTokenType(context->preprocessor);
+}
+
+// Skip to the end of the line (useful for recovering from errors in a directive)
+static void SkipToEndOfLine(PreprocessorDirectiveContext* context)
+{
+    while(!IsEndOfLine(context))
+    {
+        AdvanceRawToken(context);
+    }
+}
+
+static bool ExpectRaw(PreprocessorDirectiveContext* context, TokenType tokenType, DiagnosticInfo const& diagnostic, Token* outToken = NULL)
+{
+    if (PeekRawTokenType(context) != tokenType)
+    {
+        // Only report the first parse error within a directive
+        if (!context->parseError)
+        {
+            GetSink(context)->diagnose(PeekLoc(context), diagnostic, tokenType, GetDirectiveName(context));
+        }
+        context->parseError = true;
+        return false;
+    }
+    Token const& token = AdvanceRawToken(context);
+    if (outToken)
+        *outToken = token;
+    return true;
+}
+
+static bool Expect(PreprocessorDirectiveContext* context, TokenType tokenType, DiagnosticInfo const& diagnostic, Token* outToken = NULL)
+{
+    if (PeekTokenType(context) != tokenType)
+    {
+        // Only report the first parse error within a directive
+        if (!context->parseError)
+        {
+            GetSink(context)->diagnose(PeekLoc(context), diagnostic, tokenType, GetDirectiveName(context));
+            context->parseError = true;
+        }
+        return false;
+    }
+    Token const& token = AdvanceToken(context);
+    if (outToken)
+        *outToken = token;
+    return true;
+}
+
+
+
+//
+// Preprocessor Conditionals
+//
+
+// Determine whether the current preprocessor state means we
+// should be skipping tokens.
+static bool IsSkipping(Preprocessor* preprocessor)
+{
+    PreprocessorInputStream* inputStream = preprocessor->inputStream;
+    if (!inputStream) return false;
+
+    // If we are not inside a preprocessor conditional, then don't skip
+    PreprocessorConditional* conditional = inputStream->conditional;
+    if (!conditional) return false;
+
+    // skip tokens unless the conditional is inside its `true` case
+    return conditional->state != PreprocessorConditionalState::During;
+}
+
+// Wrapper for use inside directives
+static inline bool IsSkipping(PreprocessorDirectiveContext* context)
+{
+    return IsSkipping(context->preprocessor);
+}
+
+// Create a preprocessor conditional
+static PreprocessorConditional* CreateConditional(Preprocessor* /*preprocessor*/)
+{
+    // TODO(tfoley): allocate these more intelligently (for example,
+    // pool them on the `Preprocessor`.
+    return new PreprocessorConditional();
+}
+
+// Destroy a preprocessor conditional.
+static void DestroyConditional(PreprocessorConditional* conditional)
+{
+    delete conditional;
+}
+
+// Start a preprocessor conditional, with an initial enable/disable state.
+static void BeginConditional(PreprocessorDirectiveContext* context, bool enable)
+{
+    Preprocessor* preprocessor = context->preprocessor;
+    PreprocessorInputStream* inputStream = preprocessor->inputStream;
+    assert(inputStream);
+
+    PreprocessorConditional* conditional = CreateConditional(preprocessor);
+
+    conditional->ifToken = context->directiveToken;
+
+    // Set state of this condition appropriately.
+    //
+    // Default to the "haven't yet seen a `true` branch" state.
+    PreprocessorConditionalState state = PreprocessorConditionalState::Before;
+    //
+    // If we are nested inside a `false` branch of another condition, then
+    // we never want to enable, so we act as if we already *saw* the `true` branch.
+    //
+    if (IsSkipping(preprocessor)) state = PreprocessorConditionalState::After;
+    //
+    // Similarly, if we ran into any parse errors when dealing with the
+    // opening directive, then things are probably screwy and we should just
+    // skip all the branches.
+    if (IsSkipping(preprocessor)) state = PreprocessorConditionalState::After;
+    //
+    // Otherwise, if our condition was true, then set us to be inside the `true` branch
+    else if (enable) state = PreprocessorConditionalState::During;
+
+    conditional->state = state;
+
+    // Push conditional onto the stack
+    conditional->parent = inputStream->conditional;
+    inputStream->conditional = conditional;
+}
+
+//
+// Preprocessor Conditional Expressions
+//
+
+// Conditional expressions are always of type `int`
+typedef int PreprocessorExpressionValue;
+
+// Forward-declaretion
+static PreprocessorExpressionValue ParseAndEvaluateExpression(PreprocessorDirectiveContext* context);
+
+// Parse a unary (prefix) expression inside of a preprocessor directive.
+static PreprocessorExpressionValue ParseAndEvaluateUnaryExpression(PreprocessorDirectiveContext* context)
+{
+    switch (PeekTokenType(context))
+    {
+    // handle prefix unary ops
+    case TokenType::OpSub:
+        AdvanceToken(context);
+        return -ParseAndEvaluateUnaryExpression(context);
+    case TokenType::OpNot:
+        AdvanceToken(context);
+        return !ParseAndEvaluateUnaryExpression(context);
+    case TokenType::OpBitNot:
+        AdvanceToken(context);
+        return ~ParseAndEvaluateUnaryExpression(context);
+
+    // handle parenthized sub-expression
+    case TokenType::LParent:
+        {
+            Token leftParen = AdvanceToken(context);
+            PreprocessorExpressionValue value = ParseAndEvaluateExpression(context);
+            if (!Expect(context, TokenType::RParent, Diagnostics::expectedTokenInPreprocessorExpression))
+            {
+                GetSink(context)->diagnose(leftParen.Position, Diagnostics::seeOpeningToken, leftParen);
+            }
+            return value;
+        }
+
+    case TokenType::IntLiterial:
+        return StringToInt(AdvanceToken(context).Content);
+
+    case TokenType::Identifier:
+        {
+            Token token = AdvanceToken(context);
+            if (token.Content == "defined")
+            {
+                // handle `defined(someName)`
+
+                // Possibly parse a `(`
+                Token leftParen;
+                if (PeekRawTokenType(context) == TokenType::LParent)
+                {
+                    leftParen = AdvanceRawToken(context);
+                }
+
+                // Expect an identifier
+                Token nameToken;
+                if (!ExpectRaw(context, TokenType::Identifier, Diagnostics::expectedTokenInDefinedExpression, &nameToken))
+                {
+                    return 0;
+                }
+                String name = nameToken.Content;
+
+                // If we saw an opening `(`, then expect one to close
+                if (leftParen.Type != TokenType::Unknown)
+                {
+                    if(!ExpectRaw(context, TokenType::RParent, Diagnostics::expectedTokenInDefinedExpression))
+                    {
+                        GetSink(context)->diagnose(leftParen.Position, Diagnostics::seeOpeningToken, leftParen);
+                        return 0;
+                    }
+                }
+
+                return LookupMacro(context, name) != NULL;
+            }
+
+            // An identifier here means it was not defined as a macro (or
+            // it is defined, but as a function-like macro. These should
+            // just evaluate to zero (possibly with a warning)
+            return 0;
+        }
+
+    default:
+        GetSink(context)->diagnose(PeekLoc(context), Diagnostics::syntaxErrorInPreprocessorExpression);
+        return 0;
+    }
+}
+
+// Determine the precedence level of an infix operator
+// for use in parsing preprocessor conditionals.
+static int GetInfixOpPrecedence(Token const& opToken)
+{
+    // If token is on another line, it is not part of the
+    // expression
+    if (opToken.flags & TokenFlag::AtStartOfLine)
+        return -1;
+
+    // otherwise we look at the token type to figure
+    // out what precednece it should be parse with
+    switch (opToken.Type)
+    {
+    default:
+        // tokens that aren't infix operators should
+        // cause us to stop parsing an expression
+        return -1;
+
+    case TokenType::OpMul:     return 10;
+    case TokenType::OpDiv:     return 10;
+    case TokenType::OpMod:     return 10;
+
+    case TokenType::OpAdd:     return 9;
+    case TokenType::OpSub:     return 9;
+
+    case TokenType::OpLsh:     return 8;
+    case TokenType::OpRsh:     return 8;
+
+    case TokenType::OpLess:    return 7;
+    case TokenType::OpGreater: return 7;
+    case TokenType::OpLeq:     return 7;
+    case TokenType::OpGeq:     return 7;
+
+    case TokenType::OpEql:     return 6;
+    case TokenType::OpNeq:     return 6;
+
+    case TokenType::OpBitAnd:  return 5;
+    case TokenType::OpBitOr:   return 4;
+    case TokenType::OpBitXor:  return 3;
+    case TokenType::OpAnd:     return 2;
+    case TokenType::OpOr:      return 1;
+    }
+};
+
+// Evaluate one infix operation in a preprocessor
+// conditional expression
+static PreprocessorExpressionValue EvaluateInfixOp(
+    PreprocessorDirectiveContext*   context,
+    Token const&                    opToken,
+    PreprocessorExpressionValue     left,
+    PreprocessorExpressionValue     right)
+{
+    switch (opToken.Type)
+    {
+    default:
+//        SLANG_INTERNAL_ERROR(getSink(preprocessor), opToken);
+        return 0;
+        break;
+
+    case TokenType::OpMul:     return left * right;
+    case TokenType::OpDiv:
+    {
+        if (right == 0)
+        {
+            if (!context->parseError)
+            {
+                GetSink(context)->diagnose(opToken.Position, Diagnostics::divideByZeroInPreprocessorExpression);
+            }
+            return 0;
+        }
+        return left / right;
+    }
+    case TokenType::OpMod:
+    {
+        if (right == 0)
+        {
+            if (!context->parseError)
+            {
+                GetSink(context)->diagnose(opToken.Position, Diagnostics::divideByZeroInPreprocessorExpression);
+            }
+            return 0;
+        }
+        return left % right;
+    }
+    case TokenType::OpAdd:      return left +  right;
+    case TokenType::OpSub:      return left -  right;
+    case TokenType::OpLsh:      return left << right;
+    case TokenType::OpRsh:      return left >> right;
+    case TokenType::OpLess:     return left <  right ? 1 : 0;
+    case TokenType::OpGreater:  return left >  right ? 1 : 0;
+    case TokenType::OpLeq:      return left <= right ? 1 : 0;
+    case TokenType::OpGeq:      return left <= right ? 1 : 0;
+    case TokenType::OpEql:      return left == right ? 1 : 0;
+    case TokenType::OpNeq:      return left != right ? 1 : 0;
+    case TokenType::OpBitAnd:   return left & right;
+    case TokenType::OpBitOr:    return left | right;
+    case TokenType::OpBitXor:   return left ^ right;
+    case TokenType::OpAnd:      return left && right;
+    case TokenType::OpOr:       return left || right;
+    }
+}
+
+// Parse the rest of an infix preprocessor expression with
+// precedence greater than or equal to the given `precedence` argument.
+// The value of the left-hand-side expression is provided as
+// an argument.
+// This is used to form a simple recursive-descent expression parser.
+static PreprocessorExpressionValue ParseAndEvaluateInfixExpressionWithPrecedence(
+    PreprocessorDirectiveContext* context,
+    PreprocessorExpressionValue left,
+    int precedence)
+{
+    for (;;)
+    {
+        // Look at the next token, and see if it is an operator of
+        // high enough precedence to be included in our expression
+        Token opToken = PeekToken(context);
+        int opPrecedence = GetInfixOpPrecedence(opToken);
+
+        // If it isn't an operator of high enough precendece, we are done.
+        if(opPrecedence < precedence)
+            break;
+
+        // Otherwise we need to consume the operator token.
+        AdvanceToken(context);
+
+        // Next we parse a right-hand-side expression by starting with
+        // a unary expression and absorbing and many infix operators
+        // as possible with strictly higher precedence than the operator
+        // we found above.
+        PreprocessorExpressionValue right = ParseAndEvaluateUnaryExpression(context);
+        for (;;)
+        {
+            // Look for an operator token
+            Token rightOpToken = PeekToken(context);
+            int rightOpPrecedence = GetInfixOpPrecedence(rightOpToken);
+
+            // If no operator was found, or the operator wasn't high
+            // enough precedence to fold into the right-hand-side,
+            // exit this loop.
+            if (rightOpPrecedence <= opPrecedence)
+                break;
+
+            // Now invoke the parser recursively, passing in our
+            // existing right-hand side to form an even larger one.
+            right = ParseAndEvaluateInfixExpressionWithPrecedence(
+                context,
+                right,
+                rightOpPrecedence);
+        }
+
+        // Now combine the left- and right-hand sides using
+        // the operator we found above.
+        left = EvaluateInfixOp(context, opToken, left, right);
+    }
+    return left;
+}
+
+// Parse a complete (infix) preprocessor expression, and return its value
+static PreprocessorExpressionValue ParseAndEvaluateExpression(PreprocessorDirectiveContext* context)
+{
+    // First read in the left-hand side (or the whole expression in the unary case)
+    PreprocessorExpressionValue value = ParseAndEvaluateUnaryExpression(context);
+
+    // Try to read in trailing infix operators with correct precedence
+    return ParseAndEvaluateInfixExpressionWithPrecedence(context, value, 0);
+}
+
+// Handle a `#if` directive
+static void HandleIfDirective(PreprocessorDirectiveContext* context)
+{
+    // Parse a preprocessor expression.
+    PreprocessorExpressionValue value = ParseAndEvaluateExpression(context);
+
+    // Begin a preprocessor block, enabled based on the expression.
+    BeginConditional(context, value != 0);
+}
+
+// Handle a `#ifdef` directive
+static void HandleIfDefDirective(PreprocessorDirectiveContext* context)
+{
+    // Expect a raw identifier, so we can check if it is defined
+    Token nameToken;
+    if(!ExpectRaw(context, TokenType::Identifier, Diagnostics::expectedTokenInPreprocessorDirective, &nameToken))
+        return;
+    String name = nameToken.Content;
+
+    // Check if the name is defined.
+    BeginConditional(context, LookupMacro(context, name) != NULL);
+}
+
+// Handle a `#ifndef` directive
+static void HandleIfNDefDirective(PreprocessorDirectiveContext* context)
+{
+    // Expect a raw identifier, so we can check if it is defined
+    Token nameToken;
+    if(!ExpectRaw(context, TokenType::Identifier, Diagnostics::expectedTokenInPreprocessorDirective, &nameToken))
+        return;
+    String name = nameToken.Content;
+
+    // Check if the name is defined.
+    BeginConditional(context, LookupMacro(context, name) == NULL);
+}
+
+// Handle a `#else` directive
+static void HandleElseDirective(PreprocessorDirectiveContext* context)
+{
+    PreprocessorInputStream* inputStream = context->preprocessor->inputStream;
+    assert(inputStream);
+
+    // if we aren't inside a conditional, then error
+    PreprocessorConditional* conditional = inputStream->conditional;
+    if (!conditional)
+    {
+        GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::directiveWithoutIf, GetDirectiveName(context));
+        return;
+    }
+
+    // if we've already seen a `#else`, then it is an error
+    if (conditional->elseToken.Type != TokenType::Unknown)
+    {
+        GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::directiveAfterElse, GetDirectiveName(context));
+        GetSink(context)->diagnose(conditional->elseToken.Position, Diagnostics::seeDirective);
+        return;
+    }
+    conditional->elseToken = context->directiveToken;
+
+    switch (conditional->state)
+    {
+    case PreprocessorConditionalState::Before:
+        conditional->state = PreprocessorConditionalState::During;
+        break;
+
+    case PreprocessorConditionalState::During:
+        conditional->state = PreprocessorConditionalState::After;
+        break;
+
+    default:
+        break;
+    }
+}
+
+// Handle a `#elif` directive
+static void HandleElifDirective(PreprocessorDirectiveContext* context)
+{
+    // HACK(tfoley): handle an empty `elif` like an `else` directive
+    //
+    // This is the behavior expected by at least one input program.
+    // We will eventually want to be pedantic about this.
+    // even if t
+    if (PeekRawTokenType(context) == TokenType::EndOfDirective)
+    {
+        GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::directiveExpectsExpression, GetDirectiveName(context));
+        HandleElseDirective(context);
+        return;
+    }
+
+    PreprocessorExpressionValue value = ParseAndEvaluateExpression(context);
+
+    PreprocessorInputStream* inputStream = context->preprocessor->inputStream;
+    assert(inputStream);
+
+    // if we aren't inside a conditional, then error
+    PreprocessorConditional* conditional = inputStream->conditional;
+    if (!conditional)
+    {
+        GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::directiveWithoutIf, GetDirectiveName(context));
+        return;
+    }
+
+    // if we've already seen a `#else`, then it is an error
+    if (conditional->elseToken.Type != TokenType::Unknown)
+    {
+        GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::directiveAfterElse, GetDirectiveName(context));
+        GetSink(context)->diagnose(conditional->elseToken.Position, Diagnostics::seeDirective);
+        return;
+    }
+
+    switch (conditional->state)
+    {
+    case PreprocessorConditionalState::Before:
+        if(value)
+            conditional->state = PreprocessorConditionalState::During;
+        break;
+
+    case PreprocessorConditionalState::During:
+        conditional->state = PreprocessorConditionalState::After;
+        break;
+
+    default:
+        break;
+    }
+}
+
+// Handle a `#endif` directive
+static void HandleEndIfDirective(PreprocessorDirectiveContext* context)
+{
+    PreprocessorInputStream* inputStream = context->preprocessor->inputStream;
+    assert(inputStream);
+
+    // if we aren't inside a conditional, then error
+    PreprocessorConditional* conditional = inputStream->conditional;
+    if (!conditional)
+    {
+        GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::directiveWithoutIf, GetDirectiveName(context));
+        return;
+    }
+
+    inputStream->conditional = conditional->parent;
+    DestroyConditional(conditional);
+}
+
+// Helper routine to check that we find the end of a directive where
+// we expect it.
+//
+// Most directives do not need to call this directly, since we have
+// a catch-all case in the main `HandleDirective()` funciton.
+// The `#include` case will call it directly to avoid complications
+// when it switches the input stream.
+static void expectEndOfDirective(PreprocessorDirectiveContext* context)
+{
+    if(context->haveDoneEndOfDirectiveChecks)
+        return;
+
+    context->haveDoneEndOfDirectiveChecks = true;
+
+    if (!IsEndOfLine(context))
+    {
+        // If we already saw a previous parse error, then don't
+        // emit another one for the same directive.
+        if (!context->parseError)
+        {
+            GetSink(context)->diagnose(PeekLoc(context), Diagnostics::unexpectedTokensAfterDirective, GetDirectiveName(context));
+        }
+        SkipToEndOfLine(context);
+    }
+
+    // Clear out the end-of-directive token
+    AdvanceRawToken(context->preprocessor);
+}
+
+
+// Handle a `#include` directive
+static void HandleIncludeDirective(PreprocessorDirectiveContext* context)
+{
+    Token pathToken;
+    if(!Expect(context, TokenType::StringLiterial, Diagnostics::expectedTokenInPreprocessorDirective, &pathToken))
+        return;
+
+    String path = getFileNameTokenValue(pathToken);
+
+    // TODO(tfoley): make this robust in presence of `#line`
+    String pathIncludedFrom = GetDirectiveLoc(context).FileName;
+    String foundPath;
+    String foundSource;
+
+
+    IncludeHandler* includeHandler = context->preprocessor->includeHandler;
+    if (!includeHandler)
+    {
+        GetSink(context)->diagnose(pathToken.Position, Diagnostics::includeFailed, path);
+        GetSink(context)->diagnose(pathToken.Position, Diagnostics::noIncludeHandlerSpecified);
+        return;
+    }
+    if (!includeHandler->TryToFindIncludeFile(path, pathIncludedFrom, &foundPath, &foundSource))
+    {
+        GetSink(context)->diagnose(pathToken.Position, Diagnostics::includeFailed, path);
+        return;
+    }
+
+    // Do all checking related to the end of this directive before we push a new stream,
+    // just to avoid complications where that check would need to deal with
+    // a switch of input stream
+    expectEndOfDirective(context);
+
+    // Push the new file onto our stack of input streams
+    // TODO(tfoley): check if we have made our include stack too deep
+    PreprocessorInputStream* inputStream = CreateInputStreamForSource(context->preprocessor, foundSource, foundPath);
+    inputStream->parent = context->preprocessor->inputStream;
+    context->preprocessor->inputStream = inputStream;
+}
+
+// Handle a `#define` directive
+static void HandleDefineDirective(PreprocessorDirectiveContext* context)
+{
+    Token nameToken;
+    if (!Expect(context, TokenType::Identifier, Diagnostics::expectedTokenInPreprocessorDirective, &nameToken))
+        return;
+    String name = nameToken.Content;
+
+    PreprocessorMacro* macro = CreateMacro(context->preprocessor);
+    macro->nameToken = nameToken;
+
+    PreprocessorMacro* oldMacro = LookupMacro(&context->preprocessor->globalEnv, name);
+    if (oldMacro)
+    {
+        GetSink(context)->diagnose(nameToken.Position, Diagnostics::macroRedefinition, name);
+        GetSink(context)->diagnose(oldMacro->nameToken.Position, Diagnostics::seePreviousDefinitionOf, name);
+
+        DestroyMacro(context->preprocessor, oldMacro);
+    }
+    context->preprocessor->globalEnv.macros[name] = macro;
+
+    // If macro name is immediately followed (with no space) by `(`,
+    // then we have a function-like macro
+    if (PeekRawTokenType(context) == TokenType::LParent)
+    {
+        if (!(PeekRawToken(context).flags & TokenFlag::AfterWhitespace))
+        {
+            // This is a function-like macro, so we need to remember that
+            // and start capturing parameters
+            macro->flavor = PreprocessorMacroFlavor::FunctionLike;
+
+            AdvanceRawToken(context);
+
+            // If there are any parameters, parse them
+            if (PeekRawTokenType(context) != TokenType::RParent)
+            {
+                for (;;)
+                {
+                    // TODO: handle elipsis (`...`) for varags
+
+                    // A macro parameter name should be a raw identifier
+                    Token paramToken;
+                    if (!ExpectRaw(context, TokenType::Identifier, Diagnostics::expectedTokenInMacroParameters, &paramToken))
+                        break;
+
+                    // TODO(tfoley): some validation on parameter name.
+                    // Certain names (e.g., `defined` and `__VA_ARGS__`
+                    // are not allowed to be used as macros or parameters).
+
+                    // Add the parameter to the macro being deifned
+                    macro->params.Add(paramToken);
+
+                    // If we see `)` then we are done with arguments
+                    if (PeekRawTokenType(context) == TokenType::RParent)
+                        break;
+
+                    ExpectRaw(context, TokenType::Comma, Diagnostics::expectedTokenInMacroParameters);
+                }
+            }
+
+            ExpectRaw(context, TokenType::RParent, Diagnostics::expectedTokenInMacroParameters);
+        }
+    }
+
+    // consume tokens until end-of-line
+    for(;;)
+    {
+        Token token = AdvanceRawToken(context);
+        if( token.Type == TokenType::EndOfDirective )
+        {
+            // Last token on line will be turned into a conceptual end-of-file
+            // token for the sub-stream that the macro expands into.
+            token.Type = TokenType::EndOfFile;
+            macro->tokens.mTokens.Add(token);
+            break;
+        }
+
+        // In the ordinary case, we just add the token to the definition
+        macro->tokens.mTokens.Add(token);
+    }
+}
+
+// Handle a `#undef` directive
+static void HandleUndefDirective(PreprocessorDirectiveContext* context)
+{
+    Token nameToken;
+    if (!Expect(context, TokenType::Identifier, Diagnostics::expectedTokenInPreprocessorDirective, &nameToken))
+        return;
+    String name = nameToken.Content;
+
+    PreprocessorEnvironment* env = &context->preprocessor->globalEnv;
+    PreprocessorMacro* macro = LookupMacro(env, name);
+    if (macro != NULL)
+    {
+        // name was defined, so remove it
+        env->macros.Remove(name);
+
+        DestroyMacro(context->preprocessor, macro);
+    }
+    else
+    {
+        // name wasn't defined
+        GetSink(context)->diagnose(nameToken.Position, Diagnostics::macroNotDefined, name);
+    }
+}
+
+// Handle a `#warning` directive
+static void HandleWarningDirective(PreprocessorDirectiveContext* context)
+{
+    // TODO: read rest of line without actual tokenization
+    GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::userDefinedWarning, "user-defined warning");
+    SkipToEndOfLine(context);
+}
+
+// Handle a `#error` directive
+static void HandleErrorDirective(PreprocessorDirectiveContext* context)
+{
+    // TODO: read rest of line without actual tokenization
+    GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::userDefinedError, "user-defined warning");
+    SkipToEndOfLine(context);
+}
+
+// Handle a `#line` directive
+static void HandleLineDirective(PreprocessorDirectiveContext* context)
+{
+    int line = 0;
+    if (PeekTokenType(context) == TokenType::IntLiterial)
+    {
+        line = StringToInt(AdvanceToken(context).Content);
+    }
+    else if (PeekTokenType(context) == TokenType::Identifier
+        && PeekToken(context).Content == "default")
+    {
+        AdvanceToken(context);
+
+        // Stop overiding soure locations.
+        context->preprocessor->inputStream->isOverridingSourceLoc = false;
+        return;
+    }
+    else
+    {
+        GetSink(context)->diagnose(PeekLoc(context), Diagnostics::expected2TokensInPreprocessorDirective,
+            TokenType::IntLiterial,
+            "default",
+            GetDirectiveName(context));
+        context->parseError = true;
+        return;
+    }
+
+    CodePosition directiveLoc = GetDirectiveLoc(context);
+
+    String file;
+    if (PeekTokenType(context) == TokenType::EndOfDirective)
+    {
+        file = directiveLoc.FileName;
+    }
+    else if (PeekTokenType(context) == TokenType::StringLiterial)
+    {
+        file = AdvanceToken(context).Content;
+    }
+    else if (PeekTokenType(context) == TokenType::IntLiterial)
+    {
+        // Note(tfoley): GLSL allows the "source string" to be indicated by an integer
+        // TODO(tfoley): Figure out a better way to handle this, if it matters
+        file = AdvanceToken(context).Content;
+    }
+    else
+    {
+        Expect(context, TokenType::StringLiterial, Diagnostics::expectedTokenInPreprocessorDirective);
+        return;
+    }
+
+    PreprocessorInputStream* inputStream = context->preprocessor->inputStream;
+
+    inputStream->isOverridingSourceLoc = true;
+    inputStream->overrideFileName = file;
+    inputStream->overrideLineOffset = line - (directiveLoc.Line + 1);
+}
+
+// Handle a `#pragma` directive
+static void HandlePragmaDirective(PreprocessorDirectiveContext* context)
+{
+    // TODO(tfoley): figure out which pragmas to parse,
+    // and which to pass along
+    SkipToEndOfLine(context);
+}
+
+// Handle a `#version` directive
+static void handleGLSLVersionDirective(PreprocessorDirectiveContext* context)
+{
+    Token versionNumberToken;
+    if(!ExpectRaw(
+        context,
+        TokenType::IntLiterial,
+        Diagnostics::expectedTokenInPreprocessorDirective,
+        &versionNumberToken))
+    {
+        return;
+    }
+
+    Token glslProfileToken;
+    if(PeekTokenType(context) == TokenType::Identifier)
+    {
+        glslProfileToken = AdvanceToken(context);
+    }
+
+    // Need to construct a representation taht we can hook into our compilation result
+
+    auto modifier = new GLSLVersionDirective();
+    modifier->versionNumberToken = versionNumberToken;
+    modifier->glslProfileToken = glslProfileToken;
+
+    // Attach the modifier to the program we are parsing!
+
+    addModifier(
+        context->preprocessor->syntax,
+        modifier);
+}
+
+// Handle a `#extension` directive, e.g.,
+//
+//     #extension some_extension_name : enable
+//
+static void handleGLSLExtensionDirective(PreprocessorDirectiveContext* context)
+{
+    Token extensionNameToken;
+    if(!ExpectRaw(
+        context,
+        TokenType::Identifier,
+        Diagnostics::expectedTokenInPreprocessorDirective,
+        &extensionNameToken))
+    {
+        return;
+    }
+
+    if( !ExpectRaw(context, TokenType::Colon, Diagnostics::expectedTokenInPreprocessorDirective) )
+    {
+        return;
+    }
+
+    Token dispositionToken;
+    if(!ExpectRaw(
+        context,
+        TokenType::Identifier,
+        Diagnostics::expectedTokenInPreprocessorDirective,
+        &dispositionToken))
+    {
+        return;
+    }
+
+    // Need to construct a representation taht we can hook into our compilation result
+
+    auto modifier = new GLSLExtensionDirective();
+    modifier->extensionNameToken = extensionNameToken;
+    modifier->dispositionToken = dispositionToken;
+
+    // Attach the modifier to the program we are parsing!
+
+    addModifier(
+        context->preprocessor->syntax,
+        modifier);
+}
+
+// Handle an invalid directive
+static void HandleInvalidDirective(PreprocessorDirectiveContext* context)
+{
+    GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::unknownPreprocessorDirective, GetDirectiveName(context));
+    SkipToEndOfLine(context);
+}
+
+// Callback interface used by preprocessor directives
+typedef void (*PreprocessorDirectiveCallback)(PreprocessorDirectiveContext* context);
+
+enum PreprocessorDirectiveFlag : unsigned int
+{
+    // Should this directive be handled even when skipping disbaled code?
+    ProcessWhenSkipping = 1 << 0,
+};
+
+// Information about a specific directive
+struct PreprocessorDirective
+{
+    // Name of the directive
+    char const*                     name;
+
+    // Callback to handle the directive
+    PreprocessorDirectiveCallback   callback;
+
+    unsigned int                    flags;
+};
+
+// A simple array of all the directives we know how to handle.
+// TODO(tfoley): considering making this into a real hash map,
+// and then make it easy-ish for users of the codebase to add
+// their own directives as desired.
+static const PreprocessorDirective kDirectives[] =
+{
+    { "if",         &HandleIfDirective,         ProcessWhenSkipping },
+    { "ifdef",      &HandleIfDefDirective,      ProcessWhenSkipping },
+    { "ifndef",     &HandleIfNDefDirective,     ProcessWhenSkipping },
+    { "else",       &HandleElseDirective,       ProcessWhenSkipping },
+    { "elif",       &HandleElifDirective,       ProcessWhenSkipping },
+    { "endif",      &HandleEndIfDirective,      ProcessWhenSkipping },
+
+    { "include",    &HandleIncludeDirective,    0 },
+    { "define",     &HandleDefineDirective,     0 },
+    { "undef",      &HandleUndefDirective,      0 },
+    { "warning",    &HandleWarningDirective,    0 },
+    { "error",      &HandleErrorDirective,      0 },
+    { "line",       &HandleLineDirective,       0 },
+    { "pragma",     &HandlePragmaDirective,     0 },
+
+    // TODO(tfoley): These are specific to GLSL, and probably
+    // shouldn't be enabled for HLSL or Slang
+    { "version",    &handleGLSLVersionDirective,    0 },
+    { "extension",  &handleGLSLExtensionDirective,  0 },
+
+    { NULL, NULL },
+};
+
+static const PreprocessorDirective kInvalidDirective = {
+    NULL, &HandleInvalidDirective, 0,
+};
+
+// Look up the directive with the given name.
+static PreprocessorDirective const* FindDirective(String const& name)
+{
+    char const* nameStr = name.Buffer();
+    for (int ii = 0; kDirectives[ii].name; ++ii)
+    {
+        if (strcmp(kDirectives[ii].name, nameStr) != 0)
+            continue;
+
+        return &kDirectives[ii];
+    }
+
+    return &kInvalidDirective;
+}
+
+// Process a directive, where the preprocessor has already consumed the
+// `#` token that started the directive line.
+static void HandleDirective(PreprocessorDirectiveContext* context)
+{
+    // Try to read the directive name.
+    context->directiveToken = PeekRawToken(context);
+
+    TokenType directiveTokenType = GetDirective(context).Type;
+
+    // An empty directive is allowed, and ignored.
+    if (directiveTokenType == TokenType::EndOfDirective)
+    {
+        return;
+    }
+    // Otherwise the directive name had better be an identifier
+    else if (directiveTokenType != TokenType::Identifier)
+    {
+        GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::expectedPreprocessorDirectiveName);
+        SkipToEndOfLine(context);
+        return;
+    }
+
+    // Consume the directive name token.
+    AdvanceRawToken(context);
+
+    // Look up the handler for the directive.
+    PreprocessorDirective const* directive = FindDirective(GetDirectiveName(context));
+
+    // If we are skipping disabled code, and the directive is not one
+    // of the small number that need to run even in that case, skip it.
+    if (IsSkipping(context) && !(directive->flags & PreprocessorDirectiveFlag::ProcessWhenSkipping))
+    {
+        SkipToEndOfLine(context);
+        return;
+    }
+
+    // Apply the directive-specific callback
+    (directive->callback)(context);
+
+    // We expect the directive callback to consume the entire line, so if
+    // it hasn't that is a parse error.
+    expectEndOfDirective(context);
+}
+
+// Read one token using the full preprocessor, with all its behaviors.
+static Token ReadToken(Preprocessor* preprocessor)
+{
+    for (;;)
+    {
+        // Look at the next raw token in the input.
+        Token const& token = PeekRawToken(preprocessor);
+
+        // If we have a directive (`#` at start of line) then handle it
+        if ((token.Type == TokenType::Pound) && (token.flags & TokenFlag::AtStartOfLine))
+        {
+            // Skip the `#`
+            AdvanceRawToken(preprocessor);
+
+            // Create a context for parsing the directive
+            PreprocessorDirectiveContext directiveContext;
+            directiveContext.preprocessor = preprocessor;
+            directiveContext.parseError = false;
+            directiveContext.haveDoneEndOfDirectiveChecks = false;
+
+            // Parse and handle the directive
+            HandleDirective(&directiveContext);
+            continue;
+        }
+
+        // otherwise, if we are currently in a skipping mode, then skip tokens
+        if (IsSkipping(preprocessor))
+        {
+            AdvanceRawToken(preprocessor);
+            continue;
+        }
+
+        // otherwise read a token, which may involve macro expansion
+        return AdvanceToken(preprocessor);
+    }
+}
+
+// intialize a preprocessor context, using the given sink for errros
+static void InitializePreprocessor(
+    Preprocessor*   preprocessor,
+    DiagnosticSink* sink)
+{
+    preprocessor->sink = sink;
+    preprocessor->includeHandler = NULL;
+    preprocessor->endOfFileToken.Type = TokenType::EndOfFile;
+    preprocessor->endOfFileToken.flags = TokenFlag::AtStartOfLine;
+}
+
+// clean up after an environment
+PreprocessorEnvironment::~PreprocessorEnvironment()
+{
+    for (auto pair : this->macros)
+    {
+        DestroyMacro(NULL, pair.Value);
+    }
+}
+
+// finalize a preprocessor and free any memory still in use
+static void FinalizePreprocessor(
+    Preprocessor*   preprocessor)
+{
+    // Clear out any waiting input streams
+    PreprocessorInputStream* input = preprocessor->inputStream;
+    while (input)
+    {
+        PreprocessorInputStream* parent = input->parent;
+        DestroyInputStream(preprocessor, input);
+        input = parent;
+    }
+
+#if 0
+    // clean up any macros that were allocated
+    for (auto pair : preprocessor->globalEnv.macros)
+    {
+        DestroyMacro(preprocessor, pair.Value);
+    }
+#endif
+}
+
+// Add a simple macro definition from a string (e.g., for a
+// `-D` option passed on the command line
+static void DefineMacro(
+    Preprocessor*   preprocessor,
+    String const&   key,
+    String const&   value)
+{
+    String fileName = "command line";
+    PreprocessorMacro* macro = CreateMacro(preprocessor);
+
+    // Use existing `Lexer` to generate a token stream.
+    Lexer lexer(fileName, value, GetSink(preprocessor));
+    macro->tokens = lexer.lexAllTokens();
+    macro->nameToken = Token(TokenType::Identifier, key, 0, 0, 0, fileName);
+
+    PreprocessorMacro* oldMacro = NULL;
+    if (preprocessor->globalEnv.macros.TryGetValue(key, oldMacro))
+    {
+        DestroyMacro(preprocessor, oldMacro);
+    }
+
+    preprocessor->globalEnv.macros[key] = macro;
+}
+
+// read the entire input into tokens
+static TokenList ReadAllTokens(
+    Preprocessor*   preprocessor)
+{
+    TokenList tokens;
+    for (;;)
+    {
+        Token token = ReadToken(preprocessor);
+
+        tokens.mTokens.Add(token);
+
+        // Note: we include the EOF token in the list,
+        // since that is expected by the `TokenList` type.
+        if (token.Type == TokenType::EndOfFile)
+            break;
+    }
+    return tokens;
+}
+
+TokenList preprocessSource(
+    CoreLib::String const& source,
+    CoreLib::String const& fileName,
+    DiagnosticSink* sink,
+    IncludeHandler* includeHandler,
+    CoreLib::Dictionary<CoreLib::String, CoreLib::String>  defines,
+    ProgramSyntaxNode*  syntax)
+{
+    Preprocessor preprocessor;
+    InitializePreprocessor(&preprocessor, sink);
+    preprocessor.syntax = syntax;
+
+    preprocessor.includeHandler = includeHandler;
+    for (auto p : defines)
+    {
+        DefineMacro(&preprocessor, p.Key, p.Value);
+    }
+
+    // create an initial input stream based on the provided buffer
+    preprocessor.inputStream = CreateInputStreamForSource(&preprocessor, source, fileName);
+
+    TokenList tokens = ReadAllTokens(&preprocessor);
+
+    FinalizePreprocessor(&preprocessor);
+
+    // debugging: build the pre-processed source back together
+#if 0
+    StringBuilder sb;
+    for (auto t : tokens)
+    {
+        if (t.flags & TokenFlag::AtStartOfLine)
+        {
+            sb << "\n";
+        }
+        else if (t.flags & TokenFlag::AfterWhitespace)
+        {
+            sb << " ";
+        }
+        
+        sb << t.Content;
+    }
+
+    String s = sb.ProduceString();
+#endif
+
+    return tokens;
+}
+
+}}