Use slang- prefix on slang compiler and core source (#973)

* Prefixing source files in source/slang with slang-

* Prefix source in source/slang with slang- prefix.

* Rename core source files with slang- prefix.

* Update project files.

* Fix problems from automatic merge.

1 files changed, 2302 insertions, 0 deletions

diff --git a/source/slang/slang-preprocessor.cpp b/source/slang/slang-preprocessor.cpp
new file mode 100644
index 000000000..16a64e571
--- /dev/null
+++ b/source/slang/slang-preprocessor.cpp
@@ -0,0 +1,2302 @@
+// slang-preprocessor.cpp
+#include "slang-preprocessor.h"
+
+#include "slang-compiler.h"
+#include "slang-diagnostics.h"
+#include "slang-lexer.h"
+// Needed so that we can construct modifier syntax to represent GLSL directives
+#include "slang-syntax.h"
+
+#include <assert.h>
+
+// This file provides an implementation of a simple C-style preprocessor.
+// It does not aim for 100% compatibility with any particular preprocessor
+// specification, but the goal is to have it accept the most common
+// idioms for using the preprocessor, found in shader code in the wild.
+
+
+namespace Slang {
+
+// State of a preprocessor conditional, which can change when
+// we encounter directives like `#elif` or `#endif`
+enum class PreprocessorConditionalState
+{
+    Before, // We have not yet seen a branch with a `true` condition.
+    During, // We are inside the branch with a `true` condition.
+    After,  // We have already seen the branch with a `true` condition.
+};
+
+// Represents a preprocessor conditional that we are currently
+// nested inside.
+struct PreprocessorConditional
+{
+    // The next outer conditional in the current file/stream, or NULL.
+    PreprocessorConditional*        parent;
+
+    // The directive token that started the conditional (an `#if` or `#ifdef`)
+    Token                           ifToken;
+
+    // The `#else` directive token, if one has been seen (otherwise `TokenType::Unknown`)
+    Token                           elseToken;
+
+    // The state of the conditional
+    PreprocessorConditionalState    state;
+};
+
+struct PreprocessorMacro;
+
+struct PreprocessorEnvironment
+{
+    // The "outer" environment, to be used if lookup in this env fails
+    PreprocessorEnvironment*                parent = NULL;
+
+    // Macros defined in this environment
+    Dictionary<Name*, PreprocessorMacro*>  macros;
+
+    ~PreprocessorEnvironment();
+};
+
+// Input tokens can either come from source text, or from macro expansion.
+// In general, input streams can be nested, so we have to keep a conceptual
+// stack of input.
+
+struct PrimaryInputStream;
+
+// A stream of input tokens to be consumed
+struct PreprocessorInputStream
+{
+    // The primary input stream that is the parent to this one,
+    // or NULL if this stream is itself a primary stream.
+    PrimaryInputStream*             primaryStream;
+
+    // The next input stream up the stack, if any.
+    PreprocessorInputStream*        parent;
+
+    // Environment to use when looking up macros
+    PreprocessorEnvironment*        environment;
+
+    // Destructor is virtual so that we can clean up
+    // after concrete subtypes.
+    virtual ~PreprocessorInputStream() = default;
+};
+
+// A "primary" input stream represents the top-level context of a file
+// being parsed, and tracks things like preprocessor conditional state
+struct PrimaryInputStream : PreprocessorInputStream
+{
+    // The next *primary* input stream up the stack
+    PrimaryInputStream*             parentPrimaryInputStream;
+
+    // The deepest preprocessor conditional active for this stream.
+    PreprocessorConditional*        conditional;
+
+    // The lexer state that will provide input
+    Lexer lexer;
+
+    // One token of lookahead
+    Token token;
+};
+
+// A "secondary" input stream represents code that is being expanded
+// into the current scope, but which had already been tokenized before.
+//
+struct PretokenizedInputStream : PreprocessorInputStream
+{
+    // Reader for pre-tokenized input
+    TokenReader                     tokenReader;
+};
+
+// A pre-tokenized input stream that will only be used once, and which
+// therefore owns the memory for its tokens.
+struct SimpleTokenInputStream : PretokenizedInputStream
+{
+    // A list of raw tokens that will provide input
+    TokenList lexedTokens;
+};
+
+struct MacroExpansion : PretokenizedInputStream
+{
+    // The macro we will expand
+    PreprocessorMacro*  macro;
+};
+
+struct ObjectLikeMacroExpansion : MacroExpansion
+{
+};
+
+struct FunctionLikeMacroExpansion : MacroExpansion
+{
+    // Environment for macro arguments
+    PreprocessorEnvironment     argumentEnvironment;
+};
+
+// An enumeration for the diferent types of macros
+enum class PreprocessorMacroFlavor
+{
+    ObjectLike,
+    FunctionArg,
+    FunctionLike,
+};
+
+// In the current design (which we may want to re-consider),
+// a macro is a specialized flavor of input stream, that
+// captures the token list in its expansion, and then
+// can be "played back."
+struct PreprocessorMacro
+{
+    // The name under which the macro was `#define`d
+    NameLoc                     nameAndLoc;
+
+    // Parameters of the macro, in case of a function-like macro
+    List<NameLoc>               params;
+
+    // The tokens that make up the macro body
+    TokenList                   tokens;
+
+    // The flavor of macro
+    PreprocessorMacroFlavor     flavor;
+
+    // The environment in which this macro needs to be expanded.
+    // For ordinary macros this will be the global environment,
+    // while for function-like macro arguments, it will be
+    // the environment of the macro invocation.
+    PreprocessorEnvironment*    environment;
+
+    //
+    Name* getName()
+    {
+        return nameAndLoc.name;
+    }
+
+    SourceLoc getLoc()
+    {
+        return nameAndLoc.loc;
+    }
+};
+
+// State of the preprocessor
+struct Preprocessor
+{
+    // diagnostics sink to use when writing messages
+    DiagnosticSink*                         sink;
+
+    // An external callback interface to use when looking
+    // for files in a `#include` directive
+    IncludeHandler*                         includeHandler;
+
+    // Current input stream (top of the stack of input)
+    PreprocessorInputStream*                inputStream;
+
+    // Currently-defined macros
+    PreprocessorEnvironment                 globalEnv;
+
+    // A pre-allocated token that can be returned to
+    // represent end-of-input situations.
+    Token                                   endOfFileToken;
+
+        /// The linkage the provides the context for preprocessing
+    Linkage*                                linkage = nullptr;
+
+        /// The module, if any, that the preprocessed result will belong to
+    Module*                                 parentModule = nullptr;
+
+    // The unique identities of any paths that have issued `#pragma once` directives to
+    // stop them from being included again.
+    HashSet<String>                         pragmaOnceUniqueIdentities;
+
+    NamePool* getNamePool() { return linkage->getNamePool(); }
+    SourceManager* getSourceManager() { return linkage->getSourceManager(); }
+};
+
+// Convenience routine to access the diagnostic sink
+static DiagnosticSink* GetSink(Preprocessor* preprocessor)
+{
+    return preprocessor->sink;
+}
+
+//
+// Forward declarations
+//
+
+static void DestroyConditional(PreprocessorConditional* conditional);
+static void DestroyMacro(Preprocessor* preprocessor, PreprocessorMacro* macro);
+static bool IsSkipping(Preprocessor* preprocessor);
+
+//
+// Basic Input Handling
+//
+
+// Create a fresh input stream
+static void  initializeInputStream(Preprocessor* preprocessor, PreprocessorInputStream* inputStream)
+{
+    inputStream->parent = NULL;
+    inputStream->environment = &preprocessor->globalEnv;
+}
+
+static void initializePrimaryInputStream(Preprocessor* preprocessor, PrimaryInputStream* inputStream)
+{
+    initializeInputStream(preprocessor, inputStream);
+    inputStream->primaryStream = inputStream;
+    inputStream->conditional = NULL;
+}
+
+// Destroy an input stream
+static void destroyInputStream(Preprocessor* /*preprocessor*/, PreprocessorInputStream* inputStream)
+{
+    delete inputStream;
+}
+
+// Create an input stream to represent a pre-tokenized input file.
+// TODO(tfoley): pre-tokenizing files isn't going to work in the long run.
+static PreprocessorInputStream* CreateInputStreamForSource(
+    Preprocessor*   preprocessor,
+    SourceView*     sourceView)
+{
+    MemoryArena* memoryArena = sourceView->getSourceManager()->getMemoryArena();
+
+    PrimaryInputStream* inputStream = new PrimaryInputStream();
+    initializePrimaryInputStream(preprocessor, inputStream);
+
+    // initialize the embedded lexer so that it can generate a token stream
+    inputStream->lexer.initialize(sourceView, GetSink(preprocessor), preprocessor->getNamePool(), memoryArena);
+    inputStream->token = inputStream->lexer.lexToken();
+
+    return inputStream;
+}
+
+static PrimaryInputStream* asPrimaryInputStream(PreprocessorInputStream* inputStream)
+{
+    auto primaryStream = inputStream->primaryStream;
+    if(primaryStream == inputStream)
+        return primaryStream;
+    return nullptr;
+}
+
+
+static void PushInputStream(Preprocessor* preprocessor, PreprocessorInputStream* inputStream)
+{
+    inputStream->parent = preprocessor->inputStream;
+    if(!asPrimaryInputStream(inputStream))
+        inputStream->primaryStream = preprocessor->inputStream->primaryStream;
+    preprocessor->inputStream = inputStream;
+}
+
+// Called when we reach the end of an input stream.
+// Performs some validation and then destroys the input stream if required.
+static void EndInputStream(Preprocessor* preprocessor, PreprocessorInputStream* inputStream)
+{
+    if(auto primaryStream = asPrimaryInputStream(inputStream))
+    {
+        // If there are any conditionals that weren't completed, then it is an error
+        if (primaryStream->conditional)
+        {
+            PreprocessorConditional* conditional = primaryStream->conditional;
+
+            GetSink(preprocessor)->diagnose(conditional->ifToken.loc, Diagnostics::endOfFileInPreprocessorConditional);
+
+            while (conditional)
+            {
+                PreprocessorConditional* parent = conditional->parent;
+                DestroyConditional(conditional);
+                conditional = parent;
+            }
+        }
+    }
+
+    destroyInputStream(preprocessor, inputStream);
+}
+
+// Consume one token from an input stream
+static Token AdvanceRawToken(PreprocessorInputStream* inputStream, LexerFlags lexerFlags = 0)
+{
+    if( auto primaryStream = asPrimaryInputStream(inputStream) )
+    {
+        auto result = primaryStream->token;
+        primaryStream->token = primaryStream->lexer.lexToken(lexerFlags);
+        return result;
+    }
+    else
+    {
+        PretokenizedInputStream* pretokenized = (PretokenizedInputStream*) inputStream;
+        return pretokenized->tokenReader.AdvanceToken();
+    }
+}
+
+// Peek one token from an input stream
+static Token PeekRawToken(PreprocessorInputStream* inputStream)
+{
+    if( auto primaryStream = asPrimaryInputStream(inputStream) )
+    {
+        return primaryStream->token;
+    }
+    else
+    {
+        PretokenizedInputStream* pretokenized = (PretokenizedInputStream*) inputStream;
+        return pretokenized->tokenReader.PeekToken();
+    }
+}
+
+// Peek one token type from an input stream
+static TokenType PeekRawTokenType(PreprocessorInputStream* inputStream)
+{
+    if( auto primaryStream = asPrimaryInputStream(inputStream) )
+    {
+        return primaryStream->token.type;
+    }
+    else
+    {
+        PretokenizedInputStream* pretokenized = (PretokenizedInputStream*) inputStream;
+        return pretokenized->tokenReader.PeekTokenType();
+    }
+}
+
+
+// Read one token in "raw" mode (meaning don't expand macros)
+static Token AdvanceRawToken(Preprocessor* preprocessor, LexerFlags lexerFlags = 0)
+{
+    for(;;)
+    {
+        // Look at the input stream on top of the stack
+        PreprocessorInputStream* inputStream = preprocessor->inputStream;
+
+        // If there isn't one, then there is no more input left to read.
+        if(!inputStream)
+        {
+            return preprocessor->endOfFileToken;
+        }
+
+        // The top-most input stream may be at its end
+        if(PeekRawTokenType(inputStream) == TokenType::EndOfFile)
+        {
+            // If there is another stream remaining, switch to it
+            if(inputStream->parent)
+            {
+                preprocessor->inputStream = inputStream->parent;
+                EndInputStream(preprocessor, inputStream);
+                continue;
+            }
+        }
+
+        // Everything worked, so read a token from the top-most stream
+        return AdvanceRawToken(
+            inputStream,
+            lexerFlags | (IsSkipping(preprocessor) ? kLexerFlag_IgnoreInvalid : 0));
+    }
+}
+
+// Return the next token in "raw" mode, but don't advance the
+// current token state.
+static Token PeekRawToken(Preprocessor* preprocessor)
+{
+    // We need to find the stream that `advanceRawToken` would read from.
+    PreprocessorInputStream* inputStream = preprocessor->inputStream;
+    for (;;)
+    {
+        if (!inputStream)
+        {
+            // No more input streams left to read
+            return preprocessor->endOfFileToken;
+        }
+
+        // The top-most input stream may be at its end, so
+        // look one entry up the stack (don't actually pop
+        // here, since we are just peeking)
+        if (PeekRawTokenType(inputStream) == TokenType::EndOfFile)
+        {
+            if (inputStream->parent)
+            {
+                inputStream = inputStream->parent;
+                continue;
+            }
+        }
+
+        // Everything worked, so the token we just peeked is fine.
+        return PeekRawToken(inputStream);
+    }
+}
+
+// Get the location of the current (raw) token
+static SourceLoc PeekLoc(Preprocessor* preprocessor)
+{
+    return PeekRawToken(preprocessor).loc;
+}
+
+// Get the `TokenType` of the current (raw) token
+static TokenType PeekRawTokenType(Preprocessor* preprocessor)
+{
+    return PeekRawToken(preprocessor).type;
+}
+
+//
+// Macros
+//
+
+// Create a macro
+static PreprocessorMacro* CreateMacro(Preprocessor* preprocessor)
+{
+    // TODO(tfoley): Allocate these more intelligently.
+    // For example, consider pooling them on the preprocessor.
+
+    PreprocessorMacro* macro = new PreprocessorMacro();
+    macro->flavor = PreprocessorMacroFlavor::ObjectLike;
+    macro->environment = &preprocessor->globalEnv;
+    return macro;
+}
+
+// Destroy a macro
+static void DestroyMacro(Preprocessor* /*preprocessor*/, PreprocessorMacro* macro)
+{
+    delete macro;
+}
+
+
+// Find the currently-defined macro of the given name, or return NULL
+static PreprocessorMacro* LookupMacro(PreprocessorEnvironment* environment, Name* name)
+{
+    for(PreprocessorEnvironment* e = environment; e; e = e->parent)
+    {
+        PreprocessorMacro* macro = NULL;
+        if (e->macros.TryGetValue(name, macro))
+            return macro;
+    }
+
+    return NULL;
+}
+
+static PreprocessorEnvironment* GetCurrentEnvironment(Preprocessor* preprocessor)
+{
+    // The environment we will use for looking up a macro is associated
+    // with the current input stream (because it may include entries
+    // for macro arguments).
+    //
+    // We need to be careful, though, when we are at the end of an
+    // input stream (e.g., representing one argument), so that we
+    // don't use its environment.
+
+    PreprocessorInputStream* inputStream = preprocessor->inputStream;
+
+    for(;;)
+    {
+        // If there is no input stream that isn't at its end,
+        // then fall back to the global environment.
+        if (!inputStream)
+            return &preprocessor->globalEnv;
+
+        // If the current input stream is at its end, then
+        // fall back to its parent stream.
+        if (PeekRawTokenType(inputStream) == TokenType::EndOfFile)
+        {
+            inputStream = inputStream->parent;
+            continue;
+        }
+
+        // If we've found an active stream that isn't at its end,
+        // then use that for lookup.
+        return inputStream->environment;
+    }
+}
+
+static PreprocessorMacro* LookupMacro(Preprocessor* preprocessor, Name* name)
+{
+    return LookupMacro(GetCurrentEnvironment(preprocessor), name);
+}
+
+// A macro is "busy" if it is currently being used for expansion.
+// A macro cannot be expanded again while busy, to avoid infinite recursion.
+static bool IsMacroBusy(PreprocessorMacro* /*macro*/)
+{
+    // TODO: need to implement this correctly
+    //
+    // The challenge here is that we are implementing expansion
+    // for argumenst to function-like macros in a "lazy" fashion.
+    //
+    // The letter of the spec is that we should macro expand
+    // each argument *before* substitution, and then go and
+    // macro-expand the substituted body. This means that we
+    // can invoke a macro as part of an argument to an
+    // invocation of the same macro:
+    //
+    //     FOO( 1, FOO(22), 333 );
+    //
+    // In our implementation, the "inner" invocation of `FOO`
+    // gets expanded at the point where it gets referenced
+    // in the body of the "outer" invocation of `FOO`.
+    // Doing things this way leads to greatly simplified
+    // code for handling expansion.
+    //
+    // A proper implementation of `IsMacroBusy` needs to
+    // take context into account, so that it bans recursive
+    // use of a macro when it occurs (indirectly) through
+    // the *body* of the expansion, but not when it occcurs
+    // only through an *argument*.
+    return false;
+}
+
+//
+// Reading Tokens With Expansion
+//
+
+static void InitializeMacroExpansion(
+    Preprocessor*       preprocessor,
+    MacroExpansion*     expansion,
+    PreprocessorMacro*  macro)
+{
+    initializeInputStream(preprocessor, expansion);
+
+    expansion->parent = preprocessor->inputStream;
+    expansion->primaryStream = preprocessor->inputStream->primaryStream;
+
+    expansion->environment = macro->environment;
+    expansion->macro = macro;
+    expansion->tokenReader = TokenReader(macro->tokens);
+}
+
+static void PushMacroExpansion(
+    Preprocessor*   preprocessor,
+    MacroExpansion* expansion)
+{
+    PushInputStream(preprocessor, expansion);
+}
+
+static void AddEndOfStreamToken(
+    Preprocessor*       preprocessor,
+    PreprocessorMacro*  macro)
+{
+    Token token = PeekRawToken(preprocessor);
+    token.type = TokenType::EndOfFile;
+    macro->tokens.mTokens.add(token);
+}
+
+static SimpleTokenInputStream* createSimpleInputStream(
+    Preprocessor*   preprocessor,
+    Token const&    token)
+{
+    SimpleTokenInputStream* inputStream = new SimpleTokenInputStream();
+    initializeInputStream(preprocessor, inputStream);
+
+    inputStream->lexedTokens.mTokens.add(token);
+
+    Token eofToken;
+    eofToken.type = TokenType::EndOfFile;
+    eofToken.loc = token.loc;
+    eofToken.flags = TokenFlag::AfterWhitespace | TokenFlag::AtStartOfLine;
+    inputStream->lexedTokens.mTokens.add(eofToken);
+
+    inputStream->tokenReader = TokenReader(inputStream->lexedTokens);
+
+    return inputStream;
+}
+
+// Check whether the current token on the given input stream should be
+// treated as a macro invocation, and if so set up state for expanding
+// that macro.
+static void MaybeBeginMacroExpansion(
+    Preprocessor*               preprocessor )
+{
+    // We iterate because the first token in the expansion of one
+    // macro may be another macro invocation.
+    for (;;)
+    {
+        // Look at the next token ahead of us
+        Token token = PeekRawToken(preprocessor);
+
+        // Not an identifier? Can't be a macro.
+        if (token.type != TokenType::Identifier)
+            return;
+
+        // Look for a macro with the given name.
+        Name* name = token.getName();
+        PreprocessorMacro* macro = LookupMacro(preprocessor, name);
+
+        // Not a macro? Can't be an invocation.
+        if (!macro)
+            return;
+
+        // If the macro is busy (already being expanded),
+        // don't try to trigger recursive expansion
+        if (IsMacroBusy(macro))
+            return;
+
+        // We might already have looked at this token,
+        // and need to suppress expansion
+        if (token.flags & TokenFlag::SuppressMacroExpansion)
+            return;
+
+        // A function-style macro invocation should only match
+        // if the token *after* the identifier is `(`. This
+        // requires more lookahead than we usually have/need
+        if (macro->flavor == PreprocessorMacroFlavor::FunctionLike)
+        {
+            // Consume the token that (possibly) triggered macro expansion
+            AdvanceRawToken(preprocessor);
+
+            // Look at the next token, and see if it is an opening `(`
+            // that indicates we should actually expand a macro.
+            if(PeekRawTokenType(preprocessor) != TokenType::LParent)
+            {
+                // In this case, we are in a bit of a mess, because we have
+                // consumed the token that named the macro, but we need to
+                // make sure that token (and not whatever came after it)
+                // gets returned to the user.
+                //
+                // To work around this we will construct a short-lived input
+                // stream just to handle that one token, and also set
+                // a flag on the token to keep us from doing this logic again.
+
+                token.flags |= TokenFlag::SuppressMacroExpansion;
+
+                SimpleTokenInputStream* simpleStream = createSimpleInputStream(preprocessor, token);
+                PushInputStream(preprocessor, simpleStream);
+                return;
+            }
+
+            // Consume the opening `(`
+            Token leftParen = AdvanceRawToken(preprocessor);
+
+            FunctionLikeMacroExpansion* expansion = new FunctionLikeMacroExpansion();
+            InitializeMacroExpansion(preprocessor, expansion, macro);
+            expansion->argumentEnvironment.parent = &preprocessor->globalEnv;
+            expansion->environment = &expansion->argumentEnvironment;
+
+            // Try to read any arguments present.
+            UInt paramCount = macro->params.getCount();
+            UInt argIndex = 0;
+
+            switch (PeekRawTokenType(preprocessor))
+            {
+            case TokenType::EndOfFile:
+            case TokenType::RParent:
+                // No arguments.
+                break;
+
+            default:
+                // At least one argument
+                while(argIndex < paramCount)
+                {
+                    // Read an argument
+
+                    // Create the argument, represented as a special flavor of macro
+                    PreprocessorMacro* arg = CreateMacro(preprocessor);
+                    arg->flavor = PreprocessorMacroFlavor::FunctionArg;
+                    arg->environment = GetCurrentEnvironment(preprocessor);
+
+                    // Associate the new macro with its parameter name
+                    NameLoc paramNameAndLoc = macro->params[argIndex];
+                    Name* paramName = paramNameAndLoc.name;
+                    arg->nameAndLoc = paramNameAndLoc;
+                    expansion->argumentEnvironment.macros[paramName] = arg;
+                    argIndex++;
+
+                    // Read tokens for the argument
+
+                    // We track the nesting depth, since we don't break
+                    // arguments on a `,` nested in balanced parentheses
+                    //
+                    int nesting = 0;
+                    for (;;)
+                    {
+                        switch (PeekRawTokenType(preprocessor))
+                        {
+                        case TokenType::EndOfFile:
+                            // if we reach the end of the file,
+                            // then we have an error, and need to
+                            // bail out
+                            AddEndOfStreamToken(preprocessor, arg);
+                            goto doneWithAllArguments;
+
+                        case TokenType::RParent:
+                            // If we see a right paren when we aren't nested
+                            // then we are at the end of an argument
+                            if (nesting == 0)
+                            {
+                                AddEndOfStreamToken(preprocessor, arg);
+                                goto doneWithAllArguments;
+                            }
+                            // Otherwise we decrease our nesting depth, add
+                            // the token, and keep going
+                            nesting--;
+                            break;
+
+                        case TokenType::Comma:
+                            // If we see a comma when we aren't nested
+                            // then we are at the end of an argument
+                            if (nesting == 0)
+                            {
+                                AddEndOfStreamToken(preprocessor, arg);
+                                AdvanceRawToken(preprocessor);
+                                goto doneWithArgument;
+                            }
+                            // Otherwise we add it as a normal token
+                            break;
+
+                        case TokenType::LParent:
+                            // If we see a left paren then we need to
+                            // increase our tracking of nesting
+                            nesting++;
+                            break;
+
+                        default:
+                            break;
+                        }
+
+                        // Add the token and continue parsing.
+                        arg->tokens.mTokens.add(AdvanceRawToken(preprocessor));
+                    }
+                doneWithArgument: {}
+                    // We've parsed an argument and should move onto
+                    // the next one.
+                }
+                break;
+            }
+        doneWithAllArguments:
+            // TODO: handle possible varargs
+
+            // Expect closing right paren
+            if (PeekRawTokenType(preprocessor) == TokenType::RParent)
+            {
+                AdvanceRawToken(preprocessor);
+            }
+            else
+            {
+                GetSink(preprocessor)->diagnose(PeekLoc(preprocessor), Diagnostics::expectedTokenInMacroArguments, TokenType::RParent, PeekRawTokenType(preprocessor));
+            }
+
+            UInt argCount = argIndex;
+            if (argCount != paramCount)
+            {
+                GetSink(preprocessor)->diagnose(PeekLoc(preprocessor), Diagnostics::wrongNumberOfArgumentsToMacro, paramCount, argCount);
+            }
+
+            // We are ready to expand.
+            PushMacroExpansion(preprocessor, expansion);
+        }
+        else
+        {
+            // Consume the token that triggered macro expansion
+            AdvanceRawToken(preprocessor);
+
+            // Object-like macros are the easy case.
+            ObjectLikeMacroExpansion* expansion = new ObjectLikeMacroExpansion();
+            InitializeMacroExpansion(preprocessor, expansion, macro);
+            PushMacroExpansion(preprocessor, expansion);
+        }
+    }
+}
+
+// Read one token with macro-expansion enabled.
+static Token AdvanceToken(Preprocessor* preprocessor)
+{
+top:
+    // Check whether we need to macro expand at the cursor.
+    MaybeBeginMacroExpansion(preprocessor);
+
+    // Read a raw token (now that expansion has been triggered)
+    Token token = AdvanceRawToken(preprocessor);
+
+    // Check if we need to perform token pasting
+    if (PeekRawTokenType(preprocessor) != TokenType::PoundPound)
+    {
+        // If we aren't token pasting, then we are done
+        return token;
+    }
+    else
+    {
+        // We are pasting tokens, which could get messy
+
+        StringBuilder sb;
+        sb << token.Content;
+
+        while (PeekRawTokenType(preprocessor) == TokenType::PoundPound)
+        {
+            // Consume the `##`
+            AdvanceRawToken(preprocessor);
+
+            // Possibly macro-expand the next token
+            MaybeBeginMacroExpansion(preprocessor);
+
+            // Read the next raw token (now that expansion has been triggered)
+            Token nextToken = AdvanceRawToken(preprocessor);
+
+            sb << nextToken.Content;
+        }
+
+        // Now re-lex the input
+
+        SourceManager* sourceManager = preprocessor->getSourceManager();
+
+        // We create a dummy file to represent the token-paste operation
+        PathInfo pathInfo = PathInfo::makeTokenPaste();
+       
+        SourceFile* sourceFile = sourceManager->createSourceFileWithString(pathInfo, sb.ProduceString());
+        SourceView* sourceView = sourceManager->createSourceView(sourceFile, nullptr);
+
+        Lexer lexer;
+        lexer.initialize(sourceView, GetSink(preprocessor), preprocessor->getNamePool(), sourceManager->getMemoryArena());
+
+        SimpleTokenInputStream* inputStream = new SimpleTokenInputStream();
+        initializeInputStream(preprocessor, inputStream);
+
+        inputStream->lexedTokens = lexer.lexAllTokens();
+        inputStream->tokenReader = TokenReader(inputStream->lexedTokens);
+
+        // We expect the reuslt of lexing to be two tokens: one for the actual value,
+        // and one for the end-of-input marker.
+        if (inputStream->tokenReader.GetCount() != 2)
+        {
+            // We expect a token paste to produce a single token
+            // TODO(tfoley): emit a diagnostic here
+        }
+
+        PushInputStream(preprocessor, inputStream);
+        goto top;
+    }
+}
+
+// Read one token with macro-expansion enabled.
+//
+// Note that because triggering macro expansion may
+// involve changing the input-stream state, this
+// operation *can* have side effects.
+static Token PeekToken(Preprocessor* preprocessor)
+{
+    // Check whether we need to macro expand at the cursor.
+    MaybeBeginMacroExpansion(preprocessor);
+
+    // Peek a raw token (now that expansion has been triggered)
+    return PeekRawToken(preprocessor);
+
+    // TODO: need a plan for how to handle token pasting
+    // here without it being onerous. Would be nice if we
+    // didn't have to re-do pasting on a "peek"...
+}
+
+// Peek the type of the next token, including macro expansion.
+static TokenType PeekTokenType(Preprocessor* preprocessor)
+{
+    return PeekToken(preprocessor).type;
+}
+
+//
+// Preprocessor Directives
+//
+
+// When reading a preprocessor directive, we use a context
+// to wrap the direct preprocessor routines defines so far.
+//
+// One of the most important things the directive context
+// does is give us a convenient way to read tokens with
+// a guarantee that we won't read past the end of a line.
+struct PreprocessorDirectiveContext
+{
+    // The preprocessor that is parsing the directive.
+    Preprocessor*   preprocessor;
+
+    // The directive token (e.g., the `if` in `#if`).
+    // Useful for reference in diagnostic messages.
+    Token           directiveToken;
+
+    // Has any kind of parse error been encountered in
+    // the directive so far?
+    bool            parseError;
+
+    // Have we done the necessary checks at the end
+    // of the directive already?
+    bool            haveDoneEndOfDirectiveChecks;
+};
+
+// Get the token for  the preprocessor directive being parsed.
+inline Token const& GetDirective(PreprocessorDirectiveContext* context)
+{
+    return context->directiveToken;
+}
+
+// Get the name of the directive being parsed.
+inline UnownedStringSlice const& GetDirectiveName(PreprocessorDirectiveContext* context)
+{
+    return context->directiveToken.Content;
+}
+
+// Get the location of the directive being parsed.
+inline SourceLoc const& GetDirectiveLoc(PreprocessorDirectiveContext* context)
+{
+    return context->directiveToken.loc;
+}
+
+// Wrapper to get the diagnostic sink in the context of a directive.
+static inline DiagnosticSink* GetSink(PreprocessorDirectiveContext* context)
+{
+    return GetSink(context->preprocessor);
+}
+
+// Wrapper to get a "current" location when parsing a directive
+static SourceLoc PeekLoc(PreprocessorDirectiveContext* context)
+{
+    return PeekLoc(context->preprocessor);
+}
+
+// Wrapper to look up a macro in the context of a directive.
+static PreprocessorMacro* LookupMacro(PreprocessorDirectiveContext* context, Name* name)
+{
+    return LookupMacro(context->preprocessor, name);
+}
+
+// Determine if we have read everything on the directive's line.
+static bool IsEndOfLine(PreprocessorDirectiveContext* context)
+{
+    return PeekRawToken(context->preprocessor).type == TokenType::EndOfDirective;
+}
+
+// Peek one raw token in a directive, without going past the end of the line.
+static Token PeekRawToken(PreprocessorDirectiveContext* context)
+{
+    return PeekRawToken(context->preprocessor);
+}
+
+// Read one raw token in a directive, without going past the end of the line.
+static Token AdvanceRawToken(PreprocessorDirectiveContext* context, LexerFlags lexerFlags = 0)
+{
+    if (IsEndOfLine(context))
+        return PeekRawToken(context);
+    return AdvanceRawToken(context->preprocessor, lexerFlags);
+}
+
+// Peek next raw token type, without going past the end of the line.
+static TokenType PeekRawTokenType(PreprocessorDirectiveContext* context)
+{
+    return PeekRawTokenType(context->preprocessor);
+}
+
+// Read one token, with macro-expansion, without going past the end of the line.
+static Token AdvanceToken(PreprocessorDirectiveContext* context)
+{
+    if (IsEndOfLine(context))
+        return PeekRawToken(context);
+    return AdvanceToken(context->preprocessor);
+}
+
+// Peek one token, with macro-expansion, without going past the end of the line.
+static Token PeekToken(PreprocessorDirectiveContext* context)
+{
+    if (IsEndOfLine(context))
+        return context->preprocessor->endOfFileToken;
+    return PeekToken(context->preprocessor);
+}
+
+// Peek next token type, with macro-expansion, without going past the end of the line.
+static TokenType PeekTokenType(PreprocessorDirectiveContext* context)
+{
+    if (IsEndOfLine(context))
+        return TokenType::EndOfDirective;
+    return PeekTokenType(context->preprocessor);
+}
+
+// Skip to the end of the line (useful for recovering from errors in a directive)
+static void SkipToEndOfLine(PreprocessorDirectiveContext* context)
+{
+    while(!IsEndOfLine(context))
+    {
+        AdvanceRawToken(context);
+    }
+}
+
+static bool ExpectRaw(PreprocessorDirectiveContext* context, TokenType tokenType, DiagnosticInfo const& diagnostic, Token* outToken = NULL)
+{
+    if (PeekRawTokenType(context) != tokenType)
+    {
+        // Only report the first parse error within a directive
+        if (!context->parseError)
+        {
+            GetSink(context)->diagnose(PeekLoc(context), diagnostic, tokenType, GetDirectiveName(context));
+        }
+        context->parseError = true;
+        return false;
+    }
+    Token const& token = AdvanceRawToken(context);
+    if (outToken)
+        *outToken = token;
+    return true;
+}
+
+static bool Expect(PreprocessorDirectiveContext* context, TokenType tokenType, DiagnosticInfo const& diagnostic, Token* outToken = NULL)
+{
+    if (PeekTokenType(context) != tokenType)
+    {
+        // Only report the first parse error within a directive
+        if (!context->parseError)
+        {
+            GetSink(context)->diagnose(PeekLoc(context), diagnostic, tokenType, GetDirectiveName(context));
+            context->parseError = true;
+        }
+        return false;
+    }
+    Token const& token = AdvanceToken(context);
+    if (outToken)
+        *outToken = token;
+    return true;
+}
+
+
+
+//
+// Preprocessor Conditionals
+//
+
+// Determine whether the current preprocessor state means we
+// should be skipping tokens.
+static bool IsSkipping(Preprocessor* preprocessor)
+{
+    PreprocessorInputStream* inputStream = preprocessor->inputStream;
+    if (!inputStream) return false;
+
+    PrimaryInputStream* primaryStream = inputStream->primaryStream;
+    if(!primaryStream) return false;
+
+    // If we are not inside a preprocessor conditional, then don't skip
+    PreprocessorConditional* conditional = primaryStream->conditional;
+    if (!conditional) return false;
+
+    // skip tokens unless the conditional is inside its `true` case
+    return conditional->state != PreprocessorConditionalState::During;
+}
+
+// Wrapper for use inside directives
+static inline bool IsSkipping(PreprocessorDirectiveContext* context)
+{
+    return IsSkipping(context->preprocessor);
+}
+
+// Create a preprocessor conditional
+static PreprocessorConditional* CreateConditional(Preprocessor* /*preprocessor*/)
+{
+    // TODO(tfoley): allocate these more intelligently (for example,
+    // pool them on the `Preprocessor`.
+    return new PreprocessorConditional();
+}
+
+// Destroy a preprocessor conditional.
+static void DestroyConditional(PreprocessorConditional* conditional)
+{
+    delete conditional;
+}
+
+// Start a preprocessor conditional, with an initial enable/disable state.
+static void beginConditional(
+    PreprocessorDirectiveContext*   context,
+    PreprocessorInputStream*        inputStream,
+    bool                            enable)
+{
+    Preprocessor* preprocessor = context->preprocessor;
+    SLANG_ASSERT(inputStream);
+
+    PreprocessorConditional* conditional = CreateConditional(preprocessor);
+
+    conditional->ifToken = context->directiveToken;
+
+    // Set state of this condition appropriately.
+    //
+    // Default to the "haven't yet seen a `true` branch" state.
+    PreprocessorConditionalState state = PreprocessorConditionalState::Before;
+    //
+    // If we are nested inside a `false` branch of another condition, then
+    // we never want to enable, so we act as if we already *saw* the `true` branch.
+    //
+    if (IsSkipping(preprocessor)) state = PreprocessorConditionalState::After;
+    //
+    // Similarly, if we ran into any parse errors when dealing with the
+    // opening directive, then things are probably screwy and we should just
+    // skip all the branches.
+    if (IsSkipping(preprocessor)) state = PreprocessorConditionalState::After;
+    //
+    // Otherwise, if our condition was true, then set us to be inside the `true` branch
+    else if (enable) state = PreprocessorConditionalState::During;
+
+    conditional->state = state;
+
+    // Push conditional onto the stack
+    auto primaryStream = inputStream->primaryStream;
+    conditional->parent = primaryStream->conditional;
+    primaryStream->conditional = conditional;
+}
+
+// Start a preprocessor conditional, with an initial enable/disable state.
+static void beginConditional(
+    PreprocessorDirectiveContext*   context,
+    bool                            enable)
+{
+    beginConditional(context, context->preprocessor->inputStream, enable);
+}
+
+//
+// Preprocessor Conditional Expressions
+//
+
+// Conditional expressions are always of type `int`
+typedef int PreprocessorExpressionValue;
+
+// Forward-declaretion
+static PreprocessorExpressionValue ParseAndEvaluateExpression(PreprocessorDirectiveContext* context);
+
+// Parse a unary (prefix) expression inside of a preprocessor directive.
+static PreprocessorExpressionValue ParseAndEvaluateUnaryExpression(PreprocessorDirectiveContext* context)
+{
+    switch (PeekTokenType(context))
+    {
+    // handle prefix unary ops
+    case TokenType::OpSub:
+        AdvanceToken(context);
+        return -ParseAndEvaluateUnaryExpression(context);
+    case TokenType::OpNot:
+        AdvanceToken(context);
+        return !ParseAndEvaluateUnaryExpression(context);
+    case TokenType::OpBitNot:
+        AdvanceToken(context);
+        return ~ParseAndEvaluateUnaryExpression(context);
+
+    // handle parenthized sub-expression
+    case TokenType::LParent:
+        {
+            Token leftParen = AdvanceToken(context);
+            PreprocessorExpressionValue value = ParseAndEvaluateExpression(context);
+            if (!Expect(context, TokenType::RParent, Diagnostics::expectedTokenInPreprocessorExpression))
+            {
+                GetSink(context)->diagnose(leftParen.loc, Diagnostics::seeOpeningToken, leftParen);
+            }
+            return value;
+        }
+
+    case TokenType::IntegerLiteral:
+        return StringToInt(AdvanceToken(context).Content);
+
+    case TokenType::Identifier:
+        {
+            Token token = AdvanceToken(context);
+            if (token.Content == "defined")
+            {
+                // handle `defined(someName)`
+
+                // Possibly parse a `(`
+                Token leftParen;
+                if (PeekRawTokenType(context) == TokenType::LParent)
+                {
+                    leftParen = AdvanceRawToken(context);
+                }
+
+                // Expect an identifier
+                Token nameToken;
+                if (!ExpectRaw(context, TokenType::Identifier, Diagnostics::expectedTokenInDefinedExpression, &nameToken))
+                {
+                    return 0;
+                }
+                Name* name = nameToken.getName();
+
+                // If we saw an opening `(`, then expect one to close
+                if (leftParen.type != TokenType::Unknown)
+                {
+                    if(!ExpectRaw(context, TokenType::RParent, Diagnostics::expectedTokenInDefinedExpression))
+                    {
+                        GetSink(context)->diagnose(leftParen.loc, Diagnostics::seeOpeningToken, leftParen);
+                        return 0;
+                    }
+                }
+
+                return LookupMacro(context, name) != NULL;
+            }
+
+            // An identifier here means it was not defined as a macro (or
+            // it is defined, but as a function-like macro. These should
+            // just evaluate to zero (possibly with a warning)
+            GetSink(context)->diagnose(token.loc, Diagnostics::undefinedIdentifierInPreprocessorExpression, token.getName());
+            return 0;
+        }
+
+    default:
+        GetSink(context)->diagnose(PeekLoc(context), Diagnostics::syntaxErrorInPreprocessorExpression);
+        return 0;
+    }
+}
+
+// Determine the precedence level of an infix operator
+// for use in parsing preprocessor conditionals.
+static int GetInfixOpPrecedence(Token const& opToken)
+{
+    // If token is on another line, it is not part of the
+    // expression
+    if (opToken.flags & TokenFlag::AtStartOfLine)
+        return -1;
+
+    // otherwise we look at the token type to figure
+    // out what precedence it should be parse with
+    switch (opToken.type)
+    {
+    default:
+        // tokens that aren't infix operators should
+        // cause us to stop parsing an expression
+        return -1;
+
+    case TokenType::OpMul:     return 10;
+    case TokenType::OpDiv:     return 10;
+    case TokenType::OpMod:     return 10;
+
+    case TokenType::OpAdd:     return 9;
+    case TokenType::OpSub:     return 9;
+
+    case TokenType::OpLsh:     return 8;
+    case TokenType::OpRsh:     return 8;
+
+    case TokenType::OpLess:    return 7;
+    case TokenType::OpGreater: return 7;
+    case TokenType::OpLeq:     return 7;
+    case TokenType::OpGeq:     return 7;
+
+    case TokenType::OpEql:     return 6;
+    case TokenType::OpNeq:     return 6;
+
+    case TokenType::OpBitAnd:  return 5;
+    case TokenType::OpBitOr:   return 4;
+    case TokenType::OpBitXor:  return 3;
+    case TokenType::OpAnd:     return 2;
+    case TokenType::OpOr:      return 1;
+    }
+};
+
+// Evaluate one infix operation in a preprocessor
+// conditional expression
+static PreprocessorExpressionValue EvaluateInfixOp(
+    PreprocessorDirectiveContext*   context,
+    Token const&                    opToken,
+    PreprocessorExpressionValue     left,
+    PreprocessorExpressionValue     right)
+{
+    switch (opToken.type)
+    {
+    default:
+//        SLANG_INTERNAL_ERROR(getSink(preprocessor), opToken);
+        return 0;
+        break;
+
+    case TokenType::OpMul:     return left * right;
+    case TokenType::OpDiv:
+    {
+        if (right == 0)
+        {
+            if (!context->parseError)
+            {
+                GetSink(context)->diagnose(opToken.loc, Diagnostics::divideByZeroInPreprocessorExpression);
+            }
+            return 0;
+        }
+        return left / right;
+    }
+    case TokenType::OpMod:
+    {
+        if (right == 0)
+        {
+            if (!context->parseError)
+            {
+                GetSink(context)->diagnose(opToken.loc, Diagnostics::divideByZeroInPreprocessorExpression);
+            }
+            return 0;
+        }
+        return left % right;
+    }
+    case TokenType::OpAdd:      return left +  right;
+    case TokenType::OpSub:      return left -  right;
+    case TokenType::OpLsh:      return left << right;
+    case TokenType::OpRsh:      return left >> right;
+    case TokenType::OpLess:     return left <  right ? 1 : 0;
+    case TokenType::OpGreater:  return left >  right ? 1 : 0;
+    case TokenType::OpLeq:      return left <= right ? 1 : 0;
+    case TokenType::OpGeq:      return left >= right ? 1 : 0;
+    case TokenType::OpEql:      return left == right ? 1 : 0;
+    case TokenType::OpNeq:      return left != right ? 1 : 0;
+    case TokenType::OpBitAnd:   return left & right;
+    case TokenType::OpBitOr:    return left | right;
+    case TokenType::OpBitXor:   return left ^ right;
+    case TokenType::OpAnd:      return left && right;
+    case TokenType::OpOr:       return left || right;
+    }
+}
+
+// Parse the rest of an infix preprocessor expression with
+// precedence greater than or equal to the given `precedence` argument.
+// The value of the left-hand-side expression is provided as
+// an argument.
+// This is used to form a simple recursive-descent expression parser.
+static PreprocessorExpressionValue ParseAndEvaluateInfixExpressionWithPrecedence(
+    PreprocessorDirectiveContext* context,
+    PreprocessorExpressionValue left,
+    int precedence)
+{
+    for (;;)
+    {
+        // Look at the next token, and see if it is an operator of
+        // high enough precedence to be included in our expression
+        Token opToken = PeekToken(context);
+        int opPrecedence = GetInfixOpPrecedence(opToken);
+
+        // If it isn't an operator of high enough precedence, we are done.
+        if(opPrecedence < precedence)
+            break;
+
+        // Otherwise we need to consume the operator token.
+        AdvanceToken(context);
+
+        // Next we parse a right-hand-side expression by starting with
+        // a unary expression and absorbing and many infix operators
+        // as possible with strictly higher precedence than the operator
+        // we found above.
+        PreprocessorExpressionValue right = ParseAndEvaluateUnaryExpression(context);
+        for (;;)
+        {
+            // Look for an operator token
+            Token rightOpToken = PeekToken(context);
+            int rightOpPrecedence = GetInfixOpPrecedence(rightOpToken);
+
+            // If no operator was found, or the operator wasn't high
+            // enough precedence to fold into the right-hand-side,
+            // exit this loop.
+            if (rightOpPrecedence <= opPrecedence)
+                break;
+
+            // Now invoke the parser recursively, passing in our
+            // existing right-hand side to form an even larger one.
+            right = ParseAndEvaluateInfixExpressionWithPrecedence(
+                context,
+                right,
+                rightOpPrecedence);
+        }
+
+        // Now combine the left- and right-hand sides using
+        // the operator we found above.
+        left = EvaluateInfixOp(context, opToken, left, right);
+    }
+    return left;
+}
+
+// Parse a complete (infix) preprocessor expression, and return its value
+static PreprocessorExpressionValue ParseAndEvaluateExpression(PreprocessorDirectiveContext* context)
+{
+    // First read in the left-hand side (or the whole expression in the unary case)
+    PreprocessorExpressionValue value = ParseAndEvaluateUnaryExpression(context);
+
+    // Try to read in trailing infix operators with correct precedence
+    return ParseAndEvaluateInfixExpressionWithPrecedence(context, value, 0);
+}
+
+// Handle a `#if` directive
+static void HandleIfDirective(PreprocessorDirectiveContext* context)
+{
+    // Record current input stream in case preprocessor expression
+    // changes the input stream to a macro expansion while we
+    // are parsing.
+    auto inputStream = context->preprocessor->inputStream;
+
+    // If we are skipping, we can just consume the expression, and assume true
+    if (IsSkipping(context->preprocessor))
+    {
+        // Consume everything until the end of the line 
+        SkipToEndOfLine(context);
+        // Begin a preprocessor block, assume true based on the expression
+        // (contents will all be ignored because skipping).
+        beginConditional(context, inputStream, true);
+    }
+    else
+    {
+        // Parse a preprocessor expression.
+        PreprocessorExpressionValue value = ParseAndEvaluateExpression(context);
+
+        // Begin a preprocessor block, enabled based on the expression.
+        beginConditional(context, inputStream, value != 0);
+    }
+}
+
+// Handle a `#ifdef` directive
+static void HandleIfDefDirective(PreprocessorDirectiveContext* context)
+{
+    // Expect a raw identifier, so we can check if it is defined
+    Token nameToken;
+    if(!ExpectRaw(context, TokenType::Identifier, Diagnostics::expectedTokenInPreprocessorDirective, &nameToken))
+        return;
+    Name* name = nameToken.getName();
+
+    // Check if the name is defined.
+    beginConditional(context, LookupMacro(context, name) != NULL);
+}
+
+// Handle a `#ifndef` directive
+static void HandleIfNDefDirective(PreprocessorDirectiveContext* context)
+{
+    // Expect a raw identifier, so we can check if it is defined
+    Token nameToken;
+    if(!ExpectRaw(context, TokenType::Identifier, Diagnostics::expectedTokenInPreprocessorDirective, &nameToken))
+        return;
+    Name* name = nameToken.getName();
+
+    // Check if the name is defined.
+    beginConditional(context, LookupMacro(context, name) == NULL);
+}
+
+// Handle a `#else` directive
+static void HandleElseDirective(PreprocessorDirectiveContext* context)
+{
+    PreprocessorInputStream* inputStream = context->preprocessor->inputStream;
+    SLANG_ASSERT(inputStream);
+
+    // if we aren't inside a conditional, then error
+    PreprocessorConditional* conditional = inputStream->primaryStream->conditional;
+    if (!conditional)
+    {
+        GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::directiveWithoutIf, GetDirectiveName(context));
+        return;
+    }
+
+    // if we've already seen a `#else`, then it is an error
+    if (conditional->elseToken.type != TokenType::Unknown)
+    {
+        GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::directiveAfterElse, GetDirectiveName(context));
+        GetSink(context)->diagnose(conditional->elseToken.loc, Diagnostics::seeDirective);
+        return;
+    }
+    conditional->elseToken = context->directiveToken;
+
+    switch (conditional->state)
+    {
+    case PreprocessorConditionalState::Before:
+        conditional->state = PreprocessorConditionalState::During;
+        break;
+
+    case PreprocessorConditionalState::During:
+        conditional->state = PreprocessorConditionalState::After;
+        break;
+
+    default:
+        break;
+    }
+}
+
+// Handle a `#elif` directive
+static void HandleElifDirective(PreprocessorDirectiveContext* context)
+{
+    // Need to grab current input stream *before* we try to parse
+    // the conditional expression.
+    PreprocessorInputStream* inputStream = context->preprocessor->inputStream;
+    SLANG_ASSERT(inputStream);
+
+    // HACK(tfoley): handle an empty `elif` like an `else` directive
+    //
+    // This is the behavior expected by at least one input program.
+    // We will eventually want to be pedantic about this.
+    // even if t
+    if (PeekRawTokenType(context) == TokenType::EndOfDirective)
+    {
+        GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::directiveExpectsExpression, GetDirectiveName(context));
+        HandleElseDirective(context);
+        return;
+    }
+
+    PreprocessorExpressionValue value = ParseAndEvaluateExpression(context);
+
+    // if we aren't inside a conditional, then error
+    PreprocessorConditional* conditional = inputStream->primaryStream->conditional;
+    if (!conditional)
+    {
+        GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::directiveWithoutIf, GetDirectiveName(context));
+        return;
+    }
+
+    // if we've already seen a `#else`, then it is an error
+    if (conditional->elseToken.type != TokenType::Unknown)
+    {
+        GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::directiveAfterElse, GetDirectiveName(context));
+        GetSink(context)->diagnose(conditional->elseToken.loc, Diagnostics::seeDirective);
+        return;
+    }
+
+    switch (conditional->state)
+    {
+    case PreprocessorConditionalState::Before:
+        if(value)
+            conditional->state = PreprocessorConditionalState::During;
+        break;
+
+    case PreprocessorConditionalState::During:
+        conditional->state = PreprocessorConditionalState::After;
+        break;
+
+    default:
+        break;
+    }
+}
+
+// Handle a `#endif` directive
+static void HandleEndIfDirective(PreprocessorDirectiveContext* context)
+{
+    PreprocessorInputStream* inputStream = context->preprocessor->inputStream;
+    SLANG_ASSERT(inputStream);
+
+    // if we aren't inside a conditional, then error
+    PreprocessorConditional* conditional = inputStream->primaryStream->conditional;
+    if (!conditional)
+    {
+        GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::directiveWithoutIf, GetDirectiveName(context));
+        return;
+    }
+
+    inputStream->primaryStream->conditional = conditional->parent;
+    DestroyConditional(conditional);
+}
+
+// Helper routine to check that we find the end of a directive where
+// we expect it.
+//
+// Most directives do not need to call this directly, since we have
+// a catch-all case in the main `HandleDirective()` function.
+// The `#include` case will call it directly to avoid complications
+// when it switches the input stream.
+static void expectEndOfDirective(PreprocessorDirectiveContext* context)
+{
+    if(context->haveDoneEndOfDirectiveChecks)
+        return;
+
+    context->haveDoneEndOfDirectiveChecks = true;
+
+    if (!IsEndOfLine(context))
+    {
+        // If we already saw a previous parse error, then don't
+        // emit another one for the same directive.
+        if (!context->parseError)
+        {
+            GetSink(context)->diagnose(PeekLoc(context), Diagnostics::unexpectedTokensAfterDirective, GetDirectiveName(context));
+        }
+        SkipToEndOfLine(context);
+    }
+
+    // Clear out the end-of-directive token
+    AdvanceRawToken(context->preprocessor);
+}
+
+    /// Read a file in the context of handling a preprocessor directive
+static SlangResult readFile(
+    PreprocessorDirectiveContext*   context,
+    String const&                   path,
+    ISlangBlob**                    outBlob)
+{
+    // The actual file loading will be handled by the file system
+    // associated with the parent linkage.
+    //
+    auto linkage = context->preprocessor->linkage;
+    auto fileSystemExt = linkage->getFileSystemExt();
+    SLANG_RETURN_ON_FAIL(fileSystemExt->loadFile(path.getBuffer(), outBlob));
+
+    // If we are running the preprocessor as part of compiling a
+    // specific module, then we must keep track of the file we've
+    // read as yet another file that the module will depend on.
+    //
+    if(auto module = context->preprocessor->parentModule)
+    {
+        module->addFilePathDependency(path);
+    }
+
+    return SLANG_OK;
+}
+
+// Handle a `#include` directive
+static void HandleIncludeDirective(PreprocessorDirectiveContext* context)
+{
+    // Consume the directive, and inform the lexer to process the remainder of the line as a file path.
+    AdvanceRawToken(context, kLexerFlag_ExpectFileName);
+
+    Token pathToken;
+    if(!Expect(context, TokenType::StringLiteral, Diagnostics::expectedTokenInPreprocessorDirective, &pathToken))
+        return;
+
+    String path = getFileNameTokenValue(pathToken);
+
+    auto directiveLoc = GetDirectiveLoc(context);
+    
+    PathInfo includedFromPathInfo = context->preprocessor->getSourceManager()->getPathInfo(directiveLoc, SourceLocType::Actual);
+    
+    IncludeHandler* includeHandler = context->preprocessor->includeHandler;
+    if (!includeHandler)
+    {
+        GetSink(context)->diagnose(pathToken.loc, Diagnostics::includeFailed, path);
+        GetSink(context)->diagnose(pathToken.loc, Diagnostics::noIncludeHandlerSpecified);
+        return;
+    }
+    
+    /* Find the path relative to the foundPath */
+    PathInfo filePathInfo;
+    if (SLANG_FAILED(includeHandler->findFile(path, includedFromPathInfo.foundPath, filePathInfo)))
+    {
+        GetSink(context)->diagnose(pathToken.loc, Diagnostics::includeFailed, path);
+        return;
+    }
+
+    // We must have a uniqueIdentity to be compare
+    if (!filePathInfo.hasUniqueIdentity())
+    {
+        GetSink(context)->diagnose(pathToken.loc, Diagnostics::noUniqueIdentity, path);
+        return;
+    }
+
+    // Do all checking related to the end of this directive before we push a new stream,
+    // just to avoid complications where that check would need to deal with
+    // a switch of input stream
+    expectEndOfDirective(context);
+
+    // Check whether we've previously included this file and seen a `#pragma once` directive
+    if(context->preprocessor->pragmaOnceUniqueIdentities.Contains(filePathInfo.uniqueIdentity))
+    {
+        return;
+    }
+
+    // Simplify the path
+    filePathInfo.foundPath = includeHandler->simplifyPath(filePathInfo.foundPath);
+
+    // Push the new file onto our stack of input streams
+    // TODO(tfoley): check if we have made our include stack too deep
+    auto sourceManager = context->preprocessor->getSourceManager();
+
+    // See if this an already loaded source file
+    SourceFile* sourceFile = sourceManager->findSourceFileRecursively(filePathInfo.uniqueIdentity);
+    // If not create a new one, and add to the list of known source files
+    if (!sourceFile)
+    {
+        ComPtr<ISlangBlob> foundSourceBlob;
+        if (SLANG_FAILED(readFile(context, filePathInfo.foundPath, foundSourceBlob.writeRef())))
+        {
+            GetSink(context)->diagnose(pathToken.loc, Diagnostics::includeFailed, path);
+            return;
+        }
+
+        
+        sourceFile = sourceManager->createSourceFileWithBlob(filePathInfo, foundSourceBlob);
+        sourceManager->addSourceFile(filePathInfo.uniqueIdentity, sourceFile);
+    }
+
+    // This is a new parse (even if it's a pre-existing source file), so create a new SourceUnit
+    SourceView* sourceView = sourceManager->createSourceView(sourceFile, &filePathInfo);
+
+    PreprocessorInputStream* inputStream = CreateInputStreamForSource(context->preprocessor, sourceView);
+    inputStream->parent = context->preprocessor->inputStream;
+    context->preprocessor->inputStream = inputStream;
+}
+
+// Handle a `#define` directive
+static void HandleDefineDirective(PreprocessorDirectiveContext* context)
+{
+    Token nameToken;
+    if (!ExpectRaw(context, TokenType::Identifier, Diagnostics::expectedTokenInPreprocessorDirective, &nameToken))
+        return;
+    Name* name = nameToken.getName();
+
+    PreprocessorMacro* macro = CreateMacro(context->preprocessor);
+    macro->nameAndLoc = NameLoc(nameToken);
+
+    PreprocessorMacro* oldMacro = LookupMacro(&context->preprocessor->globalEnv, name);
+    if (oldMacro)
+    {
+        GetSink(context)->diagnose(nameToken.loc, Diagnostics::macroRedefinition, name);
+        GetSink(context)->diagnose(oldMacro->getLoc(), Diagnostics::seePreviousDefinitionOf, name);
+
+        DestroyMacro(context->preprocessor, oldMacro);
+    }
+    context->preprocessor->globalEnv.macros[name] = macro;
+
+    // If macro name is immediately followed (with no space) by `(`,
+    // then we have a function-like macro
+    if (PeekRawTokenType(context) == TokenType::LParent)
+    {
+        if (!(PeekRawToken(context).flags & TokenFlag::AfterWhitespace))
+        {
+            // This is a function-like macro, so we need to remember that
+            // and start capturing parameters
+            macro->flavor = PreprocessorMacroFlavor::FunctionLike;
+
+            AdvanceRawToken(context);
+
+            // If there are any parameters, parse them
+            if (PeekRawTokenType(context) != TokenType::RParent)
+            {
+                for (;;)
+                {
+                    // TODO: handle elipsis (`...`) for varags
+
+                    // A macro parameter name should be a raw identifier
+                    Token paramToken;
+                    if (!ExpectRaw(context, TokenType::Identifier, Diagnostics::expectedTokenInMacroParameters, &paramToken))
+                        break;
+
+                    // TODO(tfoley): some validation on parameter name.
+                    // Certain names (e.g., `defined` and `__VA_ARGS__`
+                    // are not allowed to be used as macros or parameters).
+
+                    // Add the parameter to the macro being deifned
+                    macro->params.add(paramToken);
+
+                    // If we see `)` then we are done with arguments
+                    if (PeekRawTokenType(context) == TokenType::RParent)
+                        break;
+
+                    ExpectRaw(context, TokenType::Comma, Diagnostics::expectedTokenInMacroParameters);
+                }
+            }
+
+            ExpectRaw(context, TokenType::RParent, Diagnostics::expectedTokenInMacroParameters);
+        }
+    }
+
+    // consume tokens until end-of-line
+    for(;;)
+    {
+        Token token = AdvanceRawToken(context);
+        if( token.type == TokenType::EndOfDirective )
+        {
+            // Last token on line will be turned into a conceptual end-of-file
+            // token for the sub-stream that the macro expands into.
+            token.type = TokenType::EndOfFile;
+            macro->tokens.mTokens.add(token);
+            break;
+        }
+
+        // In the ordinary case, we just add the token to the definition
+        macro->tokens.mTokens.add(token);
+    }
+}
+
+// Handle a `#undef` directive
+static void HandleUndefDirective(PreprocessorDirectiveContext* context)
+{
+    Token nameToken;
+    if (!ExpectRaw(context, TokenType::Identifier, Diagnostics::expectedTokenInPreprocessorDirective, &nameToken))
+        return;
+    Name* name = nameToken.getName();
+
+    PreprocessorEnvironment* env = &context->preprocessor->globalEnv;
+    PreprocessorMacro* macro = LookupMacro(env, name);
+    if (macro != NULL)
+    {
+        // name was defined, so remove it
+        env->macros.Remove(name);
+
+        DestroyMacro(context->preprocessor, macro);
+    }
+    else
+    {
+        // name wasn't defined
+        GetSink(context)->diagnose(nameToken.loc, Diagnostics::macroNotDefined, name);
+    }
+}
+
+// Handle a `#warning` directive
+static void HandleWarningDirective(PreprocessorDirectiveContext* context)
+{
+    // Consume the directive, and inform the lexer to process the remainder of the line as a custom message.
+    AdvanceRawToken(context, kLexerFlag_ExpectDirectiveMessage);
+
+    // Read the message token.
+    Token messageToken;
+    Expect(context, TokenType::DirectiveMessage, Diagnostics::expectedTokenInPreprocessorDirective, &messageToken);
+
+    // Report the custom error.
+    GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::userDefinedWarning, messageToken.Content);
+}
+
+// Handle a `#error` directive
+static void HandleErrorDirective(PreprocessorDirectiveContext* context)
+{
+    // Consume the directive, and inform the lexer to process the remainder of the line as a custom message.
+    AdvanceRawToken(context, kLexerFlag_ExpectDirectiveMessage);
+
+    // Read the message token.
+    Token messageToken;
+    Expect(context, TokenType::DirectiveMessage, Diagnostics::expectedTokenInPreprocessorDirective, &messageToken);
+
+    // Report the custom error.
+    GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::userDefinedError, messageToken.Content);
+}
+
+// Handle a `#line` directive
+static void HandleLineDirective(PreprocessorDirectiveContext* context)
+{
+    auto inputStream = context->preprocessor->inputStream;
+
+    int line = 0;
+
+    SourceLoc directiveLoc = GetDirectiveLoc(context);
+
+    // `#line <integer-literal> ...`
+    if (PeekTokenType(context) == TokenType::IntegerLiteral)
+    {
+        line = StringToInt(AdvanceToken(context).Content);
+    }
+    // `#line`
+    // `#line default`
+    else if (
+        PeekTokenType(context) == TokenType::EndOfDirective
+        || (PeekTokenType(context) == TokenType::Identifier
+            && PeekToken(context).Content == "default"))
+    {
+        AdvanceToken(context);
+
+        // Stop overriding source locations.
+        auto sourceView = inputStream->primaryStream->lexer.sourceView;
+        sourceView->addDefaultLineDirective(directiveLoc);
+        return;
+    }
+    else
+    {
+        GetSink(context)->diagnose(PeekLoc(context), Diagnostics::expected2TokensInPreprocessorDirective,
+            TokenType::IntegerLiteral,
+            "default",
+            GetDirectiveName(context));
+        context->parseError = true;
+        return;
+    }
+
+    auto sourceManager = context->preprocessor->getSourceManager();
+    
+    String file;
+    if (PeekTokenType(context) == TokenType::EndOfDirective)
+    {
+        file = sourceManager->getPathInfo(directiveLoc).foundPath;
+    }
+    else if (PeekTokenType(context) == TokenType::StringLiteral)
+    {
+        file = getStringLiteralTokenValue(AdvanceToken(context));
+    }
+    else if (PeekTokenType(context) == TokenType::IntegerLiteral)
+    {
+        // Note(tfoley): GLSL allows the "source string" to be indicated by an integer
+        // TODO(tfoley): Figure out a better way to handle this, if it matters
+        file = AdvanceToken(context).Content;
+    }
+    else
+    {
+        Expect(context, TokenType::StringLiteral, Diagnostics::expectedTokenInPreprocessorDirective);
+        return;
+    }
+
+    auto sourceView = inputStream->primaryStream->lexer.sourceView;
+    sourceView->addLineDirective(directiveLoc, file, line);
+}
+
+#define SLANG_PRAGMA_DIRECTIVE_CALLBACK(NAME) \
+    void NAME(PreprocessorDirectiveContext* context, Token subDirectiveToken)
+
+// Callback interface used by `#pragma` directives
+typedef SLANG_PRAGMA_DIRECTIVE_CALLBACK((*PragmaDirectiveCallback));
+
+SLANG_PRAGMA_DIRECTIVE_CALLBACK(handleUnknownPragmaDirective)
+{
+    GetSink(context)->diagnose(subDirectiveToken, Diagnostics::unknownPragmaDirectiveIgnored, subDirectiveToken.getName());
+    SkipToEndOfLine(context);
+    return;
+}
+
+SLANG_PRAGMA_DIRECTIVE_CALLBACK(handlePragmaOnceDirective)
+{
+    // We need to identify the path of the file we are preprocessing,
+    // so that we can avoid including it again.
+    //
+    // We are using the 'uniqueIdentity' as determined by the ISlangFileSystemEx interface to determine file identities.
+    
+    auto directiveLoc = GetDirectiveLoc(context);
+    auto issuedFromPathInfo = context->preprocessor->getSourceManager()->getPathInfo(directiveLoc, SourceLocType::Actual);
+
+    // Must have uniqueIdentity for a #pragma once to work
+    if (!issuedFromPathInfo.hasUniqueIdentity())
+    {
+        GetSink(context)->diagnose(subDirectiveToken, Diagnostics::pragmaOnceIgnored);
+        return;
+    }
+
+    context->preprocessor->pragmaOnceUniqueIdentities.Add(issuedFromPathInfo.uniqueIdentity);
+}
+
+// Information about a specific `#pragma` directive
+struct PragmaDirective
+{
+    // name of the directive
+    char const*             name;
+
+    // Callback to handle the directive
+    PragmaDirectiveCallback callback;
+};
+
+// A simple array of all the  `#pragma` directives we know how to handle.
+static const PragmaDirective kPragmaDirectives[] =
+{
+    { "once", &handlePragmaOnceDirective },
+
+    { NULL, NULL },
+};
+
+static const PragmaDirective kUnknownPragmaDirective = {
+    NULL, &handleUnknownPragmaDirective,
+};
+
+// Look up the `#pragma` directive with the given name.
+static PragmaDirective const* findPragmaDirective(String const& name)
+{
+    char const* nameStr = name.getBuffer();
+    for (int ii = 0; kPragmaDirectives[ii].name; ++ii)
+    {
+        if (strcmp(kPragmaDirectives[ii].name, nameStr) != 0)
+            continue;
+
+        return &kPragmaDirectives[ii];
+    }
+
+    return &kUnknownPragmaDirective;
+}
+
+// Handle a `#pragma` directive
+static void HandlePragmaDirective(PreprocessorDirectiveContext* context)
+{
+    // Try to read the sub-directive name.
+    Token subDirectiveToken = PeekRawToken(context);
+
+    // The sub-directive had better be an identifier
+    if (subDirectiveToken.type != TokenType::Identifier)
+    {
+        GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::expectedPragmaDirectiveName);
+        SkipToEndOfLine(context);
+        return;
+    }
+    AdvanceRawToken(context);
+
+    // Look up the handler for the sub-directive.
+    PragmaDirective const* subDirective = findPragmaDirective(subDirectiveToken.getName()->text);
+
+    // Apply the sub-directive-specific callback
+    (subDirective->callback)(context, subDirectiveToken);
+}
+
+// Handle an invalid directive
+static void HandleInvalidDirective(PreprocessorDirectiveContext* context)
+{
+    GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::unknownPreprocessorDirective, GetDirectiveName(context));
+    SkipToEndOfLine(context);
+}
+
+// Callback interface used by preprocessor directives
+typedef void (*PreprocessorDirectiveCallback)(PreprocessorDirectiveContext* context);
+
+enum PreprocessorDirectiveFlag : unsigned int
+{
+    // Should this directive be handled even when skipping disbaled code?
+    ProcessWhenSkipping = 1 << 0,
+
+    /// Allow the handler for this directive to advance past the
+    /// directive token itself, so that it can control lexer behavior
+    /// more closely.
+    DontConsumeDirectiveAutomatically = 1 << 1,
+};
+
+// Information about a specific directive
+struct PreprocessorDirective
+{
+    // name of the directive
+    char const*                     name;
+
+    // Callback to handle the directive
+    PreprocessorDirectiveCallback   callback;
+
+    unsigned int                    flags;
+};
+
+// A simple array of all the directives we know how to handle.
+// TODO(tfoley): considering making this into a real hash map,
+// and then make it easy-ish for users of the codebase to add
+// their own directives as desired.
+static const PreprocessorDirective kDirectives[] =
+{
+    { "if",         &HandleIfDirective,         ProcessWhenSkipping },
+    { "ifdef",      &HandleIfDefDirective,      ProcessWhenSkipping },
+    { "ifndef",     &HandleIfNDefDirective,     ProcessWhenSkipping },
+    { "else",       &HandleElseDirective,       ProcessWhenSkipping },
+    { "elif",       &HandleElifDirective,       ProcessWhenSkipping },
+    { "endif",      &HandleEndIfDirective,      ProcessWhenSkipping },
+
+    { "include",    &HandleIncludeDirective,    DontConsumeDirectiveAutomatically },
+    { "define",     &HandleDefineDirective,     0 },
+    { "undef",      &HandleUndefDirective,      0 },
+    { "warning",    &HandleWarningDirective,    DontConsumeDirectiveAutomatically },
+    { "error",      &HandleErrorDirective,      DontConsumeDirectiveAutomatically },
+    { "line",       &HandleLineDirective,       0 },
+    { "pragma",     &HandlePragmaDirective,     0 },
+
+    { nullptr, nullptr, 0 },
+};
+
+static const PreprocessorDirective kInvalidDirective = {
+    nullptr, &HandleInvalidDirective, 0,
+};
+
+// Look up the directive with the given name.
+static PreprocessorDirective const* FindDirective(String const& name)
+{
+    char const* nameStr = name.getBuffer();
+    for (int ii = 0; kDirectives[ii].name; ++ii)
+    {
+        if (strcmp(kDirectives[ii].name, nameStr) != 0)
+            continue;
+
+        return &kDirectives[ii];
+    }
+
+    return &kInvalidDirective;
+}
+
+// Process a directive, where the preprocessor has already consumed the
+// `#` token that started the directive line.
+static void HandleDirective(PreprocessorDirectiveContext* context)
+{
+    // Try to read the directive name.
+    context->directiveToken = PeekRawToken(context);
+
+    TokenType directiveTokenType = GetDirective(context).type;
+
+    // An empty directive is allowed, and ignored.
+    if (directiveTokenType == TokenType::EndOfDirective)
+    {
+        return;
+    }
+    // Otherwise the directive name had better be an identifier
+    else if (directiveTokenType != TokenType::Identifier)
+    {
+        GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::expectedPreprocessorDirectiveName);
+        SkipToEndOfLine(context);
+        return;
+    }
+
+    // Look up the handler for the directive.
+    PreprocessorDirective const* directive = FindDirective(GetDirectiveName(context));
+
+    // If we are skipping disabled code, and the directive is not one
+    // of the small number that need to run even in that case, skip it.
+    if (IsSkipping(context) && !(directive->flags & PreprocessorDirectiveFlag::ProcessWhenSkipping))
+    {
+        SkipToEndOfLine(context);
+        return;
+    }
+
+    if(!(directive->flags & PreprocessorDirectiveFlag::DontConsumeDirectiveAutomatically))
+    {
+        // Consume the directive name token.
+        AdvanceRawToken(context);
+    }
+
+    // Apply the directive-specific callback
+    (directive->callback)(context);
+
+    // We expect the directive callback to consume the entire line, so if
+    // it hasn't that is a parse error.
+    expectEndOfDirective(context);
+}
+
+// Read one token using the full preprocessor, with all its behaviors.
+static Token ReadToken(Preprocessor* preprocessor)
+{
+    for (;;)
+    {
+        // Depending on what the lookahead token is, we
+        // might need to start expanding it.
+        //
+        // Note: doing this at the start of this loop
+        // is important, in case a macro has an empty
+        // expansion, and we end up looking at a different
+        // token after applying the expansion.
+        if(!IsSkipping(preprocessor))
+        {
+            MaybeBeginMacroExpansion(preprocessor);
+        }
+
+        // Look at the next raw token in the input.
+        Token const& token = PeekRawToken(preprocessor);
+        if (token.type == TokenType::EndOfFile)
+            return token;
+
+        // If we have a directive (`#` at start of line) then handle it
+        if ((token.type == TokenType::Pound) && (token.flags & TokenFlag::AtStartOfLine))
+        {
+            // Skip the `#`
+            AdvanceRawToken(preprocessor);
+
+            // Create a context for parsing the directive
+            PreprocessorDirectiveContext directiveContext;
+            directiveContext.preprocessor = preprocessor;
+            directiveContext.parseError = false;
+            directiveContext.haveDoneEndOfDirectiveChecks = false;
+
+            // Parse and handle the directive
+            HandleDirective(&directiveContext);
+            continue;
+        }
+
+        // otherwise, if we are currently in a skipping mode, then skip tokens
+        if (IsSkipping(preprocessor))
+        {
+            AdvanceRawToken(preprocessor);
+            continue;
+        }
+
+        // otherwise read a token, which may involve macro expansion
+        return AdvanceToken(preprocessor);
+    }
+}
+
+// intialize a preprocessor context, using the given sink for errros
+static void InitializePreprocessor(
+    Preprocessor*   preprocessor,
+    DiagnosticSink* sink)
+{
+    preprocessor->sink = sink;
+    preprocessor->includeHandler = NULL;
+    preprocessor->endOfFileToken.type = TokenType::EndOfFile;
+    preprocessor->endOfFileToken.flags = TokenFlag::AtStartOfLine;
+}
+
+// clean up after an environment
+PreprocessorEnvironment::~PreprocessorEnvironment()
+{
+    for (auto pair : this->macros)
+    {
+        DestroyMacro(NULL, pair.Value);
+    }
+}
+
+// finalize a preprocessor and free any memory still in use
+static void FinalizePreprocessor(
+    Preprocessor*   preprocessor)
+{
+    // Clear out any waiting input streams
+    PreprocessorInputStream* input = preprocessor->inputStream;
+    while (input)
+    {
+        PreprocessorInputStream* parent = input->parent;
+        EndInputStream(preprocessor, input);
+        input = parent;
+    }
+
+#if 0
+    // clean up any macros that were allocated
+    for (auto pair : preprocessor->globalEnv.macros)
+    {
+        DestroyMacro(preprocessor, pair.Value);
+    }
+#endif
+}
+
+// Add a simple macro definition from a string (e.g., for a
+// `-D` option passed on the command line
+static void DefineMacro(
+    Preprocessor*   preprocessor,
+    String const&   key,
+    String const&   value)
+{
+    PathInfo pathInfo = PathInfo::makeCommandLine();
+    
+    PreprocessorMacro* macro = CreateMacro(preprocessor);
+
+    auto sourceManager = preprocessor->getSourceManager();
+
+    SourceFile* keyFile = sourceManager->createSourceFileWithString(pathInfo, key);
+    SourceFile* valueFile = sourceManager->createSourceFileWithString(pathInfo, value);
+
+    SourceView* keyView = sourceManager->createSourceView(keyFile, nullptr);
+    SourceView* valueView = sourceManager->createSourceView(valueFile, nullptr);
+
+    // Use existing `Lexer` to generate a token stream.
+    Lexer lexer;
+    lexer.initialize(valueView, GetSink(preprocessor), preprocessor->getNamePool(), sourceManager->getMemoryArena());
+    macro->tokens = lexer.lexAllTokens();
+
+    Name* keyName = preprocessor->getNamePool()->getName(key);
+
+    macro->nameAndLoc.name = keyName;
+    macro->nameAndLoc.loc = keyView->getRange().begin;
+    
+    PreprocessorMacro* oldMacro = NULL;
+    if (preprocessor->globalEnv.macros.TryGetValue(keyName, oldMacro))
+    {
+        DestroyMacro(preprocessor, oldMacro);
+    }
+
+    preprocessor->globalEnv.macros[keyName] = macro;
+}
+
+// read the entire input into tokens
+static TokenList ReadAllTokens(
+    Preprocessor*   preprocessor)
+{
+    TokenList tokens;
+    for (;;)
+    {
+        Token token = ReadToken(preprocessor);
+
+        tokens.mTokens.add(token);
+
+        // Note: we include the EOF token in the list,
+        // since that is expected by the `TokenList` type.
+        if (token.type == TokenType::EndOfFile)
+            break;
+    }
+    return tokens;
+}
+
+TokenList preprocessSource(
+    SourceFile*                 file,
+    DiagnosticSink*             sink,
+    IncludeHandler*             includeHandler,
+    Dictionary<String, String>  defines,
+    Linkage*                    linkage,
+    Module*                     parentModule)
+{
+    Preprocessor preprocessor;
+    InitializePreprocessor(&preprocessor, sink);
+    preprocessor.linkage = linkage;
+    preprocessor.parentModule = parentModule;
+
+    preprocessor.includeHandler = includeHandler;
+    for (auto p : defines)
+    {
+        DefineMacro(&preprocessor, p.Key, p.Value);
+    }
+
+    SourceManager* sourceManager = linkage->getSourceManager();
+
+    SourceView* sourceView = sourceManager->createSourceView(file, nullptr);
+
+    // create an initial input stream based on the provided buffer
+    preprocessor.inputStream = CreateInputStreamForSource(&preprocessor, sourceView);
+
+    TokenList tokens = ReadAllTokens(&preprocessor);
+
+    FinalizePreprocessor(&preprocessor);
+
+    // debugging: build the pre-processed source back together
+#if 0
+    StringBuilder sb;
+    for (auto t : tokens)
+    {
+        if (t.flags & TokenFlag::AtStartOfLine)
+        {
+            sb << "\n";
+        }
+        else if (t.flags & TokenFlag::AfterWhitespace)
+        {
+            sb << " ";
+        }
+
+        sb << t.Content;
+    }
+
+    String s = sb.ProduceString();
+#endif
+
+    return tokens;
+}
+
+}