Make source location lightweight

Fixes #24

So far the code has used a representation for source locations that is heavy-weight, but typical of research or hobby compilers: a `struct` type containing a line number and a (heap-allocated) string.
This is actually very convenient for debugging, but it means that any data structure that might contain a source location needs careful memory management (because of those strings) and has a tendency to bloat.

The new represnetation is that a source location is just a pointer-sized integer.
In the simplest mental model, you can think of this as just counting every byte of source text that is passed in, and using those to name locations.

Finding the path and line number that corresponds to a location involves a lookup step, but we can arrange to store all the files in an array sorted by their start locations, and do a binary search.
Finding line numbers inside a file is similarly fast (one you pay a one-time cost to build an array of starting offsets for lines).

More advanced compilers like clang actually go further and create a unique range of source locations to represent a file each time it gets included, so that they can track the include stack and reproduce it in diagnostic messages.
I'm not doing anything that clever here.

1 files changed, 228 insertions, 155 deletions

diff --git a/source/slang/preprocessor.cpp b/source/slang/preprocessor.cpp
index 53f0c4774..9c9a340ae 100644
--- a/source/slang/preprocessor.cpp
+++ b/source/slang/preprocessor.cpp
@@ -60,45 +60,61 @@ struct PreprocessorEnvironment
 // In general, input streams can be nested, so we have to keep a conceptual
 // stack of input.
 
+struct PrimaryInputStream;
+
 // A stream of input tokens to be consumed
 struct PreprocessorInputStream
 {
+    // The primary input stream that is the parent to this one,
+    // or NULL if this stream is itself a primary stream.
+    PrimaryInputStream*             primaryStream;
+
     // The next input stream up the stack, if any.
     PreprocessorInputStream*        parent;
 
-    // The deepest preprocessor conditional active for this stream.
-    PreprocessorConditional*        conditional;
-
     // Environment to use when looking up macros
     PreprocessorEnvironment*        environment;
 
-    // Reader for pre-tokenized input
-    TokenReader                     tokenReader;
+    // Destructor is virtual so that we can clean up
+    // after concrete subtypes.
+    virtual ~PreprocessorInputStream() = default;
+};
 
-    // If we are clobbering source locations with `#line`, then
-    // the state is tracked here:
+// A "primary" input stream represents the top-level context of a file
+// being parsed, and tracks things like preprocessor conditional state
+struct PrimaryInputStream : PreprocessorInputStream
+{
+    // The next *primary* input stream up the stack
+    PrimaryInputStream*             parentPrimaryInputStream;
 
-    // Are we overriding source locations?
-    bool                            isOverridingSourceLoc;
+    // The deepest preprocessor conditional active for this stream.
+    PreprocessorConditional*        conditional;
 
-    // What is the file name we are overriding to?
-    String                          overrideFileName;
+    // The lexer state that will provide input
+    Lexer lexer;
 
-    // What is the relative offset to apply to any line numbers?
-    int                             overrideLineOffset;
+    // One token of lookahead
+    Token token;
+};
 
-    // Destructor is virtual so that we can clean up
-    // after concrete subtypes.
-    virtual ~PreprocessorInputStream() = default;
+// A "secondary" input stream represents code that is being expanded
+// into the current scope, but which had already been tokenized before.
+//
+struct PretokenizedInputStream : PreprocessorInputStream
+{
+    // Reader for pre-tokenized input
+    TokenReader                     tokenReader;
 };
 
-struct SourceTextInputStream : PreprocessorInputStream
+// A pre-tokenized input stream that will only be used once, and which
+// therefore owns the memory for its tokens.
+struct SimpleTokenInputStream : PretokenizedInputStream
 {
-    // The pre-tokenized input
-    TokenList           lexedTokens;
+    // A list of raw tokens that will provide input
+    TokenList lexedTokens;
 };
 
-struct MacroExpansion : PreprocessorInputStream
+struct MacroExpansion : PretokenizedInputStream
 {
     // The macro we will expand
     PreprocessorMacro*  macro;
@@ -204,39 +220,55 @@ static void DestroyMacro(Preprocessor* preprocessor, PreprocessorMacro* macro);
 //
 
 // Create a fresh input stream
-static void  InitializeInputStream(Preprocessor* preprocessor, PreprocessorInputStream* inputStream)
+static void  initializeInputStream(Preprocessor* preprocessor, PreprocessorInputStream* inputStream)
 {
     inputStream->parent = NULL;
-    inputStream->conditional = NULL;
     inputStream->environment = &preprocessor->globalEnv;
 }
 
+static void initializePrimaryInputStream(Preprocessor* preprocessor, PrimaryInputStream* inputStream)
+{
+    initializeInputStream(preprocessor, inputStream);
+    inputStream->primaryStream = inputStream;
+    inputStream->conditional = NULL;
+}
+
 // Destroy an input stream
-static void DestroyInputStream(Preprocessor* /*preprocessor*/, PreprocessorInputStream* inputStream)
+static void destroyInputStream(Preprocessor* /*preprocessor*/, PreprocessorInputStream* inputStream)
 {
     delete inputStream;
 }
 
 // Create an input stream to represent a pre-tokenized input file.
 // TODO(tfoley): pre-tokenizing files isn't going to work in the long run.
-static PreprocessorInputStream* CreateInputStreamForSource(Preprocessor* preprocessor, String const& source, String const& fileName)
+static PreprocessorInputStream* CreateInputStreamForSource(
+    Preprocessor*   preprocessor,
+    SourceFile*     sourceFile)
 {
-    SourceTextInputStream* inputStream = new SourceTextInputStream();
-    InitializeInputStream(preprocessor, inputStream);
+    PrimaryInputStream* inputStream = new PrimaryInputStream();
+    initializePrimaryInputStream(preprocessor, inputStream);
 
-    // Use existing `Lexer` to generate a token stream.
-    Lexer lexer(fileName, source, GetSink(preprocessor));
-    inputStream->lexedTokens = lexer.lexAllTokens();
-    inputStream->tokenReader = TokenReader(inputStream->lexedTokens);
+    // initialize the embedded lexer so that it can generate a token stream
+    inputStream->lexer.initialize(sourceFile, GetSink(preprocessor));
+    inputStream->token = inputStream->lexer.lexToken();
 
     return inputStream;
 }
 
+static PrimaryInputStream* asPrimaryInputStream(PreprocessorInputStream* inputStream)
+{
+    auto primaryStream = inputStream->primaryStream;
+    if(primaryStream == inputStream)
+        return primaryStream;
+    return nullptr;
+}
 
 
 static void PushInputStream(Preprocessor* preprocessor, PreprocessorInputStream* inputStream)
 {
     inputStream->parent = preprocessor->inputStream;
+    if(!asPrimaryInputStream(inputStream))
+        inputStream->primaryStream = preprocessor->inputStream->primaryStream;
     preprocessor->inputStream = inputStream;
 }
 
@@ -244,52 +276,69 @@ static void PushInputStream(Preprocessor* preprocessor, PreprocessorInputStream*
 // Performs some validation and then destroys the input stream if required.
 static void EndInputStream(Preprocessor* preprocessor, PreprocessorInputStream* inputStream)
 {
-    // If there are any conditionals that weren't completed, then it is an error
-    if (inputStream->conditional)
+    if(auto primaryStream = asPrimaryInputStream(inputStream))
     {
-        PreprocessorConditional* conditional = inputStream->conditional;
+        // If there are any conditionals that weren't completed, then it is an error
+        if (primaryStream->conditional)
+        {
+            PreprocessorConditional* conditional = primaryStream->conditional;
 
-        GetSink(preprocessor)->diagnose(conditional->ifToken.Position, Diagnostics::endOfFileInPreprocessorConditional);
+            GetSink(preprocessor)->diagnose(conditional->ifToken.Position, Diagnostics::endOfFileInPreprocessorConditional);
 
-        while (conditional)
-        {
-            PreprocessorConditional* parent = conditional->parent;
-            DestroyConditional(conditional);
-            conditional = parent;
+            while (conditional)
+            {
+                PreprocessorConditional* parent = conditional->parent;
+                DestroyConditional(conditional);
+                conditional = parent;
+            }
         }
     }
 
-    DestroyInputStream(preprocessor, inputStream);
-}
-
-// Potentially clobber source location information based on `#line`
-static Token PossiblyOverrideSourceLoc(PreprocessorInputStream* inputStream, Token const& token)
-{
-    Token result = token;
-    if( inputStream->isOverridingSourceLoc )
-    {
-        result.Position.FileName = inputStream->overrideFileName;
-        result.Position.Line += inputStream->overrideLineOffset;
-    }
-    return result;
+    destroyInputStream(preprocessor, inputStream);
 }
 
 // Consume one token from an input stream
 static Token AdvanceRawToken(PreprocessorInputStream* inputStream)
 {
-    return PossiblyOverrideSourceLoc(inputStream, inputStream->tokenReader.AdvanceToken());
+    if( auto primaryStream = asPrimaryInputStream(inputStream) )
+    {
+        auto result = primaryStream->token;
+        primaryStream->token = primaryStream->lexer.lexToken();
+        return result;
+    }
+    else
+    {
+        PretokenizedInputStream* pretokenized = (PretokenizedInputStream*) inputStream;
+        return pretokenized->tokenReader.AdvanceToken();
+    }
 }
 
 // Peek one token from an input stream
 static Token PeekRawToken(PreprocessorInputStream* inputStream)
 {
-    return PossiblyOverrideSourceLoc(inputStream, inputStream->tokenReader.PeekToken());
+    if( auto primaryStream = asPrimaryInputStream(inputStream) )
+    {
+        return primaryStream->token;
+    }
+    else
+    {
+        PretokenizedInputStream* pretokenized = (PretokenizedInputStream*) inputStream;
+        return pretokenized->tokenReader.PeekToken();
+    }
 }
 
 // Peek one token type from an input stream
 static TokenType PeekRawTokenType(PreprocessorInputStream* inputStream)
 {
-    return inputStream->tokenReader.PeekTokenType();
+    if( auto primaryStream = asPrimaryInputStream(inputStream) )
+    {
+        return primaryStream->token.type;
+    }
+    else
+    {
+        PretokenizedInputStream* pretokenized = (PretokenizedInputStream*) inputStream;
+        return pretokenized->tokenReader.PeekTokenType();
+    }
 }
 
 
@@ -355,55 +404,8 @@ static Token PeekRawToken(Preprocessor* preprocessor)
     }
 }
 
-// Without advancing preprocessor state, look *two* raw tokens ahead
-// (This is only needed in order to determine when we are possibly
-// expanding a function-style macro)
-TokenType PeekSecondRawTokenType(Preprocessor* preprocessor)
-{
-    // We need to find the strema that `advanceRawToken` would read from.
-    PreprocessorInputStream* inputStream = preprocessor->inputStream;
-    int count = 1;
-    for (;;)
-    {
-        if (!inputStream)
-        {
-            // No more input streams left to read
-            return TokenType::EndOfFile;
-        }
-
-        // The top-most input stream may be at its end, so
-        // look one entry up the stack (don't actually pop
-        // here, since we are just peeking)
-
-        TokenReader reader = inputStream->tokenReader;
-        if (reader.PeekTokenType() == TokenType::EndOfFile)
-        {
-            inputStream = inputStream->parent;
-            continue;
-        }
-
-        if (count)
-        {
-            count--;
-
-            // Note: we are advancing our temporary
-            // copy of the token reader
-            reader.AdvanceToken();
-            if (reader.PeekTokenType() == TokenType::EndOfFile)
-            {
-                inputStream = inputStream->parent;
-                continue;
-            }
-        }
-
-        // Everything worked, so peek a token from the top-most stream
-        return reader.PeekTokenType();
-    }
-}
-
-
 // Get the location of the current (raw) token
-static CodePosition PeekLoc(Preprocessor* preprocessor)
+static SourceLoc PeekLoc(Preprocessor* preprocessor)
 {
     return PeekRawToken(preprocessor).Position;
 }
@@ -471,7 +473,7 @@ static PreprocessorEnvironment* GetCurrentEnvironment(Preprocessor* preprocessor
 
         // If the current input stream is at its end, then
         // fall back to its parent stream.
-        if (inputStream->tokenReader.PeekTokenType() == TokenType::EndOfFile)
+        if (PeekRawTokenType(inputStream) == TokenType::EndOfFile)
         {
             inputStream = inputStream->parent;
             continue;
@@ -528,7 +530,11 @@ static void InitializeMacroExpansion(
     MacroExpansion*     expansion,
     PreprocessorMacro*  macro)
 {
-    InitializeInputStream(preprocessor, expansion);
+    initializeInputStream(preprocessor, expansion);
+
+    expansion->parent = preprocessor->inputStream;
+    expansion->primaryStream = preprocessor->inputStream->primaryStream;
+
     expansion->environment = macro->environment;
     expansion->macro = macro;
     expansion->tokenReader = TokenReader(macro->tokens);
@@ -550,6 +556,26 @@ static void AddEndOfStreamToken(
     macro->tokens.mTokens.Add(token);
 }
 
+static SimpleTokenInputStream* createSimpleInputStream(
+    Preprocessor*   preprocessor,
+    Token const&    token)
+{
+    SimpleTokenInputStream* inputStream = new SimpleTokenInputStream();
+    initializeInputStream(preprocessor, inputStream);
+
+    inputStream->lexedTokens.mTokens.Add(token);
+
+    Token eofToken;
+    eofToken.type = TokenType::EndOfFile;
+    eofToken.Position = token.Position;
+    eofToken.flags = TokenFlag::AfterWhitespace | TokenFlag::AtStartOfLine;
+    inputStream->lexedTokens.mTokens.Add(eofToken);
+ 
+    inputStream->tokenReader = TokenReader(inputStream->lexedTokens);
+
+    return inputStream;
+}
+
 // Check whether the current token on the given input stream should be
 // treated as a macro invocation, and if so set up state for expanding
 // that macro.
@@ -561,7 +587,7 @@ static void MaybeBeginMacroExpansion(
     for (;;)
     {
         // Look at the next token ahead of us
-        Token const& token = PeekRawToken(preprocessor);
+        Token token = PeekRawToken(preprocessor);
 
         // Not an identifier? Can't be a macro.
         if (token.type != TokenType::Identifier)
@@ -585,12 +611,29 @@ static void MaybeBeginMacroExpansion(
         // requires more lookahead than we usually have/need
         if (macro->flavor == PreprocessorMacroFlavor::FunctionLike)
         {
-            if(PeekSecondRawTokenType(preprocessor) != TokenType::LParent)
-                return;
-
-            // Consume the token that triggered macro expansion
+            // Consume the token that (possibly) triggered macro expansion
             AdvanceRawToken(preprocessor);
 
+            // Look at the next token, and see if it is an opening `(`
+            // that indicates we should actually expand a macro.
+            if(PeekRawTokenType(preprocessor) != TokenType::LParent)
+            {
+                // In this case, we are in a bit of a mess, because we have
+                // consumed the token that named the macro, but we need to
+                // make sure that token (and not whatever came after it)
+                // gets returned to the user.
+                //
+                // To work around this we will construct a short-lived input
+                // stream just to handle that one token, and also set
+                // a flag on the token to keep us from doing this logic again.
+
+                token.flags |= TokenFlag::SuppressMacroExpansion;
+
+                SimpleTokenInputStream* simpleStream = createSimpleInputStream(preprocessor, token);
+                PushInputStream(preprocessor, simpleStream);
+                return;
+            }
+
             // Consume the opening `(`
             Token leftParen = AdvanceRawToken(preprocessor);
 
@@ -762,8 +805,22 @@ top:
         }
 
         // Now re-lex the input
-        PreprocessorInputStream* inputStream = CreateInputStreamForSource(preprocessor, sb.ProduceString(), "token paste");
-        if (inputStream->tokenReader.GetCount() != 1)
+
+        // We create a dummy file to represent the token-paste operation
+        SourceFile* sourceFile = preprocessor->getCompileRequest()->getSourceManager()->allocateSourceFile("token paste", sb.ProduceString());
+
+        Lexer lexer;
+        lexer.initialize(sourceFile, GetSink(preprocessor));
+
+        SimpleTokenInputStream* inputStream = new SimpleTokenInputStream();
+        initializeInputStream(preprocessor, inputStream);
+
+        inputStream->lexedTokens = lexer.lexAllTokens();
+        inputStream->tokenReader = TokenReader(inputStream->lexedTokens);
+
+        // We expect the reuslt of lexing to be two tokens: one for the actual value,
+        // and one for the end-of-input marker.
+        if (inputStream->tokenReader.GetCount() != 2)
         {
             // We expect a token paste to produce a single token
             // TODO(tfoley): emit a diagnostic here
@@ -839,7 +896,7 @@ inline String const& GetDirectiveName(PreprocessorDirectiveContext* context)
 }
 
 // Get the location of the directive being parsed.
-inline CodePosition const& GetDirectiveLoc(PreprocessorDirectiveContext* context)
+inline SourceLoc const& GetDirectiveLoc(PreprocessorDirectiveContext* context)
 {
     return context->directiveToken.Position;
 }
@@ -851,7 +908,7 @@ static inline DiagnosticSink* GetSink(PreprocessorDirectiveContext* context)
 }
 
 // Wrapper to get a "current" location when parsing a directive
-static CodePosition PeekLoc(PreprocessorDirectiveContext* context)
+static SourceLoc PeekLoc(PreprocessorDirectiveContext* context)
 {
     return PeekLoc(context->preprocessor);
 }
@@ -970,8 +1027,11 @@ static bool IsSkipping(Preprocessor* preprocessor)
     PreprocessorInputStream* inputStream = preprocessor->inputStream;
     if (!inputStream) return false;
 
+    PrimaryInputStream* primaryStream = inputStream->primaryStream;
+    if(!primaryStream) return false;
+
     // If we are not inside a preprocessor conditional, then don't skip
-    PreprocessorConditional* conditional = inputStream->conditional;
+    PreprocessorConditional* conditional = primaryStream->conditional;
     if (!conditional) return false;
 
     // skip tokens unless the conditional is inside its `true` case
@@ -1032,8 +1092,9 @@ static void beginConditional(
     conditional->state = state;
 
     // Push conditional onto the stack
-    conditional->parent = inputStream->conditional;
-    inputStream->conditional = conditional;
+    auto primaryStream = inputStream->primaryStream;
+    conditional->parent = primaryStream->conditional;
+    primaryStream->conditional = conditional;
 }
 
 // Start a preprocessor conditional, with an initial enable/disable state.
@@ -1348,7 +1409,7 @@ static void HandleElseDirective(PreprocessorDirectiveContext* context)
     SLANG_ASSERT(inputStream);
 
     // if we aren't inside a conditional, then error
-    PreprocessorConditional* conditional = inputStream->conditional;
+    PreprocessorConditional* conditional = inputStream->primaryStream->conditional;
     if (!conditional)
     {
         GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::directiveWithoutIf, GetDirectiveName(context));
@@ -1402,7 +1463,7 @@ static void HandleElifDirective(PreprocessorDirectiveContext* context)
     PreprocessorExpressionValue value = ParseAndEvaluateExpression(context);
 
     // if we aren't inside a conditional, then error
-    PreprocessorConditional* conditional = inputStream->conditional;
+    PreprocessorConditional* conditional = inputStream->primaryStream->conditional;
     if (!conditional)
     {
         GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::directiveWithoutIf, GetDirectiveName(context));
@@ -1440,14 +1501,14 @@ static void HandleEndIfDirective(PreprocessorDirectiveContext* context)
     SLANG_ASSERT(inputStream);
 
     // if we aren't inside a conditional, then error
-    PreprocessorConditional* conditional = inputStream->conditional;
+    PreprocessorConditional* conditional = inputStream->primaryStream->conditional;
     if (!conditional)
     {
         GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::directiveWithoutIf, GetDirectiveName(context));
         return;
     }
 
-    inputStream->conditional = conditional->parent;
+    inputStream->primaryStream->conditional = conditional->parent;
     DestroyConditional(conditional);
 }
 
@@ -1492,8 +1553,9 @@ static void HandleIncludeDirective(PreprocessorDirectiveContext* context)
 
     String path = getFileNameTokenValue(pathToken);
 
-    // TODO(tfoley): make this robust in presence of `#line`
-    String pathIncludedFrom = GetDirectiveLoc(context).FileName;
+    auto directiveLoc = GetDirectiveLoc(context);
+    auto expandedDirectiveLoc = context->preprocessor->translationUnit->compileRequest->getSourceManager()->expandSourceLoc(directiveLoc);
+    String pathIncludedFrom = expandedDirectiveLoc.getSpellingPath();
     String foundPath;
     String foundSource;
 
@@ -1524,7 +1586,10 @@ static void HandleIncludeDirective(PreprocessorDirectiveContext* context)
 
     // Push the new file onto our stack of input streams
     // TODO(tfoley): check if we have made our include stack too deep
-    PreprocessorInputStream* inputStream = CreateInputStreamForSource(context->preprocessor, foundSource, foundPath);
+
+    SourceFile* sourceFile = context->preprocessor->getCompileRequest()->getSourceManager()->allocateSourceFile(foundPath, foundSource);
+
+    PreprocessorInputStream* inputStream = CreateInputStreamForSource(context->preprocessor, sourceFile);
     inputStream->parent = context->preprocessor->inputStream;
     context->preprocessor->inputStream = inputStream;
 }
@@ -1654,18 +1719,26 @@ static void HandleErrorDirective(PreprocessorDirectiveContext* context)
 // Handle a `#line` directive
 static void HandleLineDirective(PreprocessorDirectiveContext* context)
 {
+    auto inputStream = context->preprocessor->inputStream;
+
     int line = 0;
+
+    // `#line <integer-literal> ...`
     if (PeekTokenType(context) == TokenType::IntegerLiteral)
     {
         line = StringToInt(AdvanceToken(context).Content);
     }
-    else if (PeekTokenType(context) == TokenType::Identifier
-        && PeekToken(context).Content == "default")
+    // `#line`
+    // `#line default`
+    else if (
+        PeekTokenType(context) == TokenType::EndOfDirective
+        || (PeekTokenType(context) == TokenType::Identifier
+            && PeekToken(context).Content == "default"))
     {
         AdvanceToken(context);
 
         // Stop overiding soure locations.
-        context->preprocessor->inputStream->isOverridingSourceLoc = false;
+        inputStream->primaryStream->lexer.stopOverridingSourceLocations();
         return;
     }
     else
@@ -1678,16 +1751,19 @@ static void HandleLineDirective(PreprocessorDirectiveContext* context)
         return;
     }
 
-    CodePosition directiveLoc = GetDirectiveLoc(context);
+    SourceLoc directiveLoc = GetDirectiveLoc(context);
+
+    auto sourceManager = context->preprocessor->translationUnit->compileRequest->getSourceManager();
+    auto expandedDirectiveLoc = sourceManager->expandSourceLoc(directiveLoc);
 
     String file;
     if (PeekTokenType(context) == TokenType::EndOfDirective)
     {
-        file = directiveLoc.FileName;
+        file = expandedDirectiveLoc.getPath();
     }
     else if (PeekTokenType(context) == TokenType::StringLiteral)
     {
-        file = AdvanceToken(context).Content;
+        file = getStringLiteralTokenValue(AdvanceToken(context));
     }
     else if (PeekTokenType(context) == TokenType::IntegerLiteral)
     {
@@ -1701,11 +1777,9 @@ static void HandleLineDirective(PreprocessorDirectiveContext* context)
         return;
     }
 
-    PreprocessorInputStream* inputStream = context->preprocessor->inputStream;
+    SourceLoc newLoc = sourceManager->allocateSourceFileForLineDirective(expandedDirectiveLoc, file, line);
 
-    inputStream->isOverridingSourceLoc = true;
-    inputStream->overrideFileName = file;
-    inputStream->overrideLineOffset = line - (directiveLoc.Line + 1);
+    inputStream->primaryStream->lexer.startOverridingSourceLocations(newLoc);
 }
 
 // Handle a `#pragma` directive
@@ -2005,10 +2079,14 @@ static void DefineMacro(
     String fileName = "command line";
     PreprocessorMacro* macro = CreateMacro(preprocessor);
 
+    SourceFile* keyFile = preprocessor->translationUnit->compileRequest->getSourceManager()->allocateSourceFile(fileName, key);
+    SourceFile* valueFile = preprocessor->translationUnit->compileRequest->getSourceManager()->allocateSourceFile(fileName, value);
+
     // Use existing `Lexer` to generate a token stream.
-    Lexer lexer(fileName, value, GetSink(preprocessor));
+    Lexer lexer;
+    lexer.initialize(valueFile, GetSink(preprocessor));
     macro->tokens = lexer.lexAllTokens();
-    macro->nameToken = Token(TokenType::Identifier, key, 0, 0, 0, fileName);
+    macro->nameToken = Token(TokenType::Identifier, key, keyFile->sourceRange.begin);
 
     PreprocessorMacro* oldMacro = NULL;
     if (preprocessor->globalEnv.macros.TryGetValue(key, oldMacro))
@@ -2039,8 +2117,7 @@ static TokenList ReadAllTokens(
 }
 
 TokenList preprocessSource(
-    String const&               source,
-    String const&               fileName,
+    SourceFile*                 file,
     DiagnosticSink*             sink,
     IncludeHandler*             includeHandler,
     Dictionary<String, String>  defines,
@@ -2057,7 +2134,7 @@ TokenList preprocessSource(
     }
 
     // create an initial input stream based on the provided buffer
-    preprocessor.inputStream = CreateInputStreamForSource(&preprocessor, source, fileName);
+    preprocessor.inputStream = CreateInputStreamForSource(&preprocessor, file);
 
     TokenList tokens = ReadAllTokens(&preprocessor);
 
@@ -2097,8 +2174,9 @@ static void HandleImportDirective(PreprocessorDirectiveContext* context)
 
     String path = getFileNameTokenValue(pathToken);
 
-    // TODO(tfoley): make this robust in presence of `#line`
-    String pathIncludedFrom = GetDirectiveLoc(context).FileName;
+    auto directiveLoc = GetDirectiveLoc(context);
+    auto expandedDirectiveLoc = context->preprocessor->translationUnit->compileRequest->getSourceManager()->expandSourceLoc(directiveLoc);
+    String pathIncludedFrom = expandedDirectiveLoc.getSpellingPath();
     String foundPath;
     String foundSource;
 
@@ -2162,7 +2240,8 @@ static void HandleImportDirective(PreprocessorDirectiveContext* context)
             PreprocessorInputStream* savedStream = preprocessor->inputStream;
 
             // Create an input stream for reading from the imported file
-            PreprocessorInputStream* subInputStream = CreateInputStreamForSource(preprocessor, foundSource, foundPath);
+            SourceFile* sourceFile = context->preprocessor->getCompileRequest()->getSourceManager()->allocateSourceFile(foundPath, foundSource);
+            PreprocessorInputStream* subInputStream = CreateInputStreamForSource(preprocessor, sourceFile);
 
             // Now preprocess that stream
             preprocessor->inputStream = subInputStream;
@@ -2180,24 +2259,18 @@ static void HandleImportDirective(PreprocessorDirectiveContext* context)
  
     // Now create a dummy token stream to represent the import request,
     // so that it can be manifest in the user's program
-    SourceTextInputStream* inputStream = new SourceTextInputStream();
- 
+
     Token token;
     token.type = TokenType::PoundImport;
     token.Position = GetDirectiveLoc(context);
     token.flags = 0;
     token.Content = foundPath;
- 
-    inputStream->lexedTokens.mTokens.Add(token);
- 
-    token.type = TokenType::EndOfFile;
-    token.flags = TokenFlag::AfterWhitespace | TokenFlag::AtStartOfLine;
-    inputStream->lexedTokens.mTokens.Add(token);
- 
-    inputStream->tokenReader = TokenReader(inputStream->lexedTokens);
- 
-    inputStream->parent = context->preprocessor->inputStream;
-    context->preprocessor->inputStream = inputStream;
+
+    SimpleTokenInputStream* inputStream = createSimpleInputStream(
+        context->preprocessor,
+        token);
+
+    PushInputStream(context->preprocessor, inputStream);
 }