Merge pull request #43 from tfoleyNV/import-macros

Add support for `#import`

1 files changed, 154 insertions, 83 deletions

diff --git a/source/slang/preprocessor.cpp b/source/slang/preprocessor.cpp
index 00abb3fe5..084f518fb 100644
--- a/source/slang/preprocessor.cpp
+++ b/source/slang/preprocessor.cpp
@@ -1,12 +1,11 @@
-// Preprocessor.cpp
-#include "Preprocessor.h"
+// preprocessor.cpp
+#include "preprocessor.h"
 
-#include "Diagnostics.h"
-#include "Lexer.h"
-
-// Needed so that we can construct modifier syntax
-// to represent GLSL directives
-#include "Syntax.h"
+#include "compiler.h"
+#include "diagnostics.h"
+#include "lexer.h"
+// Needed so that we can construct modifier syntax to represent GLSL directives
+#include "syntax.h"
 
 #include <assert.h>
 
@@ -18,15 +17,6 @@
 
 namespace Slang {
 
-// Forward declaration for code provided in `slang.cpp`
-//
-// TODO: Need an appropriate header for this.
-String autoImportModule(
-    CompileRequest*     request,
-    String const&       path,
-    String const&       source,
-    CodePosition const& loc);
-
 // State of a preprocessor conditional, which can change when
 // we encounter directives like `#elif` or `#endif`
 enum class PreprocessorConditionalState
@@ -177,11 +167,23 @@ struct Preprocessor
     // represent end-of-input situations.
     Token                                   endOfFileToken;
 
-    // Syntax for the program we are trying to parse
-    ProgramSyntaxNode*                      syntax;
+    // The translation unit that is being parsed
+    TranslationUnitRequest*                 translationUnit;
+
+    TranslationUnitRequest* getTranslationUnit()
+    {
+        return translationUnit;
+    }
+
+    ProgramSyntaxNode* getSyntax()
+    {
+        return getTranslationUnit()->SyntaxNode.Ptr();
+    }
 
-    // The over-arching compile request taht is invoking us
-    CompileRequest*                         compileRequest;
+    CompileRequest* getCompileRequest()
+    {
+        return getTranslationUnit()->compileRequest;
+    }
 };
 
 // Convenience routine to access the diagnostic sink
@@ -315,11 +317,6 @@ static Token AdvanceRawToken(Preprocessor* preprocessor)
                 EndInputStream(preprocessor, inputStream);
                 continue;
             }
-            else
-            {
-                // HACK(tfoley): A place to fall into debugger...
-                int f = 0;
-            }
         }
 
         // Everything worked, so read a token from the top-most stream
@@ -351,11 +348,6 @@ static Token PeekRawToken(Preprocessor* preprocessor)
                 inputStream = inputStream->parent;
                 continue;
             }
-            else
-            {
-                // HACK(tfoley): A place to fall into debugger...
-                int f = 0;
-            }
         }
 
         // Everything worked, so the token we just peeked is fine.
@@ -1489,6 +1481,8 @@ static void expectEndOfDirective(PreprocessorDirectiveContext* context)
     AdvanceRawToken(context->preprocessor);
 }
 
+// Handle a `#import` directive
+static void HandleImportDirective(PreprocessorDirectiveContext* context);
 
 // Handle a `#include` directive
 static void HandleIncludeDirective(PreprocessorDirectiveContext* context)
@@ -1520,7 +1514,7 @@ static void HandleIncludeDirective(PreprocessorDirectiveContext* context)
         GetSink(context)->diagnose(pathToken.Position, Diagnostics::includeFailed, path);
         return;
 
-    case IncludeResult::FoundIncludeFile:
+    case IncludeResult::Found:
         break;
     }
 
@@ -1529,50 +1523,12 @@ static void HandleIncludeDirective(PreprocessorDirectiveContext* context)
     // a switch of input stream
     expectEndOfDirective(context);
 
-    switch( includeResult )
-    {
-    case IncludeResult::FoundIncludeFile:
-        {
-            // Push the new file onto our stack of input streams
-            // TODO(tfoley): check if we have made our include stack too deep
-            PreprocessorInputStream* inputStream = CreateInputStreamForSource(context->preprocessor, foundSource, foundPath);
-            inputStream->parent = context->preprocessor->inputStream;
-            context->preprocessor->inputStream = inputStream;
-        }
-        break;
- 
-    case IncludeResult::FoundAutoImportFile:
-        {
-            //
- 
-            String autoImportName = autoImportModule(
-                context->preprocessor->compileRequest,
-                foundPath,
-                foundSource,
-                GetDirectiveLoc(context));
- 
-            SourceTextInputStream* inputStream = new SourceTextInputStream();
- 
-            Token token;
-            token.Type = TokenType::AutoImport;
-            token.Position = GetDirectiveLoc(context);
-            token.flags = 0;
-            token.Content = autoImportName;
- 
-            inputStream->lexedTokens.mTokens.Add(token);
- 
-            token.Type = TokenType::EndOfFile;
-            token.flags = TokenFlag::AfterWhitespace | TokenFlag::AtStartOfLine;
-            inputStream->lexedTokens.mTokens.Add(token);
- 
-            inputStream->tokenReader = TokenReader(inputStream->lexedTokens);
- 
-            inputStream->parent = context->preprocessor->inputStream;
-            context->preprocessor->inputStream = inputStream;
-        }
-        break;
-    }
- }
+    // Push the new file onto our stack of input streams
+    // TODO(tfoley): check if we have made our include stack too deep
+    PreprocessorInputStream* inputStream = CreateInputStreamForSource(context->preprocessor, foundSource, foundPath);
+    inputStream->parent = context->preprocessor->inputStream;
+    context->preprocessor->inputStream = inputStream;
+}
 
 // Handle a `#define` directive
 static void HandleDefineDirective(PreprocessorDirectiveContext* context)
@@ -1789,7 +1745,7 @@ static void handleGLSLVersionDirective(PreprocessorDirectiveContext* context)
     // Attach the modifier to the program we are parsing!
 
     addModifier(
-        context->preprocessor->syntax,
+        context->preprocessor->getSyntax(),
         modifier);
 }
 
@@ -1833,7 +1789,7 @@ static void handleGLSLExtensionDirective(PreprocessorDirectiveContext* context)
     // Attach the modifier to the program we are parsing!
 
     addModifier(
-        context->preprocessor->syntax,
+        context->preprocessor->getSyntax(),
         modifier);
 }
 
@@ -1878,6 +1834,7 @@ static const PreprocessorDirective kDirectives[] =
     { "elif",       &HandleElifDirective,       ProcessWhenSkipping },
     { "endif",      &HandleEndIfDirective,      ProcessWhenSkipping },
 
+    { "import",     &HandleImportDirective,     0 },
     { "include",    &HandleIncludeDirective,    0 },
     { "define",     &HandleDefineDirective,     0 },
     { "undef",      &HandleUndefDirective,      0 },
@@ -2084,14 +2041,12 @@ TokenList preprocessSource(
     String const&               fileName,
     DiagnosticSink*             sink,
     IncludeHandler*             includeHandler,
-     Dictionary<String, String> defines,
-    ProgramSyntaxNode*          syntax,
-    CompileRequest*             compileRequest)
+    Dictionary<String, String>  defines,
+    TranslationUnitRequest*     translationUnit)
 {
     Preprocessor preprocessor;
     InitializePreprocessor(&preprocessor, sink);
-    preprocessor.syntax = syntax;
-    preprocessor.compileRequest = compileRequest;
+    preprocessor.translationUnit = translationUnit;
 
     preprocessor.includeHandler = includeHandler;
     for (auto p : defines)
@@ -2129,4 +2084,120 @@ TokenList preprocessSource(
     return tokens;
 }
 
+//
+
+// Handle a `#import` directive
+static void HandleImportDirective(PreprocessorDirectiveContext* context)
+{
+    Token pathToken;
+    if(!Expect(context, TokenType::StringLiterial, Diagnostics::expectedTokenInPreprocessorDirective, &pathToken))
+        return;
+
+    String path = getFileNameTokenValue(pathToken);
+
+    // TODO(tfoley): make this robust in presence of `#line`
+    String pathIncludedFrom = GetDirectiveLoc(context).FileName;
+    String foundPath;
+    String foundSource;
+
+    IncludeHandler* includeHandler = context->preprocessor->includeHandler;
+    if (!includeHandler)
+    {
+        GetSink(context)->diagnose(pathToken.Position, Diagnostics::importFailed, path);
+        GetSink(context)->diagnose(pathToken.Position, Diagnostics::noIncludeHandlerSpecified);
+        return;
+    }
+    auto includeResult = includeHandler->TryToFindIncludeFile(path, pathIncludedFrom, &foundPath, &foundSource);
+
+    switch (includeResult)
+    {
+    case IncludeResult::NotFound:
+    case IncludeResult::Error:
+        GetSink(context)->diagnose(pathToken.Position, Diagnostics::importFailed, path);
+        return;
+
+    case IncludeResult::Found:
+        break;
+    }
+
+    // Do all checking related to the end of this directive before we push a new stream,
+    // just to avoid complications where that check would need to deal with
+    // a switch of input stream
+    expectEndOfDirective(context);
+
+    // TODO: may want to have some kind of canonicalization step here
+    String moduleKey = foundPath;
+
+    // Import code from the chosen file, if needed. We only
+    // need to import on the first `#import` directive, and
+    // after that we ignore additional `#import`s for the same file.
+    {
+        auto translationUnit = context->preprocessor->translationUnit;
+        auto request = translationUnit->compileRequest;
+
+
+        // Have we already loaded a module matching this name?
+        if (request->mapPathToLoadedModule.TryGetValue(moduleKey))
+        {
+            // The module has already been loaded, so we don't need to
+            // actually tokenize the code here. But note that we *do*
+            // go on to insert tokens for an `import` operation into
+            // the stream, so it is up to downstream code to avoid
+            // re-importing the same thing twice.
+        }
+        else
+        {
+            // We are going to preprocess the file using the *same* preprocessor
+            // state that is already active. The main alternative would be
+            // to construct a fresh preprocessor and use that. The current
+            // choice is made so that macros defined in the imported file
+            // will be made visible to the importer, rather than disappear
+            // when a sub-preprocessor gets finalized.
+            auto preprocessor = context->preprocessor;
+
+            // We need to save/restore the input stream, so that we can
+            // re-use the preprocessor
+            PreprocessorInputStream* savedStream = preprocessor->inputStream;
+
+            // Create an input stream for reading from the imported file
+            PreprocessorInputStream* subInputStream = CreateInputStreamForSource(preprocessor, foundSource, foundPath);
+
+            // Now preprocess that stream
+            preprocessor->inputStream = subInputStream;
+            TokenList subTokens = ReadAllTokens(preprocessor);
+
+            // Restore the previous input stream
+            preprocessor->inputStream = savedStream;
+
+            // Now we need to do something with those tokens we read
+            request->handlePoundImport(
+                moduleKey,
+                subTokens);
+        }
+    }
+ 
+    // Now create a dummy token stream to represent the import request,
+    // so that it can be manifest in the user's program
+    SourceTextInputStream* inputStream = new SourceTextInputStream();
+ 
+    Token token;
+    token.Type = TokenType::PoundImport;
+    token.Position = GetDirectiveLoc(context);
+    token.flags = 0;
+    token.Content = foundPath;
+ 
+    inputStream->lexedTokens.mTokens.Add(token);
+ 
+    token.Type = TokenType::EndOfFile;
+    token.flags = TokenFlag::AfterWhitespace | TokenFlag::AtStartOfLine;
+    inputStream->lexedTokens.mTokens.Add(token);
+ 
+    inputStream->tokenReader = TokenReader(inputStream->lexedTokens);
+ 
+    inputStream->parent = context->preprocessor->inputStream;
+    context->preprocessor->inputStream = inputStream;
+}
+
+
+
 }