Replace "auto-import" with `#import`

Right now `#import` only differs from `#include` in that it takes a string literal for a file name instead of a raw identifier (to which `.slang` gets appended).

The next step is to make `#import` respect preprocessor state, while `__import` doesn't.

11 files changed, 87 insertions, 109 deletions

diff --git a/slang.h b/slang.h
index 91347d024..91b9f5931 100644
--- a/slang.h
+++ b/slang.h
@@ -204,16 +204,6 @@ extern "C"
         const char*             searchDir);
 
     /*!
-    @brief Add a path to use when searching for referenced files, that automatically treats `#include` as `__import`
-    This behaves just like `spAddSearchPath()` except that any `#include` file found through this path
-    will be treated as if it was referenced with `__import`.
-    @param ctx The compilation context.
-    @param searchDir The additional search directory.
-    */
-    SLANG_API void spAddAutoImportPath(
-        SlangCompileRequest*    request,
-        const char*             searchDir);
-    /*!
     @brief Add a macro definition to be used during preprocessing.
     @param key The name of the macro to define.
     @param value The value of the macro to define.
diff --git a/source/slang/compiler.h b/source/slang/compiler.h
index 3f45ea5c7..89bb01738 100644
--- a/source/slang/compiler.h
+++ b/source/slang/compiler.h
@@ -150,21 +150,13 @@ namespace Slang
     // A directory to be searched when looking for files (e.g., `#include`)
     struct SearchDirectory
     {
-        enum Kind
-        {
-            Default,
-            AutoImport,
-        };
-
         SearchDirectory() = default;
         SearchDirectory(SearchDirectory const& other) = default;
-        SearchDirectory(String const& path, Kind kind)
+        SearchDirectory(String const& path)
             : path(path)
-            , kind(kind)
         {}
 
         String  path;
-        Kind    kind;
     };
 
     class Session;
diff --git a/source/slang/diagnostic-defs.h b/source/slang/diagnostic-defs.h
index ff93e51fb..f92f43ddb 100644
--- a/source/slang/diagnostic-defs.h
+++ b/source/slang/diagnostic-defs.h
@@ -92,6 +92,7 @@ DIAGNOSTIC(-1, Note, seeOpeningToken, "see opening '$0'")
 
 // 153xx - #include
 DIAGNOSTIC(15300, Error, includeFailed, "failed to find include file '$0'")
+DIAGNOSTIC(15301, Error, importFailed, "failed to find imported file '$0'")
 DIAGNOSTIC(-1, Error, noIncludeHandlerSpecified, "no `#include` handler was specified")
 
 // 154xx - macro definition
diff --git a/source/slang/options.cpp b/source/slang/options.cpp
index 81b893d66..62b45e025 100644
--- a/source/slang/options.cpp
+++ b/source/slang/options.cpp
@@ -375,14 +375,6 @@ struct OptionsParser
                         compileRequest,
                         String(includeDirStr).begin());
                 }
-                else if (argStr == "-auto-import-dir")
-                {
-                    char const* importDirStr = tryReadCommandLineArgumentRaw(arg, &argCursor, argEnd);
-
-                    spAddAutoImportPath(
-                        compileRequest,
-                        String(importDirStr).begin());
-                }
                 else if (argStr == "--")
                 {
                     // The `--` option causes us to stop trying to parse options,
diff --git a/source/slang/parser.cpp b/source/slang/parser.cpp
index 0598e67a7..61d7b225d 100644
--- a/source/slang/parser.cpp
+++ b/source/slang/parser.cpp
@@ -805,10 +805,10 @@ namespace Slang
         return decl;
     }
 
-    static RefPtr<Decl> parseAutoImportDecl(
+    static RefPtr<Decl> parsePoundImportDecl(
         Parser* parser)
     {
-        Token importToken = parser->ReadToken(TokenType::AutoImport);
+        Token importToken = parser->ReadToken(TokenType::PoundImport);
 
         auto decl = new ImportDecl();
         decl->nameToken = importToken;
@@ -2186,8 +2186,8 @@ namespace Slang
             decl = parseModifierDecl(parser);
         else if(parser->LookAheadToken("__import"))
             decl = parseImportDecl(parser);
-        else if(parser->LookAheadToken(TokenType::AutoImport))
-            decl = parseAutoImportDecl(parser);
+        else if(parser->LookAheadToken(TokenType::PoundImport))
+            decl = parsePoundImportDecl(parser);
         else if (AdvanceIf(parser, TokenType::Semicolon))
         {
             decl = new EmptyDecl();
diff --git a/source/slang/preprocessor.cpp b/source/slang/preprocessor.cpp
index 00abb3fe5..b93e47ec2 100644
--- a/source/slang/preprocessor.cpp
+++ b/source/slang/preprocessor.cpp
@@ -1489,9 +1489,8 @@ static void expectEndOfDirective(PreprocessorDirectiveContext* context)
     AdvanceRawToken(context->preprocessor);
 }
 
-
-// Handle a `#include` directive
-static void HandleIncludeDirective(PreprocessorDirectiveContext* context)
+// Handle a `#import` directive
+static void HandleImportDirective(PreprocessorDirectiveContext* context)
 {
     Token pathToken;
     if(!Expect(context, TokenType::StringLiterial, Diagnostics::expectedTokenInPreprocessorDirective, &pathToken))
@@ -1507,7 +1506,7 @@ static void HandleIncludeDirective(PreprocessorDirectiveContext* context)
     IncludeHandler* includeHandler = context->preprocessor->includeHandler;
     if (!includeHandler)
     {
-        GetSink(context)->diagnose(pathToken.Position, Diagnostics::includeFailed, path);
+        GetSink(context)->diagnose(pathToken.Position, Diagnostics::importFailed, path);
         GetSink(context)->diagnose(pathToken.Position, Diagnostics::noIncludeHandlerSpecified);
         return;
     }
@@ -1517,10 +1516,10 @@ static void HandleIncludeDirective(PreprocessorDirectiveContext* context)
     {
     case IncludeResult::NotFound:
     case IncludeResult::Error:
-        GetSink(context)->diagnose(pathToken.Position, Diagnostics::includeFailed, path);
+        GetSink(context)->diagnose(pathToken.Position, Diagnostics::importFailed, path);
         return;
 
-    case IncludeResult::FoundIncludeFile:
+    case IncludeResult::Found:
         break;
     }
 
@@ -1529,50 +1528,82 @@ static void HandleIncludeDirective(PreprocessorDirectiveContext* context)
     // a switch of input stream
     expectEndOfDirective(context);
 
-    switch( includeResult )
-    {
-    case IncludeResult::FoundIncludeFile:
-        {
-            // Push the new file onto our stack of input streams
-            // TODO(tfoley): check if we have made our include stack too deep
-            PreprocessorInputStream* inputStream = CreateInputStreamForSource(context->preprocessor, foundSource, foundPath);
-            inputStream->parent = context->preprocessor->inputStream;
-            context->preprocessor->inputStream = inputStream;
-        }
-        break;
- 
-    case IncludeResult::FoundAutoImportFile:
-        {
-            //
- 
-            String autoImportName = autoImportModule(
-                context->preprocessor->compileRequest,
-                foundPath,
-                foundSource,
-                GetDirectiveLoc(context));
+    // Import code from the chosen file
+    String autoImportName = autoImportModule(
+        context->preprocessor->compileRequest,
+        foundPath,
+        foundSource,
+        GetDirectiveLoc(context));
  
-            SourceTextInputStream* inputStream = new SourceTextInputStream();
+    // Now create a dummy token stream to represent the import request,
+    // so that it can be manifest in the user's program
+    SourceTextInputStream* inputStream = new SourceTextInputStream();
  
-            Token token;
-            token.Type = TokenType::AutoImport;
-            token.Position = GetDirectiveLoc(context);
-            token.flags = 0;
-            token.Content = autoImportName;
+    Token token;
+    token.Type = TokenType::PoundImport;
+    token.Position = GetDirectiveLoc(context);
+    token.flags = 0;
+    token.Content = autoImportName;
  
-            inputStream->lexedTokens.mTokens.Add(token);
+    inputStream->lexedTokens.mTokens.Add(token);
  
-            token.Type = TokenType::EndOfFile;
-            token.flags = TokenFlag::AfterWhitespace | TokenFlag::AtStartOfLine;
-            inputStream->lexedTokens.mTokens.Add(token);
+    token.Type = TokenType::EndOfFile;
+    token.flags = TokenFlag::AfterWhitespace | TokenFlag::AtStartOfLine;
+    inputStream->lexedTokens.mTokens.Add(token);
  
-            inputStream->tokenReader = TokenReader(inputStream->lexedTokens);
+    inputStream->tokenReader = TokenReader(inputStream->lexedTokens);
  
-            inputStream->parent = context->preprocessor->inputStream;
-            context->preprocessor->inputStream = inputStream;
-        }
+    inputStream->parent = context->preprocessor->inputStream;
+    context->preprocessor->inputStream = inputStream;
+}
+
+
+
+// Handle a `#include` directive
+static void HandleIncludeDirective(PreprocessorDirectiveContext* context)
+{
+    Token pathToken;
+    if(!Expect(context, TokenType::StringLiterial, Diagnostics::expectedTokenInPreprocessorDirective, &pathToken))
+        return;
+
+    String path = getFileNameTokenValue(pathToken);
+
+    // TODO(tfoley): make this robust in presence of `#line`
+    String pathIncludedFrom = GetDirectiveLoc(context).FileName;
+    String foundPath;
+    String foundSource;
+
+    IncludeHandler* includeHandler = context->preprocessor->includeHandler;
+    if (!includeHandler)
+    {
+        GetSink(context)->diagnose(pathToken.Position, Diagnostics::includeFailed, path);
+        GetSink(context)->diagnose(pathToken.Position, Diagnostics::noIncludeHandlerSpecified);
+        return;
+    }
+    auto includeResult = includeHandler->TryToFindIncludeFile(path, pathIncludedFrom, &foundPath, &foundSource);
+
+    switch (includeResult)
+    {
+    case IncludeResult::NotFound:
+    case IncludeResult::Error:
+        GetSink(context)->diagnose(pathToken.Position, Diagnostics::includeFailed, path);
+        return;
+
+    case IncludeResult::Found:
         break;
     }
- }
+
+    // Do all checking related to the end of this directive before we push a new stream,
+    // just to avoid complications where that check would need to deal with
+    // a switch of input stream
+    expectEndOfDirective(context);
+
+    // Push the new file onto our stack of input streams
+    // TODO(tfoley): check if we have made our include stack too deep
+    PreprocessorInputStream* inputStream = CreateInputStreamForSource(context->preprocessor, foundSource, foundPath);
+    inputStream->parent = context->preprocessor->inputStream;
+    context->preprocessor->inputStream = inputStream;
+}
 
 // Handle a `#define` directive
 static void HandleDefineDirective(PreprocessorDirectiveContext* context)
@@ -1878,6 +1909,7 @@ static const PreprocessorDirective kDirectives[] =
     { "elif",       &HandleElifDirective,       ProcessWhenSkipping },
     { "endif",      &HandleEndIfDirective,      ProcessWhenSkipping },
 
+    { "import",     &HandleImportDirective,     0 },
     { "include",    &HandleIncludeDirective,    0 },
     { "define",     &HandleDefineDirective,     0 },
     { "undef",      &HandleUndefDirective,      0 },
diff --git a/source/slang/preprocessor.h b/source/slang/preprocessor.h
index 0c7848bb4..ebeb810a0 100644
--- a/source/slang/preprocessor.h
+++ b/source/slang/preprocessor.h
@@ -15,8 +15,7 @@ enum class IncludeResult
 {
     Error,
     NotFound,
-    FoundIncludeFile,
-    FoundAutoImportFile,
+    Found,
 };
 
 // Callback interface for the preprocessor to use when looking
diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp
index fc18ea1cf..76cb502c0 100644
--- a/source/slang/slang.cpp
+++ b/source/slang/slang.cpp
@@ -103,21 +103,7 @@ struct IncludeHandlerImpl : IncludeHandler
 
             request->mDependencyFilePaths.Add(path);
 
-            // HACK(tfoley): We might have found the file in the same directory,
-            // but what if this is also inside an auto-import path?
-            for (auto & dir : request->searchDirectories)
-            {
-                // Only consider auto-import paths
-                if(dir.kind != SearchDirectory::Kind::AutoImport)
-                    continue;
-
-                String otherPath = Path::Combine(dir.path, pathToInclude);
-
-                if(otherPath == path)
-                    return IncludeResult::FoundAutoImportFile;
-            }
-
-            return IncludeResult::FoundIncludeFile;
+            return IncludeResult::Found;
         }
 
         for (auto & dir : request->searchDirectories)
@@ -130,14 +116,7 @@ struct IncludeHandlerImpl : IncludeHandler
 
                 request->mDependencyFilePaths.Add(path);
 
-                switch( dir.kind )
-                {
-                case SearchDirectory::Kind::Default:
-                    return IncludeResult::FoundIncludeFile;
-
-                case SearchDirectory::Kind::AutoImport:
-                    return IncludeResult::FoundAutoImportFile;
-                }
+                return IncludeResult::Found;
             }
         }
         return IncludeResult::NotFound;
@@ -674,14 +653,7 @@ SLANG_API void spAddSearchPath(
         SlangCompileRequest*    request,
         const char*             path)
 {
-    REQ(request)->searchDirectories.Add(Slang::SearchDirectory(path, Slang::SearchDirectory::Kind::Default));
-}
-
-SLANG_API void spAddAutoImportPath(
-    SlangCompileRequest*    request,
-    const char*             path)
-{
-    REQ(request)->searchDirectories.Add(Slang::SearchDirectory(path, Slang::SearchDirectory::Kind::AutoImport));
+    REQ(request)->searchDirectories.Add(Slang::SearchDirectory(path));
 }
 
 SLANG_API void spAddPreprocessorDefine(
diff --git a/source/slang/token-defs.h b/source/slang/token-defs.h
index f7800de55..0fe833e36 100644
--- a/source/slang/token-defs.h
+++ b/source/slang/token-defs.h
@@ -30,7 +30,7 @@ TOKEN(NewLine,          "newline")
 TOKEN(LineComment,      "line comment")
 TOKEN(BlockComment,     "block comment")
 
-TOKEN(AutoImport,       "automatic import directive")
+TOKEN(PoundImport,      "'#import'")
 
 #define PUNCTUATION(id, text) \
     TOKEN(id, "'" text "'")
diff --git a/tests/render/auto-import.hlsl b/tests/render/pound-import.hlsl
index 588ebc612..a9b625fb6 100644
--- a/tests/render/auto-import.hlsl
+++ b/tests/render/pound-import.hlsl
@@ -1,4 +1,4 @@
-//TEST(smoke,render):COMPARE_HLSL_GLSL_RENDER: -xslang -auto-import-dir -xslang tests/render/
+//TEST(smoke,render):COMPARE_HLSL_GLSL_RENDER:
 
 // This is a basic test case for cross-compilation behavior.
 //
@@ -8,7 +8,7 @@
 
 
 // Pull in Spire code depdendency using extended syntax:
-#include "auto-import.slang.h"
+#import "pound-import.slang.h"
 
 #if defined(__HLSL__)
 
diff --git a/tests/render/auto-import.slang.h b/tests/render/pound-import.slang.h
index d53005688..d53005688 100644
--- a/tests/render/auto-import.slang.h
+++ b/tests/render/pound-import.slang.h