Allow for automatic importing of Slang code

The basic idea of this change is that user code can just write:

    #include "foo.h"

and then if `foo.h` gets found in a list of registered directories for "auto-import," then it actually gets interpreted as if the user had writte, more or less:

    __import foo;

That is, the code in `foo.h` will be treated as Slang, and will be fully parsed and checked (no matter what the source language had been), and the scoping rules will be those of `__import` instead of `#include`.

This is a really big hammer, and I could imagine it smashing fingers if used poorly.
I'm not sure this feature will pan out, but we need to try things to know.

One big piece of that that I'll likely keep in either case is an overhaul of command-line options parsing for `slangc`. In particular, this logic has been moved into the core `slang` library (so that users can just pass options in via the API), and it is all done on UTF-8 strings rather than wide strings (which was always going to be Windows-specific).

1 files changed, 73 insertions, 15 deletions

diff --git a/source/slang/preprocessor.cpp b/source/slang/preprocessor.cpp
index b8a42d46d..6e0906d9e 100644
--- a/source/slang/preprocessor.cpp
+++ b/source/slang/preprocessor.cpp
@@ -16,7 +16,16 @@
 // idioms for using the preprocessor, found in shader code in the wild.
 
 
-namespace Slang{
+namespace Slang {
+
+// Forward declaration for code provided in `slang.cpp`
+//
+// TODO: Need an appropriate header for this.
+String autoImportModule(
+    CompileRequest*     request,
+    String const&       path,
+    String const&       source,
+    CodePosition const& loc);
 
 // State of a preprocessor conditional, which can change when
 // we encounter directives like `#elif` or `#endif`
@@ -170,6 +179,9 @@ struct Preprocessor
 
     // Syntax for the program we are trying to parse
     ProgramSyntaxNode*                      syntax;
+
+    // The over-arching compile request taht is invoking us
+    CompileRequest*                         compileRequest;
 };
 
 // Convenience routine to access the diagnostic sink
@@ -1475,7 +1487,6 @@ static void HandleIncludeDirective(PreprocessorDirectiveContext* context)
     String foundPath;
     String foundSource;
 
-
     IncludeHandler* includeHandler = context->preprocessor->includeHandler;
     if (!includeHandler)
     {
@@ -1483,10 +1494,17 @@ static void HandleIncludeDirective(PreprocessorDirectiveContext* context)
         GetSink(context)->diagnose(pathToken.Position, Diagnostics::noIncludeHandlerSpecified);
         return;
     }
-    if (!includeHandler->TryToFindIncludeFile(path, pathIncludedFrom, &foundPath, &foundSource))
+    auto includeResult = includeHandler->TryToFindIncludeFile(path, pathIncludedFrom, &foundPath, &foundSource);
+
+    switch (includeResult)
     {
+    case IncludeResult::NotFound:
+    case IncludeResult::Error:
         GetSink(context)->diagnose(pathToken.Position, Diagnostics::includeFailed, path);
         return;
+
+    case IncludeResult::FoundIncludeFile:
+        break;
     }
 
     // Do all checking related to the end of this directive before we push a new stream,
@@ -1494,12 +1512,50 @@ static void HandleIncludeDirective(PreprocessorDirectiveContext* context)
     // a switch of input stream
     expectEndOfDirective(context);
 
-    // Push the new file onto our stack of input streams
-    // TODO(tfoley): check if we have made our include stack too deep
-    PreprocessorInputStream* inputStream = CreateInputStreamForSource(context->preprocessor, foundSource, foundPath);
-    inputStream->parent = context->preprocessor->inputStream;
-    context->preprocessor->inputStream = inputStream;
-}
+    switch( includeResult )
+    {
+    case IncludeResult::FoundIncludeFile:
+        {
+            // Push the new file onto our stack of input streams
+            // TODO(tfoley): check if we have made our include stack too deep
+            PreprocessorInputStream* inputStream = CreateInputStreamForSource(context->preprocessor, foundSource, foundPath);
+            inputStream->parent = context->preprocessor->inputStream;
+            context->preprocessor->inputStream = inputStream;
+        }
+        break;
+ 
+    case IncludeResult::FoundAutoImportFile:
+        {
+            //
+ 
+            String autoImportName = autoImportModule(
+                context->preprocessor->compileRequest,
+                foundPath,
+                foundSource,
+                GetDirectiveLoc(context));
+ 
+            SourceTextInputStream* inputStream = new SourceTextInputStream();
+ 
+            Token token;
+            token.Type = TokenType::AutoImport;
+            token.Position = GetDirectiveLoc(context);
+            token.flags = 0;
+            token.Content = autoImportName;
+ 
+            inputStream->lexedTokens.mTokens.Add(token);
+ 
+            token.Type = TokenType::EndOfFile;
+            token.flags = TokenFlag::AfterWhitespace | TokenFlag::AtStartOfLine;
+            inputStream->lexedTokens.mTokens.Add(token);
+ 
+            inputStream->tokenReader = TokenReader(inputStream->lexedTokens);
+ 
+            inputStream->parent = context->preprocessor->inputStream;
+            context->preprocessor->inputStream = inputStream;
+        }
+        break;
+    }
+ }
 
 // Handle a `#define` directive
 static void HandleDefineDirective(PreprocessorDirectiveContext* context)
@@ -2007,16 +2063,18 @@ static TokenList ReadAllTokens(
 }
 
 TokenList preprocessSource(
-    String const& source,
-    String const& fileName,
-    DiagnosticSink* sink,
-    IncludeHandler* includeHandler,
-    Dictionary<String, String>  defines,
-    ProgramSyntaxNode*  syntax)
+    String const&               source,
+    String const&               fileName,
+    DiagnosticSink*             sink,
+    IncludeHandler*             includeHandler,
+     Dictionary<String, String> defines,
+    ProgramSyntaxNode*          syntax,
+    CompileRequest*             compileRequest)
 {
     Preprocessor preprocessor;
     InitializePreprocessor(&preprocessor, sink);
     preprocessor.syntax = syntax;
+    preprocessor.compileRequest = compileRequest;
 
     preprocessor.includeHandler = includeHandler;
     for (auto p : defines)