Preprocessor cleanups (#484)

* For a `#error` or `#warning`, read the rest of the line as raw text to include in the error message

* When skipping tokens (e.g., in an `#ifdef`d out block), don't emit errors on invalid characters
  * TODO: we could clearly get more efficient and skip whole raw lines in the future

* Fix an issue when a macro invocation that expands to nothing (zero tokens) is the last thing before a directive. The preprocessor was returning the `#` as an ordinary token, because it has already gone past its test for directives.

1 files changed, 6 insertions, 4 deletions

diff --git a/source/slang/lexer.h b/source/slang/lexer.h
index 23eddf04e..1f2954d27 100644
--- a/source/slang/lexer.h
+++ b/source/slang/lexer.h
@@ -64,8 +64,10 @@ namespace Slang
     typedef unsigned int LexerFlags;
     enum
     {
-        kLexerFlag_InDirective      = 1 << 0,
-        kLexerFlag_ExpectFileName   = 1 << 1,
+        kLexerFlag_InDirective      = 1 << 0, ///< Turn end-of-line and end-of-file into end-of-directive
+        kLexerFlag_ExpectFileName   = 1 << 1, ///< Support `<>` style strings for file paths
+        kLexerFlag_IgnoreInvalid    = 1 << 2, ///< Suppress errors about invalid/unsupported characters
+        kLexerFlag_ExpectDirectiveMessage = 1 << 3, ///< Don't lexer ordinary tokens, and instead consume rest of line as a string
     };
 
     struct Lexer
@@ -77,7 +79,7 @@ namespace Slang
 
         ~Lexer();
 
-        Token lexToken();
+        Token lexToken(LexerFlags extraFlags = 0);
 
         TokenList lexAllTokens();
 
@@ -121,4 +123,4 @@ namespace Slang
     FloatingPointLiteralValue getFloatingPointLiteralValue(Token const& token, String* outSuffix = 0);
 }
 
-#endif
\ No newline at end of file
+#endif