diff options
Diffstat (limited to 'source/slang')
| -rwxr-xr-x | source/slang/slang-compiler.h | 14 | ||||
| -rw-r--r-- | source/slang/slang-diagnostic-defs.h | 1 | ||||
| -rw-r--r-- | source/slang/slang-options.cpp | 4 | ||||
| -rw-r--r-- | source/slang/slang-preprocessor.cpp | 246 | ||||
| -rw-r--r-- | source/slang/slang-source-loc.cpp | 4 | ||||
| -rw-r--r-- | source/slang/slang.cpp | 52 |
6 files changed, 250 insertions, 71 deletions
diff --git a/source/slang/slang-compiler.h b/source/slang/slang-compiler.h index 01bdd8502..9ef75ab86 100755 --- a/source/slang/slang-compiler.h +++ b/source/slang/slang-compiler.h @@ -7,6 +7,8 @@ #include "../core/slang-downstream-compiler.h" #include "../core/slang-archive-file-system.h" +#include "../core/slang-std-writers.h" + #include "../../slang-com-ptr.h" #include "slang-capability.h" @@ -1547,6 +1549,8 @@ namespace Slang bool shouldDumpAST = false; bool shouldDocument = false; + bool outputPreprocessor = false; + /// If true will after lexical analysis output the hierarchy of includes to stdout bool outputIncludes = false; @@ -1564,8 +1568,11 @@ namespace Slang class FrontEndCompileRequest : public CompileRequestBase { public: + /// Note that writers can be parsed as nullptr to disable output, + /// and individual channels set to null to disable them FrontEndCompileRequest( Linkage* linkage, + StdWriters* writers, DiagnosticSink* sink); int addEntryPoint( @@ -1681,6 +1688,8 @@ namespace Slang RefPtr<ComponentType> m_globalAndEntryPointsComponentType; List<RefPtr<ComponentType>> m_unspecializedEntryPoints; + + RefPtr<StdWriters> m_writers; }; /// A visitor for use with `ComponentType`s, allowing dispatch over the concrete subclasses. @@ -2023,7 +2032,7 @@ namespace Slang List<String> const & genericTypeNames); void setWriter(WriterChannel chan, ISlangWriter* writer); - ISlangWriter* getWriter(WriterChannel chan) const { return m_writers[int(chan)]; } + ISlangWriter* getWriter(WriterChannel chan) const { return m_writers->getWriter(SlangWriterChannel(chan)); } /// The end to end request can be passed as nullptr, if not driven by one SlangResult executeActionsInner(); @@ -2066,7 +2075,8 @@ namespace Slang RefPtr<BackEndCompileRequest> m_backEndReq; // For output - ComPtr<ISlangWriter> m_writers[SLANG_WRITER_CHANNEL_COUNT_OF]; + + RefPtr<StdWriters> m_writers; }; void generateOutput( diff --git a/source/slang/slang-diagnostic-defs.h b/source/slang/slang-diagnostic-defs.h index 1739a1ee1..6387ff8b2 100644 --- a/source/slang/slang-diagnostic-defs.h +++ b/source/slang/slang-diagnostic-defs.h @@ -184,6 +184,7 @@ DIAGNOSTIC(15302, Error, noUniqueIdentity, "`#include` handler didn't generate a DIAGNOSTIC(15400, Warning, macroRedefinition, "redefinition of macro '$0'") DIAGNOSTIC(15401, Warning, macroNotDefined, "macro '$0' is not defined") DIAGNOSTIC(15403, Error, expectedTokenInMacroParameters, "expected '$0' in macro parameters") +DIAGNOSTIC(15404, Warning, builtinMacroRedefinition, "Redefinition of builtin macro '$0'") // 155xx - macro expansion DIAGNOSTIC(15500, Warning, expectedTokenInMacroArguments, "expected '$0' in macro invocation") diff --git a/source/slang/slang-options.cpp b/source/slang/slang-options.cpp index ff7923771..fcc2e0e4c 100644 --- a/source/slang/slang-options.cpp +++ b/source/slang/slang-options.cpp @@ -529,6 +529,10 @@ struct OptionsParser requestImpl->getFrontEndReq()->shouldDumpIR = true; requestImpl->getBackEndReq()->shouldDumpIR = true; } + else if (argStr == "-E" || argStr == "-output-preprocessor") + { + requestImpl->getFrontEndReq()->outputPreprocessor = true; + } else if (argStr == "-dump-ast") { requestImpl->getFrontEndReq()->shouldDumpAST = true; diff --git a/source/slang/slang-preprocessor.cpp b/source/slang/slang-preprocessor.cpp index f233d46f0..f9d18332d 100644 --- a/source/slang/slang-preprocessor.cpp +++ b/source/slang/slang-preprocessor.cpp @@ -136,7 +136,7 @@ struct MacroExpansion : PretokenizedInputStream // The macro we will expand PreprocessorMacro* macro; - /// State for marking `macro` as busy in thsi expansion + /// State for marking `macro` as busy in this expansion BusyMacro busy; // Environment for macro expansion. @@ -159,14 +159,24 @@ enum class PreprocessorMacroFlavor ObjectLike, FunctionArg, FunctionLike, + BuiltinLine, /// builtin macro __LINE__ + BuiltinFile, /// builtin macro __FILE__ }; +SLANG_FORCE_INLINE bool isBuiltinMacro(PreprocessorMacroFlavor flavor) +{ + return flavor == PreprocessorMacroFlavor::BuiltinLine || flavor == PreprocessorMacroFlavor::BuiltinFile; +} + // In the current design (which we may want to re-consider), // a macro is a specialized flavor of input stream, that // captures the token list in its expansion, and then // can be "played back." struct PreprocessorMacro { + // The flavor of macro + PreprocessorMacroFlavor flavor; + // The name under which the macro was `#define`d NameLoc nameAndLoc; @@ -176,9 +186,6 @@ struct PreprocessorMacro // The tokens that make up the macro body TokenList tokens; - // The flavor of macro - PreprocessorMacroFlavor flavor; - // The environment in which this macro needs to be expanded. // For ordinary macros this will be the global environment, // while for function-like macro arguments, it will be @@ -229,14 +236,18 @@ struct Preprocessor /// File system to use when looking up files ISlangFileSystemExt* fileSystem = nullptr; - /// Source maanger to use when loading source files + /// Source manager to use when loading source files SourceManager* sourceManager = nullptr; + /// Stores the initiating macro source location. + SourceLoc initiatingMacroSourceLoc; + NamePool* getNamePool() { return namePool; } SourceManager* getSourceManager() { return sourceManager; } }; + static Token AdvanceToken(Preprocessor* preprocessor); // Convenience routine to access the diagnostic sink @@ -497,6 +508,7 @@ static PreprocessorMacro* LookupMacro(PreprocessorEnvironment* environment, Name return NULL; } + static PreprocessorEnvironment* GetCurrentEnvironment(Preprocessor* preprocessor) { // The environment we will use for looking up a macro is associated @@ -530,6 +542,12 @@ static PreprocessorEnvironment* GetCurrentEnvironment(Preprocessor* preprocessor } } +static bool _isInMacroExpansion(Preprocessor* preprocessor) +{ + return preprocessor->inputStream->environment->busyMacros != nullptr; +} + + static PreprocessorMacro* LookupMacro(Preprocessor* preprocessor, Name* name) { return LookupMacro(GetCurrentEnvironment(preprocessor), name); @@ -636,8 +654,15 @@ static void initializeMacroExpansion( static void pushMacroExpansion( Preprocessor* preprocessor, - MacroExpansion* expansion) + MacroExpansion* expansion, + SourceLoc initiatingMacroSourceLoc) { + // Only set the initiating if outside of a macro expansion + if (!_isInMacroExpansion(preprocessor)) + { + preprocessor->initiatingMacroSourceLoc = initiatingMacroSourceLoc; + } + // Before pushing a macro as an input stream, // we need to set the appropraite "busy" state // that will be used during expansions of that @@ -901,7 +926,9 @@ static void MaybeBeginMacroExpansion( // Not a macro? Can't be an invocation. if (!macro) + { return; + } // If the macro is busy (already being expanded), // don't try to trigger recursive expansion @@ -916,76 +943,137 @@ static void MaybeBeginMacroExpansion( // A function-style macro invocation should only match // if the token *after* the identifier is `(`. This // requires more lookahead than we usually have/need - if (macro->flavor == PreprocessorMacroFlavor::FunctionLike) - { - // Consume the token that (possibly) triggered macro expansion - AdvanceRawToken(preprocessor); - // Look at the next token, and see if it is an opening `(` - // that indicates we should actually expand a macro. - if(PeekRawTokenType(preprocessor) != TokenType::LParent) + switch (macro->flavor) + { + case PreprocessorMacroFlavor::FunctionLike: { - // In this case, we are in a bit of a mess, because we have - // consumed the token that named the macro, but we need to - // make sure that token (and not whatever came after it) - // gets returned to the user. - // - // To work around this we will construct a short-lived input - // stream just to handle that one token, and also set - // a flag on the token to keep us from doing this logic again. + // Consume the token that (possibly) triggered macro expansion + AdvanceRawToken(preprocessor); - token.flags |= TokenFlag::SuppressMacroExpansion; + // Look at the next token, and see if it is an opening `(` + // that indicates we should actually expand a macro. + if(PeekRawTokenType(preprocessor) != TokenType::LParent) + { + // In this case, we are in a bit of a mess, because we have + // consumed the token that named the macro, but we need to + // make sure that token (and not whatever came after it) + // gets returned to the user. + // + // To work around this we will construct a short-lived input + // stream just to handle that one token, and also set + // a flag on the token to keep us from doing this logic again. + + token.flags |= TokenFlag::SuppressMacroExpansion; + + SimpleTokenInputStream* simpleStream = createSimpleInputStream(preprocessor, token); + PushInputStream(preprocessor, simpleStream); + return; + } - SimpleTokenInputStream* simpleStream = createSimpleInputStream(preprocessor, token); - PushInputStream(preprocessor, simpleStream); - return; - } + MacroExpansion* expansion = new MacroExpansion(); + initializeMacroExpansion(preprocessor, expansion, macro); + + // Consume the opening `(` + Token leftParen = AdvanceRawToken(preprocessor); - MacroExpansion* expansion = new MacroExpansion(); - initializeMacroExpansion(preprocessor, expansion, macro); + // Parse the arguments to the macro invocation + Index argCount = _parseMacroArgs(preprocessor, macro, expansion); - // Consume the opening `(` - Token leftParen = AdvanceRawToken(preprocessor); + // Expect a closing ')' + if(PeekRawTokenType(preprocessor) == TokenType::RParent) + { + AdvanceRawToken(preprocessor); + } + else + { + GetSink(preprocessor)->diagnose(PeekLoc(preprocessor), Diagnostics::expectedTokenInMacroArguments, TokenType::RParent, PeekRawTokenType(preprocessor)); + } - // Parse the arguments to the macro invocation - Index argCount = _parseMacroArgs(preprocessor, macro, expansion); + // If we didn't parse the expected number of arguments, + // then diagnose an error and do not attempt expansion. + // + // TODO: This check will need to be updated for variadic macros. + // + const Index paramCount = Index(macro->params.getCount()); + if (argCount != paramCount) + { + GetSink(preprocessor)->diagnose(PeekLoc(preprocessor), Diagnostics::wrongNumberOfArgumentsToMacro, paramCount, argCount); + return; + } - // Expect a closing ')' - if(PeekRawTokenType(preprocessor) == TokenType::RParent) + // Now that the arguments have been parsed and validated, + // we are ready to proceed with expansion of the macro body. + // + pushMacroExpansion(preprocessor, expansion, token.loc); + break; + } + case PreprocessorMacroFlavor::FunctionArg: + case PreprocessorMacroFlavor::ObjectLike: { + // Consume the token that triggered macro expansion AdvanceRawToken(preprocessor); + + // Object-like macros are the easy case. + MacroExpansion* expansion = new MacroExpansion(); + initializeMacroExpansion(preprocessor, expansion, macro); + pushMacroExpansion(preprocessor, expansion, token.loc); + break; } - else + case PreprocessorMacroFlavor::BuiltinLine: + case PreprocessorMacroFlavor::BuiltinFile: { - GetSink(preprocessor)->diagnose(PeekLoc(preprocessor), Diagnostics::expectedTokenInMacroArguments, TokenType::RParent, PeekRawTokenType(preprocessor)); - } + const SourceLoc loc = _isInMacroExpansion(preprocessor) ? preprocessor->initiatingMacroSourceLoc : token.loc; - // If we didn't parse the expected number of arguments, - // then diagnose an error and do not attempt expansion. - // - // TODO: This check will need to be updated for variadic macros. - // - const Index paramCount = Index(macro->params.getCount()); - if (argCount != paramCount) - { - GetSink(preprocessor)->diagnose(PeekLoc(preprocessor), Diagnostics::wrongNumberOfArgumentsToMacro, paramCount, argCount); - return; - } + if (!loc.isValid()) + { + // If we don't have a valid source location, don't expand + return; + } - // Now that the arguments have been parsed and validated, - // we are ready to proceed with expansion of the macro body. - // - pushMacroExpansion(preprocessor, expansion); - } - else - { - // Consume the token that triggered macro expansion - AdvanceRawToken(preprocessor); + AdvanceRawToken(preprocessor); + + SourceManager* sourceManager = preprocessor->getSourceManager(); + + // Since the location can be overridden by #line directives, use the slower path to get the line number + const HumaneSourceLoc humaneSourceLoc = sourceManager->getHumaneLoc(loc); + + Token newToken; + + StringBuilder buf; + if (macro->flavor == PreprocessorMacroFlavor::BuiltinLine) + { + newToken.type = TokenType::IntegerLiteral; + buf << humaneSourceLoc.line; + } + else + { + // We need to escape to a string + newToken.type = TokenType::StringLiteral; + + buf.appendChar('"'); + StringUtil::appendEscaped(humaneSourceLoc.pathInfo.foundPath.getUnownedSlice(), buf); + buf.appendChar('"'); + } + + // We are going to keep the actual text in the slice pool, so it stays in scope + // and if the value appears multiple times, it will shared + auto& pool = sourceManager->getStringSlicePool(); + + auto poolHandle = pool.add(buf.getUnownedSlice()); - // Object-like macros are the easy case. - MacroExpansion* expansion = new MacroExpansion(); - initializeMacroExpansion(preprocessor, expansion, macro); - pushMacroExpansion(preprocessor, expansion); + auto slice = pool.getSlice(poolHandle); + + newToken.setContent(slice); + + // We set the location to be the same as where the original location was + newToken.loc = token.loc; + + // Add to the start of the stream + SimpleTokenInputStream* simpleStream = createSimpleInputStream(preprocessor, newToken); + PushInputStream(preprocessor, simpleStream); + break; + } } } } @@ -1907,8 +1995,17 @@ static void HandleDefineDirective(PreprocessorDirectiveContext* context) PreprocessorMacro* oldMacro = LookupMacro(&context->preprocessor->globalEnv, name); if (oldMacro) { - GetSink(context)->diagnose(nameToken.loc, Diagnostics::macroRedefinition, name); - GetSink(context)->diagnose(oldMacro->getLoc(), Diagnostics::seePreviousDefinitionOf, name); + auto sink = GetSink(context); + + if (isBuiltinMacro(oldMacro->flavor)) + { + sink->diagnose(nameToken.loc, Diagnostics::builtinMacroRedefinition, name); + } + else + { + sink->diagnose(nameToken.loc, Diagnostics::macroRedefinition, name); + sink->diagnose(oldMacro->getLoc(), Diagnostics::seePreviousDefinitionOf, name); + } DestroyMacro(context->preprocessor, oldMacro); } @@ -2475,7 +2572,9 @@ Result findMacroValue( MacroExpansion* expansion = new MacroExpansion(); initializeMacroExpansion(preprocessor, expansion, macro); - pushMacroExpansion(preprocessor, expansion); + + // Don't set macro expansion location + pushMacroExpansion(preprocessor, expansion, SourceLoc()); String value; for(bool first = true;;first = false) @@ -2539,6 +2638,25 @@ TokenList preprocessSource( preprocessor.fileSystem = desc.fileSystem; preprocessor.namePool = desc.namePool; + // Add builtin macros + { + auto namePool = desc.namePool; + + const char*const builtinNames[] = { "__FILE__", "__LINE__" }; + const PreprocessorMacroFlavor builtinFlavors[] = { PreprocessorMacroFlavor::BuiltinFile, PreprocessorMacroFlavor::BuiltinLine }; + + for (Index i = 0; i < SLANG_COUNT_OF(builtinNames); i++) + { + auto name = namePool->getName(builtinNames[i]); + + PreprocessorMacro* macro = CreateMacro(&preprocessor); + macro->flavor = builtinFlavors[i]; + macro->nameAndLoc = NameLoc(name); + + preprocessor.globalEnv.macros[name] = macro; + } + } + auto sourceManager = desc.sourceManager; preprocessor.sourceManager = sourceManager; diff --git a/source/slang/slang-source-loc.cpp b/source/slang/slang-source-loc.cpp index 3f9f8ad31..4b589fbf3 100644 --- a/source/slang/slang-source-loc.cpp +++ b/source/slang/slang-source-loc.cpp @@ -66,9 +66,9 @@ void PathInfo::appendDisplayName(StringBuilder& out) const case Type::FromString: case Type::FoundPath: { - // TODO(JS): We might want to escape the path here if necessary + out.appendChar('"'); - out << foundPath; + StringUtil::appendEscaped(foundPath.getUnownedSlice(), out); out.appendChar('"'); break; } diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp index 93d7be310..bcc3d1c67 100644 --- a/source/slang/slang.cpp +++ b/source/slang/slang.cpp @@ -1186,7 +1186,7 @@ static ISlangWriter* _getDefaultWriter(WriterChannel chan) void EndToEndCompileRequest::setWriter(WriterChannel chan, ISlangWriter* writer) { // If the user passed in null, we will use the default writer on that channel - m_writers[int(chan)] = writer ? writer : _getDefaultWriter(chan); + m_writers->setWriter(SlangWriterChannel(chan), writer ? writer : _getDefaultWriter(chan)); // For diagnostic output, if the user passes in nullptr, we set on mSink.writer as that enables buffering on DiagnosticSink if (chan == WriterChannel::Diagnostic) @@ -1307,8 +1307,10 @@ CompileRequestBase::CompileRequestBase( FrontEndCompileRequest::FrontEndCompileRequest( Linkage* linkage, + StdWriters* writers, DiagnosticSink* sink) : CompileRequestBase(linkage, sink) + , m_writers(writers) { } @@ -1540,6 +1542,26 @@ static void _outputIncludesRec(SourceView* sourceView, Index depth, ViewInitiati } } +static void _outputPreprocessorTokens(const TokenList& toks, ISlangWriter* writer) +{ + if (writer == nullptr) + { + return; + } + + StringBuilder buf; + for (const auto& tok : toks) + { + buf << tok.getContent(); + // We'll separate tokens with space for now + buf.appendChar(' '); + } + + buf.appendChar('\n'); + + writer->write(buf.getBuffer(), buf.getLength()); +} + static void _outputIncludes(const List<SourceFile*>& sourceFiles, SourceManager* sourceManager, DiagnosticSink* sink) { // Set up the hierarchy to know how all the source views relate. This could be argued as overkill, but makes recursive @@ -1651,6 +1673,16 @@ void FrontEndCompileRequest::parseTranslationUnit( _outputIncludes(translationUnit->getSourceFiles(), getSink()->getSourceManager(), getSink()); } + if (outputPreprocessor) + { + if (m_writers) + { + _outputPreprocessorTokens(tokens, m_writers->getWriter(SLANG_WRITER_CHANNEL_STD_OUTPUT)); + } + // If we output the preprocessor output then we are done doing anything else + return; + } + parseSourceFile( astBuilder, translationUnit, @@ -1828,6 +1860,12 @@ SlangResult FrontEndCompileRequest::executeActionsInner() parseTranslationUnit(translationUnit.Ptr()); } + if (outputPreprocessor) + { + // If doing pre-processor output, then we are done + return SLANG_OK; + } + if (getSink()->getErrorCount() != 0) return SLANG_FAIL; @@ -1954,13 +1992,15 @@ void EndToEndCompileRequest::init() { m_sink.setSourceManager(m_linkage->getSourceManager()); + m_writers = new StdWriters; + // Set all the default writers for (int i = 0; i < int(WriterChannel::CountOf); ++i) { setWriter(WriterChannel(i), nullptr); } - m_frontEndReq = new FrontEndCompileRequest(getLinkage(), getSink()); + m_frontEndReq = new FrontEndCompileRequest(getLinkage(), m_writers, getSink()); m_backEndReq = new BackEndCompileRequest(getLinkage(), getSink()); } @@ -1999,6 +2039,11 @@ SlangResult EndToEndCompileRequest::executeActionsInner() SLANG_RETURN_ON_FAIL(getFrontEndReq()->executeActionsInner()); } + if (getFrontEndReq()->outputPreprocessor) + { + return SLANG_OK; + } + // If command line specifies to skip codegen, we exit here. // Note: this is a debugging option. // @@ -2288,7 +2333,7 @@ RefPtr<Module> Linkage::loadModule( SourceLoc const& srcLoc, DiagnosticSink* sink) { - RefPtr<FrontEndCompileRequest> frontEndReq = new FrontEndCompileRequest(this, sink); + RefPtr<FrontEndCompileRequest> frontEndReq = new FrontEndCompileRequest(this, nullptr, sink); RefPtr<TranslationUnitRequest> translationUnit = new TranslationUnitRequest(frontEndReq); translationUnit->compileRequest = frontEndReq; @@ -3674,6 +3719,7 @@ void Session::addBuiltinSource( DiagnosticSink sink(sourceManager, Lexer::sourceLocationLexer); RefPtr<FrontEndCompileRequest> compileRequest = new FrontEndCompileRequest( m_builtinLinkage, + nullptr, &sink); compileRequest->m_isStandardLibraryCode = true; |