diff options
Diffstat (limited to 'source')
| -rw-r--r-- | source/core/slang-string.cpp | 6 | ||||
| -rw-r--r-- | source/core/slang-string.h | 2 | ||||
| -rw-r--r-- | source/slang/compiler.cpp | 8 | ||||
| -rw-r--r-- | source/slang/compiler.h | 45 | ||||
| -rw-r--r-- | source/slang/diagnostics.cpp | 29 | ||||
| -rw-r--r-- | source/slang/diagnostics.h | 37 | ||||
| -rw-r--r-- | source/slang/emit.cpp | 125 | ||||
| -rw-r--r-- | source/slang/lexer.cpp | 81 | ||||
| -rw-r--r-- | source/slang/lexer.h | 30 | ||||
| -rw-r--r-- | source/slang/lower.cpp | 16 | ||||
| -rw-r--r-- | source/slang/options.cpp | 8 | ||||
| -rw-r--r-- | source/slang/parser.cpp | 22 | ||||
| -rw-r--r-- | source/slang/parser.h | 1 | ||||
| -rw-r--r-- | source/slang/preprocessor.cpp | 383 | ||||
| -rw-r--r-- | source/slang/preprocessor.h | 3 | ||||
| -rw-r--r-- | source/slang/slang.cpp | 55 | ||||
| -rw-r--r-- | source/slang/slang.vcxproj | 1 | ||||
| -rw-r--r-- | source/slang/slang.vcxproj.filters | 1 | ||||
| -rw-r--r-- | source/slang/source-loc.cpp | 332 | ||||
| -rw-r--r-- | source/slang/source-loc.h | 201 | ||||
| -rw-r--r-- | source/slang/syntax-base-defs.h | 2 | ||||
| -rw-r--r-- | source/slang/syntax-visitors.h | 2 | ||||
| -rw-r--r-- | source/slang/token.h | 17 |
23 files changed, 1027 insertions, 380 deletions
diff --git a/source/core/slang-string.cpp b/source/core/slang-string.cpp index 459396f69..9bc9e3a54 100644 --- a/source/core/slang-string.cpp +++ b/source/core/slang-string.cpp @@ -51,6 +51,12 @@ namespace Slang , endIndex(0) {} + StringSlice::StringSlice(String const& str) + : representation(str.buffer) + , beginIndex(0) + , endIndex(str.Length()) + {} + StringSlice::StringSlice(String const& str, UInt beginIndex, UInt endIndex) : representation(str.buffer) , beginIndex(beginIndex) diff --git a/source/core/slang-string.h b/source/core/slang-string.h index 552afe833..98fd51a07 100644 --- a/source/core/slang-string.h +++ b/source/core/slang-string.h @@ -155,6 +155,8 @@ namespace Slang public: StringSlice(); + StringSlice(String const& str); + StringSlice(String const& str, UInt beginIndex, UInt endIndex); UInt getLength() const diff --git a/source/slang/compiler.cpp b/source/slang/compiler.cpp index 60f111d83..55b79c00e 100644 --- a/source/slang/compiler.cpp +++ b/source/slang/compiler.cpp @@ -196,7 +196,7 @@ namespace Slang HMODULE d3dCompiler = LoadLibraryA(libraryName); if (!d3dCompiler) { - request->mSink.diagnose(CodePosition(), Diagnostics::failedToLoadDynamicLibrary, libraryName); + request->mSink.diagnose(SourceLoc(), Diagnostics::failedToLoadDynamicLibrary, libraryName); } return d3dCompiler; } @@ -373,7 +373,7 @@ namespace Slang HMODULE glslCompiler = LoadLibraryA(libraryName); if (!glslCompiler) { - request->mSink.diagnose(CodePosition(), Diagnostics::failedToLoadDynamicLibrary, libraryName); + request->mSink.diagnose(SourceLoc(), Diagnostics::failedToLoadDynamicLibrary, libraryName); } return glslCompiler; } @@ -611,7 +611,7 @@ namespace Slang if (count != 1) { compileRequest->mSink.diagnose( - CodePosition(), + SourceLoc(), Diagnostics::cannotWriteOutputFile, path); } @@ -630,7 +630,7 @@ namespace Slang if (!file) { compileRequest->mSink.diagnose( - CodePosition(), + SourceLoc(), Diagnostics::cannotWriteOutputFile, path); return; diff --git a/source/slang/compiler.h b/source/slang/compiler.h index fd40a62f6..5a6cb5c19 100644 --- a/source/slang/compiler.h +++ b/source/slang/compiler.h @@ -122,17 +122,7 @@ namespace Slang // GLSL, // pass through GLSL to `glslang` library }; - // Represents a single source file (either an on-disk file, or a - // "virtual" file passed in as a string) - class SourceFile : public RefObject - { - public: - // The file path for a real file, or the nominal path for a virtual file - String path; - - // The actual contents of the file - String content; - }; + class SourceFile; // A single translation unit requested to be compiled. // @@ -228,6 +218,10 @@ namespace Slang // Are we being driven by the command-line `slangc`, and should act accordingly? bool isCommandLineCompile = false; + // Source manager to help track files loaded + SourceManager sourceManagerStorage; + SourceManager* sourceManager; + // Output stuff DiagnosticSink mSink; String mDiagnosticOutput; @@ -250,12 +244,9 @@ namespace Slang Dictionary<String, RefPtr<ModuleDecl>> mapNameToLoadedModules; - CompileRequest(Session* session) - : mSession(session) - {} + CompileRequest(Session* session); - ~CompileRequest() - {} + ~CompileRequest(); void parseTranslationUnit( TranslationUnitRequest* translationUnit); @@ -267,6 +258,10 @@ namespace Slang int addTranslationUnit(SourceLanguage language, String const& name); + void addTranslationUnitSourceFile( + int translationUnitIndex, + SourceFile* sourceFile); + void addTranslationUnitSourceString( int translationUnitIndex, String const& path, @@ -285,7 +280,7 @@ namespace Slang String const& name, String const& path, String const& source, - CodePosition const& loc); + SourceLoc const& loc); void handlePoundImport( String const& path, @@ -293,7 +288,18 @@ namespace Slang RefPtr<ModuleDecl> findOrImportModule( String const& name, - CodePosition const& loc); + SourceLoc const& loc); + + SourceManager* getSourceManager() + { + return sourceManager; + } + + void setSourceManager(SourceManager* sm) + { + sourceManager = sm; + mSink.sourceManager = sm; + } }; void generateOutput( @@ -324,6 +330,9 @@ namespace Slang List<RefPtr<ModuleDecl>> loadedModuleCode; + SourceManager builtinSourceManager; + + SourceManager* getBuiltinSourceManager() { return &builtinSourceManager; } // diff --git a/source/slang/diagnostics.cpp b/source/slang/diagnostics.cpp index e0b959a81..eb6a818d3 100644 --- a/source/slang/diagnostics.cpp +++ b/source/slang/diagnostics.cpp @@ -1,7 +1,8 @@ -// Diagnostics.cpp -#include "Diagnostics.h" +// diagnostics.cpp +#include "diagnostics.h" -#include "Syntax.h" +#include "compiler.h" +#include "syntax.h" #include <assert.h> @@ -67,17 +68,17 @@ void printDiagnosticArg(StringBuilder& sb, Token const& token) sb << token.Content; } -CodePosition const& getDiagnosticPos(SyntaxNode const* syntax) +SourceLoc const& getDiagnosticPos(SyntaxNode const* syntax) { return syntax->Position; } -CodePosition const& getDiagnosticPos(Token const& token) +SourceLoc const& getDiagnosticPos(Token const& token) { return token.Position; } -CodePosition const& getDiagnosticPos(TypeExp const& typeExp) +SourceLoc const& getDiagnosticPos(TypeExp const& typeExp) { return typeExp.exp->Position; } @@ -140,12 +141,18 @@ static void formatDiagnosticMessage(StringBuilder& sb, char const* format, int a } static void formatDiagnostic( + DiagnosticSink* sink, StringBuilder& sb, Diagnostic const& diagnostic) { - sb << diagnostic.Position.FileName; + auto sourceManager = sink->sourceManager; + + auto expandedLoc = sourceManager->expandSourceLoc(diagnostic.Position); + auto humaneLoc = sourceManager->getHumaneLoc(expandedLoc); + + sb << humaneLoc.getPath(); sb << "("; - sb << diagnostic.Position.Line; + sb << humaneLoc.line; sb << "): "; sb << getSeverityName(diagnostic.severity); sb << " "; @@ -155,7 +162,7 @@ static void formatDiagnostic( sb << "\n"; } -void DiagnosticSink::diagnoseImpl(CodePosition const& pos, DiagnosticInfo const& info, int argCount, DiagnosticArg const* const* args) +void DiagnosticSink::diagnoseImpl(SourceLoc const& pos, DiagnosticInfo const& info, int argCount, DiagnosticArg const* const* args) { StringBuilder sb; formatDiagnosticMessage(sb, info.messageFormat, argCount, args); @@ -176,7 +183,7 @@ void DiagnosticSink::diagnoseImpl(CodePosition const& pos, DiagnosticInfo const& { // If so, pass the error string along to them StringBuilder messageBuilder; - formatDiagnostic(messageBuilder, diagnostic); + formatDiagnostic(this, messageBuilder, diagnostic); callback(messageBuilder.ProduceString().begin(), callbackUserData); } @@ -184,7 +191,7 @@ void DiagnosticSink::diagnoseImpl(CodePosition const& pos, DiagnosticInfo const& { // If the user doesn't have a callback, then just // collect our diagnostic messages into a buffer - formatDiagnostic(outputBuffer, diagnostic); + formatDiagnostic(this, outputBuffer, diagnostic); } if (diagnostic.severity >= Severity::Fatal) diff --git a/source/slang/diagnostics.h b/source/slang/diagnostics.h index 6ea677b38..e22320d3d 100644 --- a/source/slang/diagnostics.h +++ b/source/slang/diagnostics.h @@ -46,7 +46,7 @@ namespace Slang { public: String Message; - CodePosition Position; + SourceLoc Position; int ErrorID; Severity severity; @@ -57,7 +57,7 @@ namespace Slang Diagnostic( const String & msg, int id, - const CodePosition & pos, + const SourceLoc & pos, Severity severity) : severity(severity) { @@ -93,16 +93,16 @@ namespace Slang printDiagnosticArg(sb, ptr.Ptr()); } - inline CodePosition const& getDiagnosticPos(CodePosition const& pos) { return pos; } + inline SourceLoc const& getDiagnosticPos(SourceLoc const& pos) { return pos; } class SyntaxNode; class ShaderClosure; - CodePosition const& getDiagnosticPos(SyntaxNode const* syntax); - CodePosition const& getDiagnosticPos(Token const& token); - CodePosition const& getDiagnosticPos(TypeExp const& typeExp); + SourceLoc const& getDiagnosticPos(SyntaxNode const* syntax); + SourceLoc const& getDiagnosticPos(Token const& token); + SourceLoc const& getDiagnosticPos(TypeExp const& typeExp); template<typename T> - CodePosition getDiagnosticPos(RefPtr<T> const& ptr) + SourceLoc getDiagnosticPos(RefPtr<T> const& ptr) { return getDiagnosticPos(ptr.Ptr()); } @@ -128,6 +128,9 @@ namespace Slang class DiagnosticSink { public: + // The source manager to use when mapping source locations to file+line info + SourceManager* sourceManager; + StringBuilder outputBuffer; // List<Diagnostic> diagnostics; int errorCount = 0; @@ -136,43 +139,43 @@ namespace Slang void* callbackUserData = nullptr; /* - void Error(int id, const String & msg, const CodePosition & pos) + void Error(int id, const String & msg, const SourceLoc & pos) { diagnostics.Add(Diagnostic(msg, id, pos, Severity::Error)); errorCount++; } - void Warning(int id, const String & msg, const CodePosition & pos) + void Warning(int id, const String & msg, const SourceLoc & pos) { diagnostics.Add(Diagnostic(msg, id, pos, Severity::Warning)); } */ int GetErrorCount() { return errorCount; } - void diagnoseDispatch(CodePosition const& pos, DiagnosticInfo const& info) + void diagnoseDispatch(SourceLoc const& pos, DiagnosticInfo const& info) { diagnoseImpl(pos, info, 0, NULL); } - void diagnoseDispatch(CodePosition const& pos, DiagnosticInfo const& info, DiagnosticArg const& arg0) + void diagnoseDispatch(SourceLoc const& pos, DiagnosticInfo const& info, DiagnosticArg const& arg0) { DiagnosticArg const* args[] = { &arg0 }; diagnoseImpl(pos, info, 1, args); } - void diagnoseDispatch(CodePosition const& pos, DiagnosticInfo const& info, DiagnosticArg const& arg0, DiagnosticArg const& arg1) + void diagnoseDispatch(SourceLoc const& pos, DiagnosticInfo const& info, DiagnosticArg const& arg0, DiagnosticArg const& arg1) { DiagnosticArg const* args[] = { &arg0, &arg1 }; diagnoseImpl(pos, info, 2, args); } - void diagnoseDispatch(CodePosition const& pos, DiagnosticInfo const& info, DiagnosticArg const& arg0, DiagnosticArg const& arg1, DiagnosticArg const& arg2) + void diagnoseDispatch(SourceLoc const& pos, DiagnosticInfo const& info, DiagnosticArg const& arg0, DiagnosticArg const& arg1, DiagnosticArg const& arg2) { DiagnosticArg const* args[] = { &arg0, &arg1, &arg2 }; diagnoseImpl(pos, info, 3, args); } - void diagnoseDispatch(CodePosition const& pos, DiagnosticInfo const& info, DiagnosticArg const& arg0, DiagnosticArg const& arg1, DiagnosticArg const& arg2, DiagnosticArg const& arg3) + void diagnoseDispatch(SourceLoc const& pos, DiagnosticInfo const& info, DiagnosticArg const& arg0, DiagnosticArg const& arg1, DiagnosticArg const& arg2, DiagnosticArg const& arg3) { DiagnosticArg const* args[] = { &arg0, &arg1, &arg2, &arg3 }; diagnoseImpl(pos, info, 4, args); @@ -184,7 +187,7 @@ namespace Slang diagnoseDispatch(getDiagnosticPos(pos), info, args...); } - void diagnoseImpl(CodePosition const& pos, DiagnosticInfo const& info, int argCount, DiagnosticArg const* const* args); + void diagnoseImpl(SourceLoc const& pos, DiagnosticInfo const& info, int argCount, DiagnosticArg const* const* args); // Add a diagnostic with raw text // (used when we get errors from a downstream compiler) @@ -202,9 +205,9 @@ namespace Slang #ifdef _DEBUG #define SLANG_INTERNAL_ERROR(sink, pos) \ - (sink)->diagnose(Slang::CodePosition(__LINE__, 0, 0, __FILE__), Slang::Diagnostics::internalCompilerError) + (sink)->diagnose(Slang::SourceLoc(__LINE__, 0, 0, __FILE__), Slang::Diagnostics::internalCompilerError) #define SLANG_UNIMPLEMENTED(sink, pos, what) \ - (sink)->diagnose(Slang::CodePosition(__LINE__, 0, 0, __FILE__), Slang::Diagnostics::unimplemented, what) + (sink)->diagnose(Slang::SourceLoc(__LINE__, 0, 0, __FILE__), Slang::Diagnostics::unimplemented, what) #else #define SLANG_INTERNAL_ERROR(sink, pos) \ diff --git a/source/slang/emit.cpp b/source/slang/emit.cpp index 72632260f..2a5f40787 100644 --- a/source/slang/emit.cpp +++ b/source/slang/emit.cpp @@ -58,8 +58,8 @@ struct SharedEmitContext StringBuilder sb; // Current source position for tracking purposes... - CodePosition loc; - CodePosition nextSourceLocation; + HumaneSourceLoc loc; + HumaneSourceLoc nextSourceLocation; bool needToUpdateSourceLocation; // For GLSL output, we can't emit traidtional `#line` directives @@ -344,7 +344,7 @@ struct EDeclarator // Used for `Flavor::Name` String name; - CodePosition loc; + SourceLoc loc; // Used for `Flavor::Array` IntVal* elementCount; @@ -407,7 +407,7 @@ struct EmitVisitor // Update our logical position // TODO(tfoley): Need to make "corelib" not use `int` for pointer-sized things... auto len = int(textEnd - textBegin); - context->shared->loc.Col += len; + context->shared->loc.column += len; } void Emit(char const* textBegin, char const* textEnd) @@ -431,8 +431,8 @@ struct EmitVisitor // At the end of a line, we need to update our tracking // information on code positions emitTextSpan(spanBegin, spanEnd); - context->shared->loc.Line++; - context->shared->loc.Col = 1; + context->shared->loc.line++; + context->shared->loc.column = 1; // Start a new span for emit purposes spanBegin = spanEnd; @@ -452,7 +452,7 @@ struct EmitVisitor void emitName( String const& inName, - CodePosition const& loc) + SourceLoc const& loc) { String name = inName; @@ -467,7 +467,7 @@ struct EmitVisitor void emitName(String const& name) { - emitName(name, CodePosition()); + emitName(name, SourceLoc()); } void Emit(IntegerLiteralValue value) @@ -504,12 +504,12 @@ struct EmitVisitor // Emit a `#line` directive to the output. // Doesn't udpate state of source-location tracking. void emitLineDirective( - CodePosition const& sourceLocation) + HumaneSourceLoc const& sourceLocation) { emitRawText("\n#line "); char buffer[16]; - sprintf(buffer, "%d", sourceLocation.Line); + sprintf(buffer, "%d", sourceLocation.line); emitRawText(buffer); emitRawText(" "); @@ -547,7 +547,7 @@ struct EmitVisitor if(shouldUseGLSLStyleLineDirective) { - auto path = sourceLocation.FileName; + auto path = sourceLocation.getPath(); // GLSL doesn't support the traditional form of a `#line` directive without // an extension. Rather than depend on that extension we will output @@ -578,7 +578,7 @@ struct EmitVisitor // in a module that tracks source files. emitRawText("\""); - for(auto c : sourceLocation.FileName) + for(auto c : sourceLocation.getPath()) { char charBuffer[] = { c, 0 }; switch(c) @@ -607,17 +607,18 @@ struct EmitVisitor // ensure that source location tracking information // is correct based on the directive we just output. void emitLineDirectiveAndUpdateSourceLocation( - CodePosition const& sourceLocation) + HumaneSourceLoc const& sourceLocation) { emitLineDirective(sourceLocation); - - context->shared->loc.FileName = sourceLocation.FileName; - context->shared->loc.Line = sourceLocation.Line; - context->shared->loc.Col = 1; + + HumaneSourceLoc newLoc = sourceLocation; + newLoc.column = 1; + + context->shared->loc = newLoc; } void emitLineDirectiveIfNeeded( - CodePosition const& sourceLocation) + HumaneSourceLoc const& sourceLocation) { // Don't do any of this work if the user has requested that we // not emit line directives. @@ -626,31 +627,31 @@ struct EmitVisitor return; // Ignore invalid source locations - if(sourceLocation.Line <= 0) + if(sourceLocation.line <= 0) return; // If we are currently emitting code at a source location with // a differnet file or line, *or* if the source location is // somehow later on the line than what we want to emit, // then we need to emit a new `#line` directive. - if(sourceLocation.FileName != context->shared->loc.FileName - || sourceLocation.Line != context->shared->loc.Line - || sourceLocation.Col < context->shared->loc.Col) + if(sourceLocation.path != context->shared->loc.path + || sourceLocation.line != context->shared->loc.line + || sourceLocation.column < context->shared->loc.column) { // Special case: if we are in the same file, and within a small number // of lines of the target location, then go ahead and output newlines // to get us caught up. enum { kSmallLineCount = 3 }; - auto lineDiff = sourceLocation.Line - context->shared->loc.Line; - if(sourceLocation.FileName == context->shared->loc.FileName - && sourceLocation.Line > context->shared->loc.Line + auto lineDiff = sourceLocation.line - context->shared->loc.line; + if(sourceLocation.path == context->shared->loc.path + && sourceLocation.line > context->shared->loc.line && lineDiff <= kSmallLineCount) { for(int ii = 0; ii < lineDiff; ++ii ) { Emit("\n"); } - SLANG_RELEASE_ASSERT(sourceLocation.Line == context->shared->loc.Line); + SLANG_RELEASE_ASSERT(sourceLocation.line == context->shared->loc.line); } else { @@ -666,28 +667,39 @@ struct EmitVisitor // came in as spaces or tabs, so there is necessarily going to be // coupling between how the downstream compiler counts columns, // and how we do. - if(sourceLocation.Col > context->shared->loc.Col) + if(sourceLocation.column > context->shared->loc.column) { - int delta = sourceLocation.Col - context->shared->loc.Col; + int delta = sourceLocation.column - context->shared->loc.column; for( int ii = 0; ii < delta; ++ii ) { emitRawText(" "); } - context->shared->loc.Col = sourceLocation.Col; + context->shared->loc.column = sourceLocation.column; } } void advanceToSourceLocation( - CodePosition const& sourceLocation) + HumaneSourceLoc const& sourceLocation) { // Skip invalid locations - if(sourceLocation.Line <= 0) + if(sourceLocation.line <= 0) return; context->shared->needToUpdateSourceLocation = true; context->shared->nextSourceLocation = sourceLocation; } + SourceManager* getSourceManager() + { + return context->shared->entryPoint->compileRequest->getSourceManager(); + } + + void advanceToSourceLocation( + SourceLoc const& sourceLocation) + { + advanceToSourceLocation(getSourceManager()->getHumaneLoc(sourceLocation)); + } + void flushSourceLocationChange() { if(!context->shared->needToUpdateSourceLocation) @@ -707,9 +719,8 @@ struct EmitVisitor if (mode == LineDirectiveMode::None) return; - if ((mode == LineDirectiveMode::None) - || token.Position.FileName.Length() == 0) + || !token.Position.isValid()) { // If we don't have the original position info, or we are in the // mode where the user didn't want line directives, we need to play @@ -755,7 +766,7 @@ struct EmitVisitor } else { - SLANG_DIAGNOSE_UNEXPECTED(getSink(), CodePosition(), "unknown type of integer constant value"); + SLANG_DIAGNOSE_UNEXPECTED(getSink(), SourceLoc(), "unknown type of integer constant value"); } } @@ -787,7 +798,7 @@ struct EmitVisitor break; default: - SLANG_DIAGNOSE_UNEXPECTED(getSink(), CodePosition(), "unknown declarator flavor"); + SLANG_DIAGNOSE_UNEXPECTED(getSink(), SourceLoc(), "unknown declarator flavor"); break; } } @@ -808,7 +819,7 @@ struct EmitVisitor case BaseType::Bool: Emit("b"); break; case BaseType::Double: Emit("d"); break; default: - SLANG_DIAGNOSE_UNEXPECTED(getSink(), CodePosition(), "unhandled GLSL type prefix"); + SLANG_DIAGNOSE_UNEXPECTED(getSink(), SourceLoc(), "unhandled GLSL type prefix"); break; } } @@ -822,7 +833,7 @@ struct EmitVisitor } else { - SLANG_DIAGNOSE_UNEXPECTED(getSink(), CodePosition(), "unhandled GLSL type prefix"); + SLANG_DIAGNOSE_UNEXPECTED(getSink(), SourceLoc(), "unhandled GLSL type prefix"); } } @@ -851,7 +862,7 @@ struct EmitVisitor break; default: - SLANG_DIAGNOSE_UNEXPECTED(getSink(), CodePosition(), "unhandled resource access mode"); + SLANG_DIAGNOSE_UNEXPECTED(getSink(), SourceLoc(), "unhandled resource access mode"); break; } @@ -863,7 +874,7 @@ struct EmitVisitor case TextureType::ShapeCube: Emit("TextureCube"); break; case TextureType::ShapeBuffer: Emit("Buffer"); break; default: - SLANG_DIAGNOSE_UNEXPECTED(getSink(), CodePosition(), "unhandled resource shape"); + SLANG_DIAGNOSE_UNEXPECTED(getSink(), SourceLoc(), "unhandled resource shape"); break; } @@ -895,7 +906,7 @@ struct EmitVisitor case TextureType::ShapeCube: Emit("Cube"); break; case TextureType::ShapeBuffer: Emit("Buffer"); break; default: - SLANG_DIAGNOSE_UNEXPECTED(getSink(), CodePosition(), "unhandled resource shape"); + SLANG_DIAGNOSE_UNEXPECTED(getSink(), SourceLoc(), "unhandled resource shape"); break; } @@ -941,7 +952,7 @@ struct EmitVisitor break; default: - SLANG_DIAGNOSE_UNEXPECTED(getSink(), CodePosition(), "unhandled code generation target"); + SLANG_DIAGNOSE_UNEXPECTED(getSink(), SourceLoc(), "unhandled code generation target"); break; } } @@ -956,7 +967,7 @@ struct EmitVisitor break; default: - SLANG_DIAGNOSE_UNEXPECTED(getSink(), CodePosition(), "this target should see combined texture-sampler types"); + SLANG_DIAGNOSE_UNEXPECTED(getSink(), SourceLoc(), "this target should see combined texture-sampler types"); break; } } @@ -975,7 +986,7 @@ struct EmitVisitor break; default: - SLANG_DIAGNOSE_UNEXPECTED(getSink(), CodePosition(), "this target should see GLSL image types"); + SLANG_DIAGNOSE_UNEXPECTED(getSink(), SourceLoc(), "this target should see GLSL image types"); break; } } @@ -1026,7 +1037,7 @@ struct EmitVisitor case BaseType::Bool: Emit("bool"); break; case BaseType::Double: Emit("double"); break; default: - SLANG_DIAGNOSE_UNEXPECTED(getSink(), CodePosition(), "unhandled scalar type"); + SLANG_DIAGNOSE_UNEXPECTED(getSink(), SourceLoc(), "unhandled scalar type"); break; } @@ -1058,7 +1069,7 @@ struct EmitVisitor break; default: - SLANG_DIAGNOSE_UNEXPECTED(getSink(), CodePosition(), "unhandled code generation target"); + SLANG_DIAGNOSE_UNEXPECTED(getSink(), SourceLoc(), "unhandled code generation target"); break; } @@ -1096,7 +1107,7 @@ struct EmitVisitor break; default: - SLANG_DIAGNOSE_UNEXPECTED(getSink(), CodePosition(), "unhandled code generation target"); + SLANG_DIAGNOSE_UNEXPECTED(getSink(), SourceLoc(), "unhandled code generation target"); break; } @@ -1136,7 +1147,7 @@ struct EmitVisitor case SamplerStateType::Flavor::SamplerState: Emit("SamplerState"); break; case SamplerStateType::Flavor::SamplerComparisonState: Emit("SamplerComparisonState"); break; default: - SLANG_DIAGNOSE_UNEXPECTED(getSink(), CodePosition(), "unhandled sampler state flavor"); + SLANG_DIAGNOSE_UNEXPECTED(getSink(), SourceLoc(), "unhandled sampler state flavor"); break; } break; @@ -1147,7 +1158,7 @@ struct EmitVisitor case SamplerStateType::Flavor::SamplerState: Emit("sampler"); break; case SamplerStateType::Flavor::SamplerComparisonState: Emit("samplerShadow"); break; default: - SLANG_DIAGNOSE_UNEXPECTED(getSink(), CodePosition(), "unhandled sampler state flavor"); + SLANG_DIAGNOSE_UNEXPECTED(getSink(), SourceLoc(), "unhandled sampler state flavor"); break; } break; @@ -1187,9 +1198,9 @@ struct EmitVisitor void EmitType( RefPtr<Type> type, - CodePosition const& typeLoc, + SourceLoc const& typeLoc, String const& name, - CodePosition const& nameLoc) + SourceLoc const& nameLoc) { advanceToSourceLocation(typeLoc); @@ -1203,7 +1214,7 @@ struct EmitVisitor void EmitType(RefPtr<Type> type, Token const& nameToken) { - EmitType(type, CodePosition(), nameToken.Content, nameToken.Position); + EmitType(type, SourceLoc(), nameToken.Content, nameToken.Position); } void EmitType(RefPtr<Type> type) @@ -1232,13 +1243,13 @@ struct EmitVisitor } } - void EmitType(TypeExp const& typeExp, String const& name, CodePosition const& nameLoc) + void EmitType(TypeExp const& typeExp, String const& name, SourceLoc const& nameLoc) { if (!typeExp.type || typeExp.type->As<ErrorType>()) { if (!typeExp.exp) { - SLANG_DIAGNOSE_UNEXPECTED(getSink(), CodePosition(), "unresolved type expression should have expression part"); + SLANG_DIAGNOSE_UNEXPECTED(getSink(), SourceLoc(), "unresolved type expression should have expression part"); } EDeclarator nameDeclarator; @@ -1251,7 +1262,7 @@ struct EmitVisitor else { EmitType(typeExp.type, - typeExp.exp ? typeExp.exp->Position : CodePosition(), + typeExp.exp ? typeExp.exp->Position : SourceLoc(), name, nameLoc); } } @@ -1263,7 +1274,7 @@ struct EmitVisitor void EmitType(TypeExp const& typeExp, String const& name) { - EmitType(typeExp, name, CodePosition()); + EmitType(typeExp, name, SourceLoc()); } void emitTypeExp(TypeExp const& typeExp) @@ -1463,7 +1474,7 @@ struct EmitVisitor switch(context->shared->target) { default: - SLANG_DIAGNOSE_UNEXPECTED(getSink(), CodePosition(), "unhandled code generation target"); + SLANG_DIAGNOSE_UNEXPECTED(getSink(), SourceLoc(), "unhandled code generation target"); return false; case CodeGenTarget::GLSL: return targetName == "glsl"; @@ -3203,7 +3214,7 @@ struct EmitVisitor Emit("s"); break; default: - SLANG_DIAGNOSE_UNEXPECTED(getSink(), CodePosition(), "unhandled HLSL register type"); + SLANG_DIAGNOSE_UNEXPECTED(getSink(), SourceLoc(), "unhandled HLSL register type"); break; } Emit(info.index); diff --git a/source/slang/lexer.cpp b/source/slang/lexer.cpp index 84b34c9a9..fe01878d1 100644 --- a/source/slang/lexer.cpp +++ b/source/slang/lexer.cpp @@ -1,4 +1,12 @@ -#include "Lexer.h" +// lexer.cpp +#include "lexer.h" + +// This file implements the lexer/scanner, which is responsible for taking a raw stream of +// input bytes and turning it into semantically useful tokens. +// + +#include "compiler.h" +#include "source-loc.h" #include <assert.h> @@ -6,7 +14,7 @@ namespace Slang { static Token GetEndOfFileToken() { - return Token(TokenType::EndOfFile, "", 0, 0, 0, ""); + return Token(TokenType::EndOfFile, "", SourceLoc()); } Token* TokenList::begin() const @@ -52,10 +60,10 @@ namespace Slang return mCursor->type; } - CodePosition TokenReader::PeekLoc() const + SourceLoc TokenReader::PeekLoc() const { if (!mCursor) - return CodePosition(); + return SourceLoc(); SLANG_ASSERT(mCursor); return mCursor->Position; } @@ -75,18 +83,22 @@ namespace Slang // Lexer - Lexer::Lexer( - String const& path, - String const& content, - DiagnosticSink* sink) - : path(path) - , content(content) - , sink(sink) + void Lexer::initialize( + SourceFile* inSourceFile, + DiagnosticSink* inSink) { + sourceFile = inSourceFile; + sink = inSink; + + auto content = inSourceFile->content; + + begin = content.begin(); cursor = content.begin(); end = content.end(); - loc = CodePosition(1, 1, 0, path); + spellingStartLoc = inSourceFile->sourceRange.begin; + presumedStartLoc = spellingStartLoc; + tokenFlags = TokenFlag::AtStartOfLine | TokenFlag::AfterWhitespace; lexerFlags = 0; } @@ -142,9 +154,6 @@ namespace Slang { advanceRaw(lexer); } - - lexer->loc.Line++; - lexer->loc.Col = 1; } // Look ahead one code point, dealing with complications like @@ -224,12 +233,7 @@ namespace Slang // TODO: Need to handle non-ASCII code points. - // Default case is to advance by one location - // and return the raw byte we saw. - - lexer->loc.Col++; - lexer->loc.Pos++; - + // Default case is to return the raw byte we saw. return c; } } @@ -323,6 +327,27 @@ namespace Slang } } + static SourceLoc getSourceLoc(Lexer* lexer) + { + return lexer->presumedStartLoc + (lexer->cursor - lexer->begin); + } + + // Begin overriding the reported locations of tokens, + // based on a `#line` directives + void Lexer::startOverridingSourceLocations( + SourceLoc loc) + { + if(loc.isValid()) + { + presumedStartLoc = loc; + } + } + + void Lexer::stopOverridingSourceLocations() + { + presumedStartLoc = spellingStartLoc; + } + static void lexDigits(Lexer* lexer, int base) { for(;;) @@ -355,7 +380,7 @@ namespace Slang if(digitVal >= base) { char buffer[] = { (char) c, 0 }; - lexer->sink->diagnose(lexer->loc, Diagnostics::invalidDigitForBase, buffer, base); + lexer->sink->diagnose(getSourceLoc(lexer), Diagnostics::invalidDigitForBase, buffer, base); } advance(lexer); @@ -695,11 +720,11 @@ namespace Slang switch(c) { case kEOF: - lexer->sink->diagnose(lexer->loc, Diagnostics::endOfFileInLiteral); + lexer->sink->diagnose(getSourceLoc(lexer), Diagnostics::endOfFileInLiteral); return; case '\n': case '\r': - lexer->sink->diagnose(lexer->loc, Diagnostics::newlineInLiteral); + lexer->sink->diagnose(getSourceLoc(lexer), Diagnostics::newlineInLiteral); return; case '\\': @@ -952,7 +977,7 @@ namespace Slang case '0': { - auto loc = lexer->loc; + auto loc = getSourceLoc(lexer); advance(lexer); switch(peek(lexer)) { @@ -1170,7 +1195,7 @@ namespace Slang // If none of the above cases matched, then we have an // unexpected/invalid character. - auto loc = lexer->loc; + auto loc = getSourceLoc(lexer); auto sink = lexer->sink; int c = advance(lexer); if(c >= 0x20 && c <= 0x7E) @@ -1194,7 +1219,7 @@ namespace Slang for(;;) { Token token; - token.Position = loc; + token.Position = getSourceLoc(this); char const* textBegin = cursor; @@ -1314,7 +1339,7 @@ namespace Slang TokenList Lexer::Parse(const String & fileName, const String & str, DiagnosticSink * sink) { TokenList tokenList; - tokenList.mTokens = TokenizeText(fileName, str, [&](TokenizeErrorType errType, CodePosition pos) + tokenList.mTokens = TokenizeText(fileName, str, [&](TokenizeErrorType errType, SourceLoc pos) { auto curChar = str[pos.Pos]; switch (errType) diff --git a/source/slang/lexer.h b/source/slang/lexer.h index 71b2e85b1..9bb4d34aa 100644 --- a/source/slang/lexer.h +++ b/source/slang/lexer.h @@ -47,7 +47,7 @@ namespace Slang bool IsAtEnd() const { return mCursor == mEnd; } Token PeekToken() const; TokenType PeekTokenType() const; - CodePosition PeekLoc() const; + SourceLoc PeekLoc() const; Token AdvanceToken(); @@ -66,9 +66,8 @@ namespace Slang struct Lexer { - Lexer( - String const& path, - String const& content, + void initialize( + SourceFile* sourceFile, DiagnosticSink* sink); ~Lexer(); @@ -77,13 +76,30 @@ namespace Slang TokenList lexAllTokens(); - String path; - String content; + // Begin overriding the reported locations of tokens, + // based on a `#line` directives + void startOverridingSourceLocations(SourceLoc loc); + + // Stop overriding source locations, and go back + // to reporting source locations in the original file + void stopOverridingSourceLocations(); + + SourceFile* sourceFile; DiagnosticSink* sink; char const* cursor; + + char const* begin; char const* end; - CodePosition loc; + + // The starting source location for the code as written, + // which cannot be overridden. + SourceLoc spellingStartLoc; + + // The nominal starting location for the file, taking + // any active `#line` directive into account. + SourceLoc presumedStartLoc; + TokenFlags tokenFlags; LexerFlags lexerFlags; }; diff --git a/source/slang/lower.cpp b/source/slang/lower.cpp index 611180415..e253abca6 100644 --- a/source/slang/lower.cpp +++ b/source/slang/lower.cpp @@ -318,7 +318,7 @@ class PseudoVarDecl : public RefObject { public: Token Name; - CodePosition Position; + SourceLoc Position; TypeExp type; }; @@ -339,7 +339,7 @@ public: class PseudoExpr : public RefObject { public: - CodePosition Position; + SourceLoc Position; QualType type; }; @@ -384,7 +384,7 @@ public: List<Element> elements; }; -static CodePosition getPosition(LoweredExpr const& expr) +static SourceLoc getPosition(LoweredExpr const& expr) { switch (expr.getFlavor()) { @@ -393,7 +393,7 @@ static CodePosition getPosition(LoweredExpr const& expr) case LoweredExpr::Flavor::VaryingTuple: return expr.getVaryingTupleExpr()->Position; default: SLANG_UNREACHABLE("all cases handled"); - return CodePosition(); + return SourceLoc(); } } @@ -804,7 +804,7 @@ struct LoweringVisitor } RefPtr<Expr> createSimpleVarRef( - CodePosition const& loc, + SourceLoc const& loc, VarDeclBase* decl) { RefPtr<VarExpr> result = new VarExpr(); @@ -816,7 +816,7 @@ struct LoweringVisitor } LoweredExpr createVarRef( - CodePosition const& loc, + SourceLoc const& loc, LoweredDecl const& decl) { switch (decl.getFlavor()) @@ -838,7 +838,7 @@ struct LoweringVisitor LoweredExpr createTupleRef( - CodePosition const& loc, + SourceLoc const& loc, TupleVarDecl* decl) { RefPtr<TupleExpr> result = new TupleExpr(); @@ -868,7 +868,7 @@ struct LoweringVisitor } LoweredExpr createVaryingTupleRef( - CodePosition const& /*loc*/, + SourceLoc const& /*loc*/, VaryingTupleVarDecl* decl) { return decl->expr; diff --git a/source/slang/options.cpp b/source/slang/options.cpp index 49f33f58d..a360695f2 100644 --- a/source/slang/options.cpp +++ b/source/slang/options.cpp @@ -596,7 +596,7 @@ struct OptionsParser else if (rawOutputPaths.Count() > rawEntryPoints.Count()) { requestImpl->mSink.diagnose( - CodePosition(), + SourceLoc(), Diagnostics::tooManyOutputPathsSpecified, rawOutputPaths.Count(), rawEntryPoints.Count()); @@ -611,7 +611,7 @@ struct OptionsParser if (entryPoint.outputPathIndex < 0) { requestImpl->mSink.diagnose( - CodePosition(), + SourceLoc(), Diagnostics::noOutputPathSpecifiedForEntryPoint, entryPoint.name); @@ -639,7 +639,7 @@ struct OptionsParser // This file didn't imply a target, and that // needs to be an error: requestImpl->mSink.diagnose( - CodePosition(), + SourceLoc(), Diagnostics::cannotDeduceOutputFormatFromPath, rawOutputPath.path); @@ -669,7 +669,7 @@ struct OptionsParser // This file didn't imply a target, and that // needs to be an error: requestImpl->mSink.diagnose( - CodePosition(), + SourceLoc(), Diagnostics::outputPathsImplyDifferentFormats, rawOutputPaths[0].path, rawOutputPath.path); diff --git a/source/slang/parser.cpp b/source/slang/parser.cpp index 41a4411d2..cc6a0dd6f 100644 --- a/source/slang/parser.cpp +++ b/source/slang/parser.cpp @@ -41,7 +41,6 @@ namespace Slang TokenReader tokenReader; DiagnosticSink * sink; - String fileName; int genericDepth = 0; // Have we seen any `import` declarations? If so, we need @@ -73,11 +72,9 @@ namespace Slang Parser( TokenSpan const& _tokens, DiagnosticSink * sink, - String _fileName, RefPtr<Scope> const& outerScope) : tokenReader(_tokens) , sink(sink) - , fileName(_fileName) , outerScope(outerScope) {} @@ -614,7 +611,7 @@ namespace Slang RefPtr<Modifier>* modifierLink = &modifiers.first; for (;;) { - CodePosition loc = parser->tokenReader.PeekLoc(); + SourceLoc loc = parser->tokenReader.PeekLoc(); if (0) {} @@ -1003,7 +1000,7 @@ namespace Slang struct PointerDeclarator : Declarator { // location of the `*` token - CodePosition starLoc; + SourceLoc starLoc; RefPtr<Declarator> inner; }; @@ -1014,7 +1011,7 @@ namespace Slang RefPtr<Declarator> inner; // location of the `[` token - CodePosition openBracketLoc; + SourceLoc openBracketLoc; // The expression that yields the element count, or NULL RefPtr<Expr> elementCountExpr; @@ -1377,7 +1374,7 @@ namespace Slang // Either a single declaration, or a group of them struct DeclGroupBuilder { - CodePosition startPosition; + SourceLoc startPosition; RefPtr<Decl> decl; RefPtr<DeclGroup> group; @@ -1546,7 +1543,7 @@ namespace Slang Parser* parser, ContainerDecl* containerDecl) { - CodePosition startPosition = parser->tokenReader.PeekLoc(); + SourceLoc startPosition = parser->tokenReader.PeekLoc(); auto typeSpec = parseTypeSpec(parser); @@ -1780,7 +1777,7 @@ namespace Slang // We first look at the declaration keywrod to determine // the type of buffer to declare: String bufferWrapperTypeName; - CodePosition bufferWrapperTypeNamePos = parser->tokenReader.PeekLoc(); + SourceLoc bufferWrapperTypeNamePos = parser->tokenReader.PeekLoc(); if (AdvanceIf(parser, "cbuffer")) { bufferWrapperTypeName = "ConstantBuffer"; @@ -1914,7 +1911,7 @@ namespace Slang // will see through it to the members inside. - CodePosition pos = parser->tokenReader.PeekLoc(); + SourceLoc pos = parser->tokenReader.PeekLoc(); // The initial name before the `{` is only supposed // to be made visible to reflection @@ -2446,7 +2443,7 @@ namespace Slang } PushScope(program); - program->Position = CodePosition(0, 0, 0, fileName); + program->Position = tokenReader.PeekLoc(); ParseDeclBody(this, program, TokenType::EndOfFile); PopScope(); @@ -3853,10 +3850,9 @@ namespace Slang TranslationUnitRequest* translationUnit, TokenSpan const& tokens, DiagnosticSink* sink, - String const& fileName, RefPtr<Scope> const& outerScope) { - Parser parser(tokens, sink, fileName, outerScope); + Parser parser(tokens, sink, outerScope); parser.translationUnit = translationUnit; diff --git a/source/slang/parser.h b/source/slang/parser.h index 8dfef4c12..41a82602a 100644 --- a/source/slang/parser.h +++ b/source/slang/parser.h @@ -12,7 +12,6 @@ namespace Slang TranslationUnitRequest* translationUnit, TokenSpan const& tokens, DiagnosticSink* sink, - String const& fileName, RefPtr<Scope> const& outerScope); ; } diff --git a/source/slang/preprocessor.cpp b/source/slang/preprocessor.cpp index 53f0c4774..9c9a340ae 100644 --- a/source/slang/preprocessor.cpp +++ b/source/slang/preprocessor.cpp @@ -60,45 +60,61 @@ struct PreprocessorEnvironment // In general, input streams can be nested, so we have to keep a conceptual // stack of input. +struct PrimaryInputStream; + // A stream of input tokens to be consumed struct PreprocessorInputStream { + // The primary input stream that is the parent to this one, + // or NULL if this stream is itself a primary stream. + PrimaryInputStream* primaryStream; + // The next input stream up the stack, if any. PreprocessorInputStream* parent; - // The deepest preprocessor conditional active for this stream. - PreprocessorConditional* conditional; - // Environment to use when looking up macros PreprocessorEnvironment* environment; - // Reader for pre-tokenized input - TokenReader tokenReader; + // Destructor is virtual so that we can clean up + // after concrete subtypes. + virtual ~PreprocessorInputStream() = default; +}; - // If we are clobbering source locations with `#line`, then - // the state is tracked here: +// A "primary" input stream represents the top-level context of a file +// being parsed, and tracks things like preprocessor conditional state +struct PrimaryInputStream : PreprocessorInputStream +{ + // The next *primary* input stream up the stack + PrimaryInputStream* parentPrimaryInputStream; - // Are we overriding source locations? - bool isOverridingSourceLoc; + // The deepest preprocessor conditional active for this stream. + PreprocessorConditional* conditional; - // What is the file name we are overriding to? - String overrideFileName; + // The lexer state that will provide input + Lexer lexer; - // What is the relative offset to apply to any line numbers? - int overrideLineOffset; + // One token of lookahead + Token token; +}; - // Destructor is virtual so that we can clean up - // after concrete subtypes. - virtual ~PreprocessorInputStream() = default; +// A "secondary" input stream represents code that is being expanded +// into the current scope, but which had already been tokenized before. +// +struct PretokenizedInputStream : PreprocessorInputStream +{ + // Reader for pre-tokenized input + TokenReader tokenReader; }; -struct SourceTextInputStream : PreprocessorInputStream +// A pre-tokenized input stream that will only be used once, and which +// therefore owns the memory for its tokens. +struct SimpleTokenInputStream : PretokenizedInputStream { - // The pre-tokenized input - TokenList lexedTokens; + // A list of raw tokens that will provide input + TokenList lexedTokens; }; -struct MacroExpansion : PreprocessorInputStream +struct MacroExpansion : PretokenizedInputStream { // The macro we will expand PreprocessorMacro* macro; @@ -204,39 +220,55 @@ static void DestroyMacro(Preprocessor* preprocessor, PreprocessorMacro* macro); // // Create a fresh input stream -static void InitializeInputStream(Preprocessor* preprocessor, PreprocessorInputStream* inputStream) +static void initializeInputStream(Preprocessor* preprocessor, PreprocessorInputStream* inputStream) { inputStream->parent = NULL; - inputStream->conditional = NULL; inputStream->environment = &preprocessor->globalEnv; } +static void initializePrimaryInputStream(Preprocessor* preprocessor, PrimaryInputStream* inputStream) +{ + initializeInputStream(preprocessor, inputStream); + inputStream->primaryStream = inputStream; + inputStream->conditional = NULL; +} + // Destroy an input stream -static void DestroyInputStream(Preprocessor* /*preprocessor*/, PreprocessorInputStream* inputStream) +static void destroyInputStream(Preprocessor* /*preprocessor*/, PreprocessorInputStream* inputStream) { delete inputStream; } // Create an input stream to represent a pre-tokenized input file. // TODO(tfoley): pre-tokenizing files isn't going to work in the long run. -static PreprocessorInputStream* CreateInputStreamForSource(Preprocessor* preprocessor, String const& source, String const& fileName) +static PreprocessorInputStream* CreateInputStreamForSource( + Preprocessor* preprocessor, + SourceFile* sourceFile) { - SourceTextInputStream* inputStream = new SourceTextInputStream(); - InitializeInputStream(preprocessor, inputStream); + PrimaryInputStream* inputStream = new PrimaryInputStream(); + initializePrimaryInputStream(preprocessor, inputStream); - // Use existing `Lexer` to generate a token stream. - Lexer lexer(fileName, source, GetSink(preprocessor)); - inputStream->lexedTokens = lexer.lexAllTokens(); - inputStream->tokenReader = TokenReader(inputStream->lexedTokens); + // initialize the embedded lexer so that it can generate a token stream + inputStream->lexer.initialize(sourceFile, GetSink(preprocessor)); + inputStream->token = inputStream->lexer.lexToken(); return inputStream; } +static PrimaryInputStream* asPrimaryInputStream(PreprocessorInputStream* inputStream) +{ + auto primaryStream = inputStream->primaryStream; + if(primaryStream == inputStream) + return primaryStream; + return nullptr; +} static void PushInputStream(Preprocessor* preprocessor, PreprocessorInputStream* inputStream) { inputStream->parent = preprocessor->inputStream; + if(!asPrimaryInputStream(inputStream)) + inputStream->primaryStream = preprocessor->inputStream->primaryStream; preprocessor->inputStream = inputStream; } @@ -244,52 +276,69 @@ static void PushInputStream(Preprocessor* preprocessor, PreprocessorInputStream* // Performs some validation and then destroys the input stream if required. static void EndInputStream(Preprocessor* preprocessor, PreprocessorInputStream* inputStream) { - // If there are any conditionals that weren't completed, then it is an error - if (inputStream->conditional) + if(auto primaryStream = asPrimaryInputStream(inputStream)) { - PreprocessorConditional* conditional = inputStream->conditional; + // If there are any conditionals that weren't completed, then it is an error + if (primaryStream->conditional) + { + PreprocessorConditional* conditional = primaryStream->conditional; - GetSink(preprocessor)->diagnose(conditional->ifToken.Position, Diagnostics::endOfFileInPreprocessorConditional); + GetSink(preprocessor)->diagnose(conditional->ifToken.Position, Diagnostics::endOfFileInPreprocessorConditional); - while (conditional) - { - PreprocessorConditional* parent = conditional->parent; - DestroyConditional(conditional); - conditional = parent; + while (conditional) + { + PreprocessorConditional* parent = conditional->parent; + DestroyConditional(conditional); + conditional = parent; + } } } - DestroyInputStream(preprocessor, inputStream); -} - -// Potentially clobber source location information based on `#line` -static Token PossiblyOverrideSourceLoc(PreprocessorInputStream* inputStream, Token const& token) -{ - Token result = token; - if( inputStream->isOverridingSourceLoc ) - { - result.Position.FileName = inputStream->overrideFileName; - result.Position.Line += inputStream->overrideLineOffset; - } - return result; + destroyInputStream(preprocessor, inputStream); } // Consume one token from an input stream static Token AdvanceRawToken(PreprocessorInputStream* inputStream) { - return PossiblyOverrideSourceLoc(inputStream, inputStream->tokenReader.AdvanceToken()); + if( auto primaryStream = asPrimaryInputStream(inputStream) ) + { + auto result = primaryStream->token; + primaryStream->token = primaryStream->lexer.lexToken(); + return result; + } + else + { + PretokenizedInputStream* pretokenized = (PretokenizedInputStream*) inputStream; + return pretokenized->tokenReader.AdvanceToken(); + } } // Peek one token from an input stream static Token PeekRawToken(PreprocessorInputStream* inputStream) { - return PossiblyOverrideSourceLoc(inputStream, inputStream->tokenReader.PeekToken()); + if( auto primaryStream = asPrimaryInputStream(inputStream) ) + { + return primaryStream->token; + } + else + { + PretokenizedInputStream* pretokenized = (PretokenizedInputStream*) inputStream; + return pretokenized->tokenReader.PeekToken(); + } } // Peek one token type from an input stream static TokenType PeekRawTokenType(PreprocessorInputStream* inputStream) { - return inputStream->tokenReader.PeekTokenType(); + if( auto primaryStream = asPrimaryInputStream(inputStream) ) + { + return primaryStream->token.type; + } + else + { + PretokenizedInputStream* pretokenized = (PretokenizedInputStream*) inputStream; + return pretokenized->tokenReader.PeekTokenType(); + } } @@ -355,55 +404,8 @@ static Token PeekRawToken(Preprocessor* preprocessor) } } -// Without advancing preprocessor state, look *two* raw tokens ahead -// (This is only needed in order to determine when we are possibly -// expanding a function-style macro) -TokenType PeekSecondRawTokenType(Preprocessor* preprocessor) -{ - // We need to find the strema that `advanceRawToken` would read from. - PreprocessorInputStream* inputStream = preprocessor->inputStream; - int count = 1; - for (;;) - { - if (!inputStream) - { - // No more input streams left to read - return TokenType::EndOfFile; - } - - // The top-most input stream may be at its end, so - // look one entry up the stack (don't actually pop - // here, since we are just peeking) - - TokenReader reader = inputStream->tokenReader; - if (reader.PeekTokenType() == TokenType::EndOfFile) - { - inputStream = inputStream->parent; - continue; - } - - if (count) - { - count--; - - // Note: we are advancing our temporary - // copy of the token reader - reader.AdvanceToken(); - if (reader.PeekTokenType() == TokenType::EndOfFile) - { - inputStream = inputStream->parent; - continue; - } - } - - // Everything worked, so peek a token from the top-most stream - return reader.PeekTokenType(); - } -} - - // Get the location of the current (raw) token -static CodePosition PeekLoc(Preprocessor* preprocessor) +static SourceLoc PeekLoc(Preprocessor* preprocessor) { return PeekRawToken(preprocessor).Position; } @@ -471,7 +473,7 @@ static PreprocessorEnvironment* GetCurrentEnvironment(Preprocessor* preprocessor // If the current input stream is at its end, then // fall back to its parent stream. - if (inputStream->tokenReader.PeekTokenType() == TokenType::EndOfFile) + if (PeekRawTokenType(inputStream) == TokenType::EndOfFile) { inputStream = inputStream->parent; continue; @@ -528,7 +530,11 @@ static void InitializeMacroExpansion( MacroExpansion* expansion, PreprocessorMacro* macro) { - InitializeInputStream(preprocessor, expansion); + initializeInputStream(preprocessor, expansion); + + expansion->parent = preprocessor->inputStream; + expansion->primaryStream = preprocessor->inputStream->primaryStream; + expansion->environment = macro->environment; expansion->macro = macro; expansion->tokenReader = TokenReader(macro->tokens); @@ -550,6 +556,26 @@ static void AddEndOfStreamToken( macro->tokens.mTokens.Add(token); } +static SimpleTokenInputStream* createSimpleInputStream( + Preprocessor* preprocessor, + Token const& token) +{ + SimpleTokenInputStream* inputStream = new SimpleTokenInputStream(); + initializeInputStream(preprocessor, inputStream); + + inputStream->lexedTokens.mTokens.Add(token); + + Token eofToken; + eofToken.type = TokenType::EndOfFile; + eofToken.Position = token.Position; + eofToken.flags = TokenFlag::AfterWhitespace | TokenFlag::AtStartOfLine; + inputStream->lexedTokens.mTokens.Add(eofToken); + + inputStream->tokenReader = TokenReader(inputStream->lexedTokens); + + return inputStream; +} + // Check whether the current token on the given input stream should be // treated as a macro invocation, and if so set up state for expanding // that macro. @@ -561,7 +587,7 @@ static void MaybeBeginMacroExpansion( for (;;) { // Look at the next token ahead of us - Token const& token = PeekRawToken(preprocessor); + Token token = PeekRawToken(preprocessor); // Not an identifier? Can't be a macro. if (token.type != TokenType::Identifier) @@ -585,12 +611,29 @@ static void MaybeBeginMacroExpansion( // requires more lookahead than we usually have/need if (macro->flavor == PreprocessorMacroFlavor::FunctionLike) { - if(PeekSecondRawTokenType(preprocessor) != TokenType::LParent) - return; - - // Consume the token that triggered macro expansion + // Consume the token that (possibly) triggered macro expansion AdvanceRawToken(preprocessor); + // Look at the next token, and see if it is an opening `(` + // that indicates we should actually expand a macro. + if(PeekRawTokenType(preprocessor) != TokenType::LParent) + { + // In this case, we are in a bit of a mess, because we have + // consumed the token that named the macro, but we need to + // make sure that token (and not whatever came after it) + // gets returned to the user. + // + // To work around this we will construct a short-lived input + // stream just to handle that one token, and also set + // a flag on the token to keep us from doing this logic again. + + token.flags |= TokenFlag::SuppressMacroExpansion; + + SimpleTokenInputStream* simpleStream = createSimpleInputStream(preprocessor, token); + PushInputStream(preprocessor, simpleStream); + return; + } + // Consume the opening `(` Token leftParen = AdvanceRawToken(preprocessor); @@ -762,8 +805,22 @@ top: } // Now re-lex the input - PreprocessorInputStream* inputStream = CreateInputStreamForSource(preprocessor, sb.ProduceString(), "token paste"); - if (inputStream->tokenReader.GetCount() != 1) + + // We create a dummy file to represent the token-paste operation + SourceFile* sourceFile = preprocessor->getCompileRequest()->getSourceManager()->allocateSourceFile("token paste", sb.ProduceString()); + + Lexer lexer; + lexer.initialize(sourceFile, GetSink(preprocessor)); + + SimpleTokenInputStream* inputStream = new SimpleTokenInputStream(); + initializeInputStream(preprocessor, inputStream); + + inputStream->lexedTokens = lexer.lexAllTokens(); + inputStream->tokenReader = TokenReader(inputStream->lexedTokens); + + // We expect the reuslt of lexing to be two tokens: one for the actual value, + // and one for the end-of-input marker. + if (inputStream->tokenReader.GetCount() != 2) { // We expect a token paste to produce a single token // TODO(tfoley): emit a diagnostic here @@ -839,7 +896,7 @@ inline String const& GetDirectiveName(PreprocessorDirectiveContext* context) } // Get the location of the directive being parsed. -inline CodePosition const& GetDirectiveLoc(PreprocessorDirectiveContext* context) +inline SourceLoc const& GetDirectiveLoc(PreprocessorDirectiveContext* context) { return context->directiveToken.Position; } @@ -851,7 +908,7 @@ static inline DiagnosticSink* GetSink(PreprocessorDirectiveContext* context) } // Wrapper to get a "current" location when parsing a directive -static CodePosition PeekLoc(PreprocessorDirectiveContext* context) +static SourceLoc PeekLoc(PreprocessorDirectiveContext* context) { return PeekLoc(context->preprocessor); } @@ -970,8 +1027,11 @@ static bool IsSkipping(Preprocessor* preprocessor) PreprocessorInputStream* inputStream = preprocessor->inputStream; if (!inputStream) return false; + PrimaryInputStream* primaryStream = inputStream->primaryStream; + if(!primaryStream) return false; + // If we are not inside a preprocessor conditional, then don't skip - PreprocessorConditional* conditional = inputStream->conditional; + PreprocessorConditional* conditional = primaryStream->conditional; if (!conditional) return false; // skip tokens unless the conditional is inside its `true` case @@ -1032,8 +1092,9 @@ static void beginConditional( conditional->state = state; // Push conditional onto the stack - conditional->parent = inputStream->conditional; - inputStream->conditional = conditional; + auto primaryStream = inputStream->primaryStream; + conditional->parent = primaryStream->conditional; + primaryStream->conditional = conditional; } // Start a preprocessor conditional, with an initial enable/disable state. @@ -1348,7 +1409,7 @@ static void HandleElseDirective(PreprocessorDirectiveContext* context) SLANG_ASSERT(inputStream); // if we aren't inside a conditional, then error - PreprocessorConditional* conditional = inputStream->conditional; + PreprocessorConditional* conditional = inputStream->primaryStream->conditional; if (!conditional) { GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::directiveWithoutIf, GetDirectiveName(context)); @@ -1402,7 +1463,7 @@ static void HandleElifDirective(PreprocessorDirectiveContext* context) PreprocessorExpressionValue value = ParseAndEvaluateExpression(context); // if we aren't inside a conditional, then error - PreprocessorConditional* conditional = inputStream->conditional; + PreprocessorConditional* conditional = inputStream->primaryStream->conditional; if (!conditional) { GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::directiveWithoutIf, GetDirectiveName(context)); @@ -1440,14 +1501,14 @@ static void HandleEndIfDirective(PreprocessorDirectiveContext* context) SLANG_ASSERT(inputStream); // if we aren't inside a conditional, then error - PreprocessorConditional* conditional = inputStream->conditional; + PreprocessorConditional* conditional = inputStream->primaryStream->conditional; if (!conditional) { GetSink(context)->diagnose(GetDirectiveLoc(context), Diagnostics::directiveWithoutIf, GetDirectiveName(context)); return; } - inputStream->conditional = conditional->parent; + inputStream->primaryStream->conditional = conditional->parent; DestroyConditional(conditional); } @@ -1492,8 +1553,9 @@ static void HandleIncludeDirective(PreprocessorDirectiveContext* context) String path = getFileNameTokenValue(pathToken); - // TODO(tfoley): make this robust in presence of `#line` - String pathIncludedFrom = GetDirectiveLoc(context).FileName; + auto directiveLoc = GetDirectiveLoc(context); + auto expandedDirectiveLoc = context->preprocessor->translationUnit->compileRequest->getSourceManager()->expandSourceLoc(directiveLoc); + String pathIncludedFrom = expandedDirectiveLoc.getSpellingPath(); String foundPath; String foundSource; @@ -1524,7 +1586,10 @@ static void HandleIncludeDirective(PreprocessorDirectiveContext* context) // Push the new file onto our stack of input streams // TODO(tfoley): check if we have made our include stack too deep - PreprocessorInputStream* inputStream = CreateInputStreamForSource(context->preprocessor, foundSource, foundPath); + + SourceFile* sourceFile = context->preprocessor->getCompileRequest()->getSourceManager()->allocateSourceFile(foundPath, foundSource); + + PreprocessorInputStream* inputStream = CreateInputStreamForSource(context->preprocessor, sourceFile); inputStream->parent = context->preprocessor->inputStream; context->preprocessor->inputStream = inputStream; } @@ -1654,18 +1719,26 @@ static void HandleErrorDirective(PreprocessorDirectiveContext* context) // Handle a `#line` directive static void HandleLineDirective(PreprocessorDirectiveContext* context) { + auto inputStream = context->preprocessor->inputStream; + int line = 0; + + // `#line <integer-literal> ...` if (PeekTokenType(context) == TokenType::IntegerLiteral) { line = StringToInt(AdvanceToken(context).Content); } - else if (PeekTokenType(context) == TokenType::Identifier - && PeekToken(context).Content == "default") + // `#line` + // `#line default` + else if ( + PeekTokenType(context) == TokenType::EndOfDirective + || (PeekTokenType(context) == TokenType::Identifier + && PeekToken(context).Content == "default")) { AdvanceToken(context); // Stop overiding soure locations. - context->preprocessor->inputStream->isOverridingSourceLoc = false; + inputStream->primaryStream->lexer.stopOverridingSourceLocations(); return; } else @@ -1678,16 +1751,19 @@ static void HandleLineDirective(PreprocessorDirectiveContext* context) return; } - CodePosition directiveLoc = GetDirectiveLoc(context); + SourceLoc directiveLoc = GetDirectiveLoc(context); + + auto sourceManager = context->preprocessor->translationUnit->compileRequest->getSourceManager(); + auto expandedDirectiveLoc = sourceManager->expandSourceLoc(directiveLoc); String file; if (PeekTokenType(context) == TokenType::EndOfDirective) { - file = directiveLoc.FileName; + file = expandedDirectiveLoc.getPath(); } else if (PeekTokenType(context) == TokenType::StringLiteral) { - file = AdvanceToken(context).Content; + file = getStringLiteralTokenValue(AdvanceToken(context)); } else if (PeekTokenType(context) == TokenType::IntegerLiteral) { @@ -1701,11 +1777,9 @@ static void HandleLineDirective(PreprocessorDirectiveContext* context) return; } - PreprocessorInputStream* inputStream = context->preprocessor->inputStream; + SourceLoc newLoc = sourceManager->allocateSourceFileForLineDirective(expandedDirectiveLoc, file, line); - inputStream->isOverridingSourceLoc = true; - inputStream->overrideFileName = file; - inputStream->overrideLineOffset = line - (directiveLoc.Line + 1); + inputStream->primaryStream->lexer.startOverridingSourceLocations(newLoc); } // Handle a `#pragma` directive @@ -2005,10 +2079,14 @@ static void DefineMacro( String fileName = "command line"; PreprocessorMacro* macro = CreateMacro(preprocessor); + SourceFile* keyFile = preprocessor->translationUnit->compileRequest->getSourceManager()->allocateSourceFile(fileName, key); + SourceFile* valueFile = preprocessor->translationUnit->compileRequest->getSourceManager()->allocateSourceFile(fileName, value); + // Use existing `Lexer` to generate a token stream. - Lexer lexer(fileName, value, GetSink(preprocessor)); + Lexer lexer; + lexer.initialize(valueFile, GetSink(preprocessor)); macro->tokens = lexer.lexAllTokens(); - macro->nameToken = Token(TokenType::Identifier, key, 0, 0, 0, fileName); + macro->nameToken = Token(TokenType::Identifier, key, keyFile->sourceRange.begin); PreprocessorMacro* oldMacro = NULL; if (preprocessor->globalEnv.macros.TryGetValue(key, oldMacro)) @@ -2039,8 +2117,7 @@ static TokenList ReadAllTokens( } TokenList preprocessSource( - String const& source, - String const& fileName, + SourceFile* file, DiagnosticSink* sink, IncludeHandler* includeHandler, Dictionary<String, String> defines, @@ -2057,7 +2134,7 @@ TokenList preprocessSource( } // create an initial input stream based on the provided buffer - preprocessor.inputStream = CreateInputStreamForSource(&preprocessor, source, fileName); + preprocessor.inputStream = CreateInputStreamForSource(&preprocessor, file); TokenList tokens = ReadAllTokens(&preprocessor); @@ -2097,8 +2174,9 @@ static void HandleImportDirective(PreprocessorDirectiveContext* context) String path = getFileNameTokenValue(pathToken); - // TODO(tfoley): make this robust in presence of `#line` - String pathIncludedFrom = GetDirectiveLoc(context).FileName; + auto directiveLoc = GetDirectiveLoc(context); + auto expandedDirectiveLoc = context->preprocessor->translationUnit->compileRequest->getSourceManager()->expandSourceLoc(directiveLoc); + String pathIncludedFrom = expandedDirectiveLoc.getSpellingPath(); String foundPath; String foundSource; @@ -2162,7 +2240,8 @@ static void HandleImportDirective(PreprocessorDirectiveContext* context) PreprocessorInputStream* savedStream = preprocessor->inputStream; // Create an input stream for reading from the imported file - PreprocessorInputStream* subInputStream = CreateInputStreamForSource(preprocessor, foundSource, foundPath); + SourceFile* sourceFile = context->preprocessor->getCompileRequest()->getSourceManager()->allocateSourceFile(foundPath, foundSource); + PreprocessorInputStream* subInputStream = CreateInputStreamForSource(preprocessor, sourceFile); // Now preprocess that stream preprocessor->inputStream = subInputStream; @@ -2180,24 +2259,18 @@ static void HandleImportDirective(PreprocessorDirectiveContext* context) // Now create a dummy token stream to represent the import request, // so that it can be manifest in the user's program - SourceTextInputStream* inputStream = new SourceTextInputStream(); - + Token token; token.type = TokenType::PoundImport; token.Position = GetDirectiveLoc(context); token.flags = 0; token.Content = foundPath; - - inputStream->lexedTokens.mTokens.Add(token); - - token.type = TokenType::EndOfFile; - token.flags = TokenFlag::AfterWhitespace | TokenFlag::AtStartOfLine; - inputStream->lexedTokens.mTokens.Add(token); - - inputStream->tokenReader = TokenReader(inputStream->lexedTokens); - - inputStream->parent = context->preprocessor->inputStream; - context->preprocessor->inputStream = inputStream; + + SimpleTokenInputStream* inputStream = createSimpleInputStream( + context->preprocessor, + token); + + PushInputStream(context->preprocessor, inputStream); } diff --git a/source/slang/preprocessor.h b/source/slang/preprocessor.h index 008707ffb..64591ef03 100644 --- a/source/slang/preprocessor.h +++ b/source/slang/preprocessor.h @@ -31,8 +31,7 @@ struct IncludeHandler // Take a string of source code and preprocess it into a list of tokens. TokenList preprocessSource( - String const& source, - String const& fileName, + SourceFile* file, DiagnosticSink* sink, IncludeHandler* includeHandler, Dictionary<String, String> defines, diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp index 0dcbf44ea..e66884923 100644 --- a/source/slang/slang.cpp +++ b/source/slang/slang.cpp @@ -20,6 +20,9 @@ namespace Slang { Session::Session() { + // Make sure our source manager is initialized + builtinSourceManager.initialize(nullptr); + // Initialize representations of some very basic types: initializeTypes(); @@ -82,6 +85,16 @@ struct IncludeHandlerImpl : IncludeHandler } }; +CompileRequest::CompileRequest(Session* session) + : mSession(session) +{ + setSourceManager(&sourceManagerStorage); + + sourceManager->initialize(session->getBuiltinSourceManager()); +} + +CompileRequest::~CompileRequest() +{} void CompileRequest::parseTranslationUnit( TranslationUnitRequest* translationUnit) @@ -117,12 +130,8 @@ void CompileRequest::parseTranslationUnit( for (auto sourceFile : translationUnit->sourceFiles) { - auto sourceFilePath = sourceFile->path; - String source = sourceFile->content; - auto tokens = preprocessSource( - source, - sourceFilePath, + sourceFile, &mSink, &includeHandler, combinedPreprocessorDefinitions, @@ -132,7 +141,6 @@ void CompileRequest::parseTranslationUnit( translationUnit, tokens, &mSink, - sourceFilePath, languageScope); } } @@ -297,16 +305,21 @@ int CompileRequest::addTranslationUnit(SourceLanguage language, String const&) return (int) result; } +void CompileRequest::addTranslationUnitSourceFile( + int translationUnitIndex, + SourceFile* sourceFile) +{ + translationUnits[translationUnitIndex]->sourceFiles.Add(sourceFile); +} + void CompileRequest::addTranslationUnitSourceString( int translationUnitIndex, String const& path, String const& source) { - RefPtr<SourceFile> sourceFile = new SourceFile(); - sourceFile->path = path; - sourceFile->content = source; + RefPtr<SourceFile> sourceFile = getSourceManager()->allocateSourceFile(path, source); - translationUnits[translationUnitIndex]->sourceFiles.Add(sourceFile); + addTranslationUnitSourceFile(translationUnitIndex, sourceFile); } void CompileRequest::addTranslationUnitSourceFile( @@ -322,7 +335,7 @@ void CompileRequest::addTranslationUnitSourceFile( { // Emit a diagnostic! mSink.diagnose( - CodePosition(0, 0, 0, path), + SourceLoc(), Diagnostics::cannotOpenFile, path); return; @@ -359,7 +372,7 @@ RefPtr<ModuleDecl> CompileRequest::loadModule( String const& name, String const& path, String const& source, - CodePosition const&) + SourceLoc const&) { RefPtr<TranslationUnitRequest> translationUnit = new TranslationUnitRequest(); translationUnit->compileRequest = this; @@ -370,9 +383,7 @@ RefPtr<ModuleDecl> CompileRequest::loadModule( // // TODO: decide which options, if any, should be inherited. - RefPtr<SourceFile> sourceFile = new SourceFile(); - sourceFile->path = path; - sourceFile->content = source; + RefPtr<SourceFile> sourceFile = getSourceManager()->allocateSourceFile(path, source); translationUnit->sourceFiles.Add(sourceFile); @@ -413,7 +424,6 @@ void CompileRequest::handlePoundImport( translationUnit.Ptr(), tokens, &mSink, - path, languageScope); // TODO: handle errors @@ -438,7 +448,7 @@ void CompileRequest::handlePoundImport( RefPtr<ModuleDecl> CompileRequest::findOrImportModule( String const& name, - CodePosition const& loc) + SourceLoc const& loc) { // Have we already loaded a module matching this name? // If so, return it. @@ -470,7 +480,9 @@ RefPtr<ModuleDecl> CompileRequest::findOrImportModule( IncludeHandlerImpl includeHandler; includeHandler.request = this; - String pathIncludedFrom = loc.FileName; + auto expandedLoc = getSourceManager()->expandSourceLoc(loc); + + String pathIncludedFrom = expandedLoc.getSpellingPath(); String foundPath; String foundSource; @@ -508,7 +520,7 @@ RefPtr<ModuleDecl> CompileRequest::findOrImportModule( RefPtr<ModuleDecl> findOrImportModule( CompileRequest* request, String const& name, - CodePosition const& loc) + SourceLoc const& loc) { return request->findOrImportModule(name, loc); } @@ -519,9 +531,12 @@ void Session::addBuiltinSource( String const& source) { RefPtr<CompileRequest> compileRequest = new CompileRequest(this); + compileRequest->setSourceManager(getBuiltinSourceManager()); auto translationUnitIndex = compileRequest->addTranslationUnit(SourceLanguage::Slang, path); + RefPtr<SourceFile> sourceFile = builtinSourceManager.allocateSourceFile(path, source); + compileRequest->addTranslationUnitSourceString( translationUnitIndex, path, @@ -818,7 +833,7 @@ SLANG_API int spCompile( } catch (...) { - req->mSink.diagnose(Slang::CodePosition(), Slang::Diagnostics::compilationAborted); + req->mSink.diagnose(Slang::SourceLoc(), Slang::Diagnostics::compilationAborted); return 1; } } diff --git a/source/slang/slang.vcxproj b/source/slang/slang.vcxproj index 4a1e75bc3..7da5c39d6 100644 --- a/source/slang/slang.vcxproj +++ b/source/slang/slang.vcxproj @@ -213,6 +213,7 @@ <ClCompile Include="reflection.cpp" /> <ClCompile Include="slang-stdlib.cpp" /> <ClCompile Include="slang.cpp" /> + <ClCompile Include="source-loc.cpp" /> <ClCompile Include="syntax.cpp" /> <ClCompile Include="token.cpp" /> <ClCompile Include="type-layout.cpp" /> diff --git a/source/slang/slang.vcxproj.filters b/source/slang/slang.vcxproj.filters index c43cd9dbb..2241dc67c 100644 --- a/source/slang/slang.vcxproj.filters +++ b/source/slang/slang.vcxproj.filters @@ -56,5 +56,6 @@ <ClCompile Include="type-layout.cpp" /> <ClCompile Include="options.cpp" /> <ClCompile Include="lower.cpp" /> + <ClCompile Include="source-loc.cpp" /> </ItemGroup> </Project>
\ No newline at end of file diff --git a/source/slang/source-loc.cpp b/source/slang/source-loc.cpp new file mode 100644 index 000000000..cea8441e6 --- /dev/null +++ b/source/slang/source-loc.cpp @@ -0,0 +1,332 @@ +// source-loc.cpp +#include "source-loc.h" + +namespace Slang { + +String ExpandedSourceLoc::getPath() const +{ + if(!sourceManager) + return String(); + + return sourceManager->sourceFiles[entryIndex].path; +} + +String ExpandedSourceLoc::getSpellingPath() const +{ + if(!sourceManager) + return String(); + + return sourceManager->sourceFiles[entryIndex].sourceFile->path; +} + +SourceFile* ExpandedSourceLoc::getSourceFile() const +{ + if(!sourceManager) + return nullptr; + + return sourceManager->sourceFiles[entryIndex].sourceFile; +} + +void SourceManager::initialize( + SourceManager* p) +{ + parent = p; + + if( p ) + { + // If we have a parent source manager, then we assume that all code at that level + // has already been loaded, and it is safe to start our own source locations + // right after those from the parent. + // + // TODO: more clever allocation in cases where that might not be reasonable + startLoc = p->nextLoc; + } + else + { + // Location zero is reserved for an invalid location, + // so we need to start reserving locations starting at 1. + startLoc = SourceLoc::fromRaw(1); + } + + nextLoc = startLoc; +} + +SourceRange SourceManager::allocateSourceRange(UInt size) +{ + // TODO: consider using atomics here + + + SourceLoc beginLoc = nextLoc; + SourceLoc endLoc = beginLoc + size; + + // We need to be able to represent the location that is *at* the end of + // the input source, so the next available location for a new file + // must be placed one after the end of this one. + + nextLoc = endLoc + 1; + + return SourceRange(beginLoc, endLoc); +} + +SourceFile* SourceManager::allocateSourceFile( + String const& path, + String const& content) +{ + UInt size = content.Length(); + + SourceRange sourceRange = allocateSourceRange(size); + + SourceFile* sourceFile = new SourceFile(); + sourceFile->path = path; + sourceFile->content = content; + sourceFile->sourceRange = sourceRange; + + Entry entry; + entry.sourceFile = sourceFile; + entry.startLoc = sourceRange.begin; + entry.path = path; + + sourceFiles.Add(entry); + + return sourceFile; +} + +SourceLoc SourceManager::allocateSourceFileForLineDirective( + SourceLoc const& directiveLoc, + String const& path, + UInt line) +{ + // First, we need to find out what file we are being asked to remap + ExpandedSourceLoc expandedDirectiveLoc = expandSourceLoc(getSpellingLoc(directiveLoc)); + HumaneSourceLoc humaneDirectiveLoc = getHumaneLoc(expandedDirectiveLoc); + + SourceFile* sourceFile = expandedDirectiveLoc.getSourceFile(); + if(!sourceFile) + return SourceLoc(); + + // We are going to be wasteful here and allocate a range of source locations + // that can cover the entire input file. This will lead to a problem with + // memory usage if we ever had a large input file that used many `#line` directives, + // since our usage of ranges would be quadratic! + + // Count how many locations we'd need to reserve for a complete clone of the input + UInt size = sourceFile->sourceRange.end.getRaw() - sourceFile->sourceRange.begin.getRaw(); + + // Allocate a fresh range for our logically remapped file + SourceRange sourceRange = allocateSourceRange(size); + + // Now fill in an entry that will point at the original source file, + // but use our new range. + Entry entry; + entry.sourceFile = sourceFile; + entry.startLoc = sourceRange.begin; + entry.path = path; + + // We also need to make sure that any lookups for line numbers will + // get corrected based on this files location. + entry.lineAdjust = Int(line) - Int(humaneDirectiveLoc.line + 1); + + sourceFiles.Add(entry); + + return entry.startLoc; +} + +static ExpandedSourceLoc expandSourceLoc( + SourceManager* inSourceManager, + SourceLoc const& loc) +{ + SourceManager* sourceManager = inSourceManager; + + ExpandedSourceLoc expanded; + + SourceLoc::RawValue rawValue = loc.getRaw(); + + // Invalid location? -> invalid expanded location + if(rawValue == 0) + return expanded; + + // Past the end of what we can handle? -> invalid + if(rawValue >= sourceManager->nextLoc.getRaw()) + return expanded; + + // Maybe the location came from a parent source manager + while( rawValue < sourceManager->startLoc.getRaw() + && sourceManager->parent) + { + sourceManager = sourceManager->parent; + } + + SLANG_ASSERT(sourceManager->sourceFiles.Count() > 0); + + UInt lo = 0; + UInt hi = sourceManager->sourceFiles.Count(); + + while( lo+1 < hi ) + { + UInt mid = lo + (hi - lo) / 2; + + SourceManager::Entry const& midEntry = sourceManager->sourceFiles[mid]; + SourceLoc::RawValue midValue = midEntry.startLoc.getRaw(); + + if( midValue <= rawValue ) + { + // The location we seek is at or after this entry + lo = mid; + } + else + { + // The location we seek is before this entry + hi = mid; + } + } + + // `lo` should now point at the entry we want + UInt entryIndex = lo; + + expanded.setRaw(loc.getRaw()); + expanded.sourceManager = sourceManager; + expanded.entryIndex = entryIndex; + + return expanded; + + +} + +ExpandedSourceLoc SourceManager::expandSourceLoc(SourceLoc const& loc) +{ + return Slang::expandSourceLoc(this, loc); +} + +HumaneSourceLoc SourceManager::getHumaneLoc(ExpandedSourceLoc const& loc) +{ + // First check if this location maps to an actual file. + SourceFile* sourceFile = loc.getSourceFile(); + if(!sourceFile) + return HumaneSourceLoc(); + + auto& entry = sourceFiles[loc.entryIndex]; + UInt offset = loc.getRaw() - entry.startLoc.getRaw(); + + // We now have a raw input file that we can search for line breaks. + // We obviously don't want to do a linear scan over and over, so we will + // cache an array of line break locations in the file. + auto& lineBreakOffsets = sourceFile->lineBreakOffsets; + if( lineBreakOffsets.Count() == 0 ) + { + char const* begin = sourceFile->content.begin(); + char const* end = sourceFile->content.end(); + + char const* cursor = begin; + + // Treat the beginning of the file as a line break + lineBreakOffsets.Add(0); + + while( cursor != end ) + { + int c = *cursor++; + switch( c ) + { + case '\r': case '\n': + { + // When we see a line-break character we need + // to record the line break, but we also need + // to deal with the annoying issue of encodings, + // where a multi-byte sequence might encode + // the line break. + + int d = *cursor; + if( (c^d) == ('\r' ^ '\n')) + cursor++; + + lineBreakOffsets.Add(cursor - begin); + } + break; + + default: + break; + } + } + + // Note taht we do *not* treat the end of the file as a line + // break, because otherwise we would report errors like + // "end of file inside string literal" with a line number + // that points at a line that doesn't exist. + } + + // At this point we can assume the `lineBreakOffsets` array has been filled in. + // We will use a binary search to find the line index that contains our + // chosen offset. + UInt lo = 0; + UInt hi = lineBreakOffsets.Count(); + + while( lo+1 < hi ) + { + UInt mid = lo + (hi - lo)/2; + + UInt midOffset = lineBreakOffsets[mid]; + if( midOffset <= offset ) + { + lo = mid; + } + else + { + hi = mid; + } + } + + UInt lineIndex = lo; + UInt byteIndexInLine = offset - lineBreakOffsets[lineIndex]; + + // Apply adjustment to the line number + lineIndex = lineIndex + entry.lineAdjust; + + // TODO: we should really translate the byte index in the line + // to deal with: + // + // - Non-ASCII characters, while might consume multiple bytes + // + // - Tab characters, which should really adjust how we report + // columns (although how are we supposed to know the setting + // that an IDE expects us to use when reporting locations?) + + HumaneSourceLoc humaneLoc; + humaneLoc.path = entry.path; + humaneLoc.line = lineIndex + 1; + humaneLoc.column = byteIndexInLine + 1; + + return humaneLoc; +} + +HumaneSourceLoc SourceManager::getHumaneLoc(SourceLoc const& loc) +{ + return getHumaneLoc(expandSourceLoc(loc)); + +} + +SourceLoc SourceManager::getSpellingLoc(ExpandedSourceLoc const& loc) +{ + // First check if this location maps to some raw source file, + // so that a "spelling" is even possible + SourceFile* sourceFile = loc.getSourceFile(); + if(!sourceFile) + return loc; + + // If we mapped to a source file, then the location must represent + // some offset from an entry in our array. + auto& entry = sourceFiles[loc.entryIndex]; + + // We extract the offset of the location from the start of the entry + SourceLoc::RawValue offsetFromStart = loc.getRaw() - entry.startLoc.getRaw(); + + // And instead apply that offset to the spelling location of the file start + SourceLoc result = sourceFile->sourceRange.begin + offsetFromStart; + + return result; +} + +SourceLoc SourceManager::getSpellingLoc(SourceLoc const& loc) +{ + return getSpellingLoc(expandSourceLoc(loc)); +} + +} // namespace Slang diff --git a/source/slang/source-loc.h b/source/slang/source-loc.h index 59f2f10ea..8bb89c645 100644 --- a/source/slang/source-loc.h +++ b/source/slang/source-loc.h @@ -6,36 +6,181 @@ namespace Slang { -class CodePosition +class SourceLoc { public: - int Line = -1, Col = -1, Pos = -1; - String FileName; - String ToString() - { - StringBuilder sb(100); - sb << FileName; - if (Line != -1) - sb << "(" << Line << ")"; - return sb.ProduceString(); - } - CodePosition() = default; - CodePosition(int line, int col, int pos, String fileName) - { - Line = line; - Col = col; - Pos = pos; - this->FileName = fileName; - } - bool operator < (const CodePosition & pos) const - { - return FileName < pos.FileName || (FileName == pos.FileName && Line < pos.Line) || - (FileName == pos.FileName && Line == pos.Line && Col < pos.Col); - } - bool operator == (const CodePosition & pos) const - { - return FileName == pos.FileName && Line == pos.Line && Col == pos.Col; - } + typedef UInt RawValue; + +private: + RawValue raw; + +public: + SourceLoc() + : raw(0) + {} + + SourceLoc( + SourceLoc const& loc) + : raw(loc.raw) + {} + + RawValue getRaw() const { return raw; } + void setRaw(RawValue value) { raw = value; } + + static SourceLoc fromRaw(RawValue value) + { + SourceLoc result; + result.setRaw(value); + return result; + } + + bool isValid() const + { + return raw != 0; + } +}; + +inline SourceLoc operator+(SourceLoc loc, Int offset) +{ + return SourceLoc::fromRaw(loc.getRaw() + UInt(offset)); +} + +// A range of locations in the input source +struct SourceRange +{ + SourceRange() + {} + + SourceRange(SourceLoc loc) + : begin(loc) + , end(loc) + {} + + SourceRange(SourceLoc begin, SourceLoc end) + : begin(begin) + , end(end) + {} + + SourceLoc begin; + SourceLoc end; +}; + +// A logical or phyiscal storage object for a range of input code +// that has logically contiguous source locations. +class SourceFile : public RefObject +{ +public: + // The logical file path to report for locations inside this span. + String path; + + // The actual contents of the file. + String content; + + // The range of source locations that the span covers + SourceRange sourceRange; + + // In order to speed up lookup of line number information, + // we will cache the starting offset of each line break in + // the input file: + List<UInt> lineBreakOffsets; +}; + +struct SourceManager; + +// A source location in a format a human might like to see +struct HumaneSourceLoc +{ + String path; + Int line = 0; + Int column = 0; + + String const& getPath() const { return path; } + Int getLine() const { return line; } + Int getColumn() const { return column; } +}; + +// A source location that has been expanded with the info +// needed to reconstruct a "humane" location if needed. +struct ExpandedSourceLoc : public SourceLoc +{ + // The source manager that owns this location + SourceManager* sourceManager = nullptr; + + // The entry index that is used to understand the location + UInt entryIndex = 0; + + // Get the nominal path for this location + String getPath() const; + + // Get the actual file path where this location appears + String getSpellingPath() const; + + // Get the original source file that holds this location + SourceFile* getSourceFile() const; +}; + + + +struct SourceManager +{ + // Initialize a source manager, with an optional parent + void initialize( + SourceManager* parent); + + SourceRange allocateSourceRange(UInt size); + + SourceFile* allocateSourceFile( + String const& path, + String const& content); + + SourceLoc allocateSourceFileForLineDirective( + SourceLoc const& directiveLoc, + String const& path, + UInt line); + + // Expand a source location to include more explicit info + ExpandedSourceLoc expandSourceLoc(SourceLoc const& loc); + + // Get a "humane" version of a source location + HumaneSourceLoc getHumaneLoc(ExpandedSourceLoc const& loc); + HumaneSourceLoc getHumaneLoc(SourceLoc const& loc); + + + // Get the source location that represents the spelling location corresponding to a location. + SourceLoc getSpellingLoc(ExpandedSourceLoc const& loc); + SourceLoc getSpellingLoc(SourceLoc const& loc); + + // The first location available to this source manager + // (may not be the first location of all, because we might + // have a parent source manager) + SourceLoc startLoc; + + // The "parent" source manager that owns locations ahead of `startLoc` + SourceManager* parent = nullptr; + + // The location to be used by the next source file to be loaded + SourceLoc nextLoc; + + // Each entry represents some contiguous span of locations that + // all map to the same logical file. + struct Entry + { + // Where does this entry begin? + SourceLoc startLoc; + + // The soure file that represents the actual data + RefPtr<SourceFile> sourceFile; + + // What is the presumed path for this entry + String path; + + // Adjustment to apply to source line numbers when printing presumed locations + Int lineAdjust = 0; + }; + + // An array of soure files we have loaded, ordered by + // increasing starting location + List<Entry> sourceFiles; }; diff --git a/source/slang/syntax-base-defs.h b/source/slang/syntax-base-defs.h index d76607106..711c4425b 100644 --- a/source/slang/syntax-base-defs.h +++ b/source/slang/syntax-base-defs.h @@ -9,7 +9,7 @@ ABSTRACT_SYNTAX_CLASS(SyntaxNodeBase, RefObject) // The primary source location associated with this AST node - FIELD(CodePosition, Position) + FIELD(SourceLoc, Position) RAW( // Allow dynamic casting with a convenient syntax diff --git a/source/slang/syntax-visitors.h b/source/slang/syntax-visitors.h index f2c080a57..0780bab2a 100644 --- a/source/slang/syntax-visitors.h +++ b/source/slang/syntax-visitors.h @@ -26,7 +26,7 @@ namespace Slang RefPtr<ModuleDecl> findOrImportModule( CompileRequest* request, String const& name, - CodePosition const& loc); + SourceLoc const& loc); } #endif
\ No newline at end of file diff --git a/source/slang/token.h b/source/slang/token.h index f29f2b4c6..306d4a2f2 100644 --- a/source/slang/token.h +++ b/source/slang/token.h @@ -18,8 +18,9 @@ char const* TokenTypeToString(TokenType type); enum TokenFlag : unsigned int { - AtStartOfLine = 1 << 0, - AfterWhitespace = 1 << 1, + AtStartOfLine = 1 << 0, + AfterWhitespace = 1 << 1, + SuppressMacroExpansion = 1 << 2, }; typedef unsigned int TokenFlags; @@ -28,15 +29,21 @@ class Token public: TokenType type = TokenType::Unknown; String Content; - CodePosition Position; + SourceLoc Position; TokenFlags flags = 0; + Token() = default; - Token(TokenType type, const String & content, int line, int col, int pos, String fileName, TokenFlags flags = 0) + + Token( + TokenType type, + const String & content, + SourceLoc loc, + TokenFlags flags = 0) : flags(flags) { type = type; Content = content; - Position = CodePosition(line, col, pos, fileName); + Position = loc; } }; |