From fa31d21ba92669a521a7768467246918e3947e02 Mon Sep 17 00:00:00 2001 From: jsmall-nvidia Date: Thu, 1 Apr 2021 13:39:11 -0400 Subject: Added compiler-core project (#1775) * #include an absolute path didn't work - because paths were taken to always be relative. * Split out compiler-core initially with just slang-source-loc.cpp * More lexer, name, token to compiler-core. * Split Lexer and Core diagnostics. * Move slang-file-system to core. * Add slang-file-system to core. * More DownstreamCompiler into compiler-core * Fix typo. * Add compiler-core to bootstrap proj. * Small fixes to premake * For linux try with compiler-core * Remove compiler-core from examples. * Added NameConventionUtil to compiler-core * Add global function to CharUtil to *hopefully* avoid linking issue. * Hack to make linkage of CharUtil work on linux. --- source/compiler-core/slang-core-diagnostics.cpp | 50 + source/compiler-core/slang-core-diagnostics.h | 32 + source/compiler-core/slang-diagnostic-sink.cpp | 537 ++++++++ source/compiler-core/slang-diagnostic-sink.h | 316 +++++ source/compiler-core/slang-downstream-compiler.cpp | 716 ++++++++++ source/compiler-core/slang-downstream-compiler.h | 501 +++++++ source/compiler-core/slang-gcc-compiler-util.cpp | 645 +++++++++ source/compiler-core/slang-gcc-compiler-util.h | 56 + source/compiler-core/slang-lexer-diagnostic-defs.h | 33 + source/compiler-core/slang-lexer.cpp | 1400 ++++++++++++++++++++ source/compiler-core/slang-lexer.h | 164 +++ source/compiler-core/slang-misc-diagnostic-defs.h | 27 + .../compiler-core/slang-name-convention-util.cpp | 213 +++ source/compiler-core/slang-name-convention-util.h | 54 + source/compiler-core/slang-name.cpp | 42 + source/compiler-core/slang-name.h | 89 ++ source/compiler-core/slang-nvrtc-compiler.cpp | 773 +++++++++++ source/compiler-core/slang-nvrtc-compiler.h | 19 + source/compiler-core/slang-source-loc.cpp | 689 ++++++++++ source/compiler-core/slang-source-loc.h | 487 +++++++ source/compiler-core/slang-token-defs.h | 96 ++ source/compiler-core/slang-token.cpp | 21 + source/compiler-core/slang-token.h | 140 ++ .../slang-visual-studio-compiler-util.cpp | 452 +++++++ .../slang-visual-studio-compiler-util.h | 42 + .../windows/slang-win-visual-studio-util.cpp | 366 +++++ .../windows/slang-win-visual-studio-util.h | 68 + source/core/slang-char-util.cpp | 11 +- source/core/slang-char-util.h | 9 +- source/core/slang-downstream-compiler.cpp | 716 ---------- source/core/slang-downstream-compiler.h | 501 ------- source/core/slang-file-system.cpp | 888 +++++++++++++ source/core/slang-file-system.h | 253 ++++ source/core/slang-gcc-compiler-util.cpp | 645 --------- source/core/slang-gcc-compiler-util.h | 56 - source/core/slang-name-convention-util.cpp | 213 --- source/core/slang-name-convention-util.h | 54 - source/core/slang-nvrtc-compiler.cpp | 773 ----------- source/core/slang-nvrtc-compiler.h | 19 - source/core/slang-string.cpp | 6 + source/core/slang-visual-studio-compiler-util.cpp | 452 ------- source/core/slang-visual-studio-compiler-util.h | 42 - .../core/windows/slang-win-visual-studio-util.cpp | 366 ----- source/core/windows/slang-win-visual-studio-util.h | 68 - source/slang/slang-ast-support-types.h | 6 +- source/slang/slang-compiler.cpp | 4 +- source/slang/slang-compiler.h | 9 +- source/slang/slang-diagnostic-defs.h | 16 - source/slang/slang-diagnostics.cpp | 544 +------- source/slang/slang-diagnostics.h | 273 +--- source/slang/slang-emit-c-like.cpp | 4 +- source/slang/slang-emit.cpp | 4 +- source/slang/slang-file-system.cpp | 888 ------------- source/slang/slang-file-system.h | 253 ---- source/slang/slang-include-system.h | 2 +- source/slang/slang-ir-legalize-types.cpp | 3 +- source/slang/slang-ir.h | 5 +- source/slang/slang-legalize-types.h | 3 +- source/slang/slang-lexer.cpp | 1400 -------------------- source/slang/slang-lexer.h | 164 --- source/slang/slang-lookup.cpp | 3 +- source/slang/slang-mangle.cpp | 2 +- source/slang/slang-name.cpp | 42 - source/slang/slang-name.h | 89 -- source/slang/slang-options.cpp | 3 +- source/slang/slang-parser.h | 3 +- source/slang/slang-preprocessor.cpp | 2 +- source/slang/slang-preprocessor.h | 2 +- source/slang/slang-repro.cpp | 2 +- source/slang/slang-repro.h | 3 +- source/slang/slang-serialize-ir-types.h | 4 +- source/slang/slang-serialize-misc-type-info.h | 2 +- source/slang/slang-serialize-reflection.h | 2 +- source/slang/slang-serialize-source-loc.h | 4 +- source/slang/slang-serialize.h | 2 +- source/slang/slang-source-loc.cpp | 689 ---------- source/slang/slang-source-loc.h | 487 ------- source/slang/slang-token-defs.h | 96 -- source/slang/slang-token.cpp | 24 - source/slang/slang-token.h | 140 -- source/slang/slang.cpp | 5 +- 81 files changed, 9258 insertions(+), 9026 deletions(-) create mode 100644 source/compiler-core/slang-core-diagnostics.cpp create mode 100644 source/compiler-core/slang-core-diagnostics.h create mode 100644 source/compiler-core/slang-diagnostic-sink.cpp create mode 100644 source/compiler-core/slang-diagnostic-sink.h create mode 100644 source/compiler-core/slang-downstream-compiler.cpp create mode 100644 source/compiler-core/slang-downstream-compiler.h create mode 100644 source/compiler-core/slang-gcc-compiler-util.cpp create mode 100644 source/compiler-core/slang-gcc-compiler-util.h create mode 100644 source/compiler-core/slang-lexer-diagnostic-defs.h create mode 100644 source/compiler-core/slang-lexer.cpp create mode 100644 source/compiler-core/slang-lexer.h create mode 100644 source/compiler-core/slang-misc-diagnostic-defs.h create mode 100644 source/compiler-core/slang-name-convention-util.cpp create mode 100644 source/compiler-core/slang-name-convention-util.h create mode 100644 source/compiler-core/slang-name.cpp create mode 100644 source/compiler-core/slang-name.h create mode 100644 source/compiler-core/slang-nvrtc-compiler.cpp create mode 100644 source/compiler-core/slang-nvrtc-compiler.h create mode 100644 source/compiler-core/slang-source-loc.cpp create mode 100644 source/compiler-core/slang-source-loc.h create mode 100644 source/compiler-core/slang-token-defs.h create mode 100644 source/compiler-core/slang-token.cpp create mode 100644 source/compiler-core/slang-token.h create mode 100644 source/compiler-core/slang-visual-studio-compiler-util.cpp create mode 100644 source/compiler-core/slang-visual-studio-compiler-util.h create mode 100644 source/compiler-core/windows/slang-win-visual-studio-util.cpp create mode 100644 source/compiler-core/windows/slang-win-visual-studio-util.h delete mode 100644 source/core/slang-downstream-compiler.cpp delete mode 100644 source/core/slang-downstream-compiler.h create mode 100644 source/core/slang-file-system.cpp create mode 100644 source/core/slang-file-system.h delete mode 100644 source/core/slang-gcc-compiler-util.cpp delete mode 100644 source/core/slang-gcc-compiler-util.h delete mode 100644 source/core/slang-name-convention-util.cpp delete mode 100644 source/core/slang-name-convention-util.h delete mode 100644 source/core/slang-nvrtc-compiler.cpp delete mode 100644 source/core/slang-nvrtc-compiler.h delete mode 100644 source/core/slang-visual-studio-compiler-util.cpp delete mode 100644 source/core/slang-visual-studio-compiler-util.h delete mode 100644 source/core/windows/slang-win-visual-studio-util.cpp delete mode 100644 source/core/windows/slang-win-visual-studio-util.h delete mode 100644 source/slang/slang-file-system.cpp delete mode 100644 source/slang/slang-file-system.h delete mode 100644 source/slang/slang-lexer.cpp delete mode 100644 source/slang/slang-lexer.h delete mode 100644 source/slang/slang-name.cpp delete mode 100644 source/slang/slang-name.h delete mode 100644 source/slang/slang-source-loc.cpp delete mode 100644 source/slang/slang-source-loc.h delete mode 100644 source/slang/slang-token-defs.h delete mode 100644 source/slang/slang-token.cpp delete mode 100644 source/slang/slang-token.h (limited to 'source') diff --git a/source/compiler-core/slang-core-diagnostics.cpp b/source/compiler-core/slang-core-diagnostics.cpp new file mode 100644 index 000000000..7aa6c5c14 --- /dev/null +++ b/source/compiler-core/slang-core-diagnostics.cpp @@ -0,0 +1,50 @@ +// slang-core-diagnostics.cpp +#include "slang-core-diagnostics.h" + +namespace Slang { + +namespace MiscDiagnostics +{ +#define DIAGNOSTIC(id, severity, name, messageFormat) const DiagnosticInfo name = { id, Severity::severity, #name, messageFormat }; +#include "slang-misc-diagnostic-defs.h" +#undef DIAGNOSTIC +} + +static const DiagnosticInfo* const kMiscDiagnostics[] = +{ +#define DIAGNOSTIC(id, severity, name, messageFormat) &MiscDiagnostics::name, +#include "slang-misc-diagnostic-defs.h" +#undef DIAGNOSTIC +}; + + +namespace LexerDiagnostics +{ +#define DIAGNOSTIC(id, severity, name, messageFormat) const DiagnosticInfo name = { id, Severity::severity, #name, messageFormat }; +#include "slang-lexer-diagnostic-defs.h" +#undef DIAGNOSTIC +} + +static const DiagnosticInfo* const kLexerDiagnostics[] = +{ +#define DIAGNOSTIC(id, severity, name, messageFormat) &LexerDiagnostics::name, +#include "slang-lexer-diagnostic-defs.h" +#undef DIAGNOSTIC +}; + +static DiagnosticsLookup* _newCoreDiagnosticsLookup() +{ + auto lookup = new DiagnosticsLookup; + lookup->add(kMiscDiagnostics, SLANG_COUNT_OF(kMiscDiagnostics)); + lookup->add(kLexerDiagnostics, SLANG_COUNT_OF(kLexerDiagnostics)); + + return lookup; +} + +DiagnosticsLookup* getCoreDiagnosticsLookup() +{ + static RefPtr s_lookup = _newCoreDiagnosticsLookup(); + return s_lookup; +} + +} // namespace Slang diff --git a/source/compiler-core/slang-core-diagnostics.h b/source/compiler-core/slang-core-diagnostics.h new file mode 100644 index 000000000..2d56591fd --- /dev/null +++ b/source/compiler-core/slang-core-diagnostics.h @@ -0,0 +1,32 @@ +#ifndef SLANG_CORE_DIAGNOSTICS_H +#define SLANG_CORE_DIAGNOSTICS_H + +#include "../core/slang-basic.h" +#include "../core/slang-writer.h" + +#include "slang-source-loc.h" +#include "slang-diagnostic-sink.h" +#include "slang-token.h" + +#include "../../slang.h" + +namespace Slang +{ + +DiagnosticsLookup* getCoreDiagnosticsLookup(); + +namespace MiscDiagnostics +{ +#define DIAGNOSTIC(id, severity, name, messageFormat) extern const DiagnosticInfo name; +#include "slang-misc-diagnostic-defs.h" +} + +namespace LexerDiagnostics +{ +#define DIAGNOSTIC(id, severity, name, messageFormat) extern const DiagnosticInfo name; +#include "slang-lexer-diagnostic-defs.h" +} + +} + +#endif diff --git a/source/compiler-core/slang-diagnostic-sink.cpp b/source/compiler-core/slang-diagnostic-sink.cpp new file mode 100644 index 000000000..2adf31f69 --- /dev/null +++ b/source/compiler-core/slang-diagnostic-sink.cpp @@ -0,0 +1,537 @@ +// slang-diagnostic-sink.cpp +#include "slang-diagnostic-sink.h" + +#include "slang-name.h" +#include "slang-core-diagnostics.h" +#include "slang-name-convention-util.h" + +#include "../core/slang-memory-arena.h" +#include "../core/slang-dictionary.h" +#include "../core/slang-string-util.h" +#include "../core/slang-char-util.h" + +namespace Slang { + +void printDiagnosticArg(StringBuilder& sb, char const* str) +{ + sb << str; +} + +void printDiagnosticArg(StringBuilder& sb, int32_t val) +{ + sb << val; +} + +void printDiagnosticArg(StringBuilder& sb, uint32_t val) +{ + sb << val; +} + +void printDiagnosticArg(StringBuilder& sb, int64_t val) +{ + sb << val; +} + +void printDiagnosticArg(StringBuilder& sb, uint64_t val) +{ + sb << val; +} + +void printDiagnosticArg(StringBuilder& sb, double val) +{ + sb << val; +} + +void printDiagnosticArg(StringBuilder& sb, Slang::String const& str) +{ + sb << str; +} + +void printDiagnosticArg(StringBuilder& sb, Slang::UnownedStringSlice const& str) +{ + sb.append(str); +} + + +void printDiagnosticArg(StringBuilder& sb, Name* name) +{ + sb << getText(name); +} + + +void printDiagnosticArg(StringBuilder& sb, TokenType tokenType) +{ + sb << TokenTypeToString(tokenType); +} + +void printDiagnosticArg(StringBuilder& sb, Token const& token) +{ + sb << token.getContent(); +} + +SourceLoc const& getDiagnosticPos(Token const& token) +{ + return token.loc; +} + +// Take the format string for a diagnostic message, along with its arguments, and turn it into a +static void formatDiagnosticMessage(StringBuilder& sb, char const* format, int argCount, DiagnosticArg const* const* args) +{ + char const* spanBegin = format; + for(;;) + { + char const* spanEnd = spanBegin; + while (int c = *spanEnd) + { + if (c == '$') + break; + spanEnd++; + } + + sb.Append(spanBegin, int(spanEnd - spanBegin)); + if (!*spanEnd) + return; + + SLANG_ASSERT(*spanEnd == '$'); + spanEnd++; + int d = *spanEnd++; + switch (d) + { + // A double dollar sign `$$` is used to emit a single `$` + case '$': + sb.Append('$'); + break; + + // A single digit means to emit the corresponding argument. + // TODO: support more than 10 arguments, and add options + // to control formatting, etc. + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + { + int index = d - '0'; + if (index >= argCount) + { + // TODO(tfoley): figure out what a good policy will be for "panic" situations like this + throw InvalidOperationException("too few arguments for diagnostic message"); + } + else + { + DiagnosticArg const* arg = args[index]; + arg->printFunc(sb, arg->data); + } + } + break; + + default: + throw InvalidOperationException("invalid diagnostic message format"); + break; + } + + spanBegin = spanEnd; + } +} + +static void formatDiagnostic(const HumaneSourceLoc& humaneLoc, Diagnostic const& diagnostic, StringBuilder& outBuilder) +{ + outBuilder << humaneLoc.pathInfo.foundPath; + outBuilder << "("; + outBuilder << Int32(humaneLoc.line); + outBuilder << "): "; + + outBuilder << getSeverityName(diagnostic.severity); + + if (diagnostic.ErrorID >= 0) + { + outBuilder << " "; + outBuilder << diagnostic.ErrorID; + } + + outBuilder << ": "; + outBuilder << diagnostic.Message; + outBuilder << "\n"; +} + +static void _replaceTabWithSpaces(const UnownedStringSlice& slice, Int tabSize, StringBuilder& out) +{ + const char* start = slice.begin(); + const char*const end = slice.end(); + + const Index startLength = out.getLength(); + + for (const char* cur = start; cur < end; cur++) + { + if (*cur == '\t') + { + if (start < cur) + { + out.append(start, cur); + } + + // The amount of spaces we add depends on the current position. + const Index lastPosition = out.getLength() - startLength; + Index tabPosition = lastPosition; + + // Strip the tabPosition so it's back to the tab stop + // Special case if tabSize is a power of 2 + if ((tabSize & (tabSize - 1)) == 0) + { + tabPosition = tabPosition & ~Index(tabSize - 1); + } + else + { + tabPosition -= tabPosition % tabSize; + } + + // Move to next tab + tabPosition += tabSize; + + // The amount of spaces to simulate the tab + const Index spacesCount = tabPosition - lastPosition; + + // Add the spaces + out.appendRepeatedChar(' ', spacesCount); + + // Set the start at the first character past + start = cur + 1; + } + } + + if (start < end) + { + out.append(start, end); + } +} + +// Given multi-line text, and a position within the text (as a pointer into the memory of text) +// extract the line that contains pos +static UnownedStringSlice _extractLineContainingPosition(const UnownedStringSlice& text, const char* pos) +{ + SLANG_ASSERT(text.isMemoryContained(pos)); + + const char*const contentStart = text.begin(); + const char*const contentEnd = text.end(); + + // We want to determine the start of the line, and the end of the line + const char* start = pos; + for (; start > contentStart; --start) + { + const char c = *start; + if (c == '\n' || c == '\r') + { + // We want the character after, but we can only do this if not already at pos + start += int(start < pos); + break; + } + } + const char* end = pos; + for (; end < contentEnd; ++end) + { + const char c = *end; + if (c == '\n' || c == '\r') + { + break; + } + } + + return UnownedStringSlice(start, end); +} + +static void _sourceLocationNoteDiagnostic(SourceView* sourceView, SourceLoc sourceLoc, DiagnosticSink::SourceLocationLexer lexer, StringBuilder& sb) +{ + SourceFile* sourceFile = sourceView->getSourceFile(); + if (!sourceFile) + { + return; + } + + UnownedStringSlice content = sourceFile->getContent(); + + // Make sure the offset is within content. + // This is important because it's possible to have a 'SourceFile' that doesn't contain any content + // (for example when reconstructed via serialization with just line offsets, the actual source text 'content' isn't available). + const int offset = sourceView->getRange().getOffset(sourceLoc); + if (offset < 0 || offset >= content.getLength()) + { + return; + } + + // Work out the position of the SourceLoc in the source + const char*const pos = content.begin() + offset; + + UnownedStringSlice line = _extractLineContainingPosition(content, pos); + + // Trim any trailing white space + line = UnownedStringSlice(line.begin(), line.trim().end()); + + // TODO(JS): The tab size should ideally be configurable from command line. + // For now just go with 4. + const Index tabSize = 4; + + StringBuilder sourceLine; + StringBuilder caretLine; + + // First work out the sourceLine + _replaceTabWithSpaces(line, tabSize, sourceLine); + + // Now the caretLine which appears underneath the sourceLine + { + // Produce the text up to the caret position (at pos), taking into account tabs + _replaceTabWithSpaces(UnownedStringSlice(line.begin(), pos), tabSize, caretLine); + + // Now make all spaces + const Index length = caretLine.getLength(); + caretLine.Clear(); + caretLine.appendRepeatedChar(' ', length); + + // Add caret + caretLine << "^"; + + if (lexer) + { + UnownedStringSlice token = lexer(UnownedStringSlice(pos, line.end())); + + if (token.getLength() > 1) + { + caretLine.appendRepeatedChar('~', token.getLength() - 1); + } + } + } + + // We could have handling here for if the line is too long, that we surround the important section + // will ellipsis for example. + // For now we just output. + + sb << sourceLine << "\n"; + sb << caretLine << "\n"; +} + + +static void formatDiagnostic( + DiagnosticSink* sink, + Diagnostic const& diagnostic, + StringBuilder& sb) +{ + auto sourceManager = sink->getSourceManager(); + + SourceView* sourceView = nullptr; + HumaneSourceLoc humaneLoc; + const auto sourceLoc = diagnostic.loc; + { + sourceView = sourceManager->findSourceViewRecursively(sourceLoc); + if (sourceView) + { + humaneLoc = sourceView->getHumaneLoc(sourceLoc); + } + formatDiagnostic(humaneLoc, diagnostic, sb); + + { + SourceView* currentView = sourceView; + + while (currentView && currentView->getInitiatingSourceLoc().isValid() && currentView->getSourceFile()->getPathInfo().type == PathInfo::Type::TokenPaste) + { + SourceView* initiatingView = sourceManager->findSourceView(currentView->getInitiatingSourceLoc()); + if (initiatingView == nullptr) + { + break; + } + + const DiagnosticInfo& diagnosticInfo = MiscDiagnostics::seeTokenPasteLocation; + + // Turn the message format into a message. For the moment it assumes no parameters. + StringBuilder msg; + formatDiagnosticMessage(msg, diagnosticInfo.messageFormat, 0, nullptr); + + // Set up the diagnostic. + Diagnostic initiationDiagnostic; + initiationDiagnostic.ErrorID = diagnosticInfo.id; + initiationDiagnostic.Message = msg.ProduceString(); + initiationDiagnostic.loc = sourceView->getInitiatingSourceLoc(); + initiationDiagnostic.severity = diagnosticInfo.severity; + + // TODO(JS): + // Not 100% clear what the best sourceLoc type is most useful here - we will go with default for now + HumaneSourceLoc pasteHumaneLoc = initiatingView->getHumaneLoc(sourceView->getInitiatingSourceLoc()); + + // Okay we should output where the token paste took place + formatDiagnostic(pasteHumaneLoc, initiationDiagnostic, sb); + + // Make the initiatingView the current view + currentView = initiatingView; + } + } + } + + // We don't don't output source line information if this is a 'note' as a note is extra information for one + // of the other main severity types, and so the information should already be output on the initial line + if (sourceView && sink->isFlagSet(DiagnosticSink::Flag::SourceLocationLine) && diagnostic.severity != Severity::Note) + { + _sourceLocationNoteDiagnostic(sourceView, sourceLoc, sink->getSourceLocationLexer(), sb); + } + + if (sourceView && sink->isFlagSet(DiagnosticSink::Flag::VerbosePath)) + { + auto actualHumaneLoc = sourceView->getHumaneLoc(diagnostic.loc, SourceLocType::Actual); + + // Look up the path verbosely (will get the canonical path if necessary) + actualHumaneLoc.pathInfo.foundPath = sourceView->getSourceFile()->calcVerbosePath(); + + // Only output if it's actually different + if (actualHumaneLoc.pathInfo.foundPath != humaneLoc.pathInfo.foundPath || + actualHumaneLoc.line != humaneLoc.line || + actualHumaneLoc.column != humaneLoc.column) + { + formatDiagnostic(actualHumaneLoc, diagnostic, sb); + } + } +} + +void DiagnosticSink::diagnoseImpl(SourceLoc const& pos, DiagnosticInfo const& info, int argCount, DiagnosticArg const* const* args) +{ + StringBuilder sb; + formatDiagnosticMessage(sb, info.messageFormat, argCount, args); + + Diagnostic diagnostic; + diagnostic.ErrorID = info.id; + diagnostic.Message = sb.ProduceString(); + diagnostic.loc = pos; + diagnostic.severity = info.severity; + + if (diagnostic.severity >= Severity::Error) + { + m_errorCount++; + } + + // Did the client supply a callback for us to use? + if( writer ) + { + // If so, pass the error string along to them + StringBuilder messageBuilder; + formatDiagnostic(this, diagnostic, messageBuilder); + + writer->write(messageBuilder.getBuffer(), messageBuilder.getLength()); + } + else + { + // If the user doesn't have a callback, then just + // collect our diagnostic messages into a buffer + formatDiagnostic(this, diagnostic, outputBuffer); + } + + if (diagnostic.severity >= Severity::Fatal) + { + // TODO: figure out a better policy for aborting compilation + throw AbortCompilationException(); + } +} + +void DiagnosticSink::diagnoseRaw( + Severity severity, + char const* message) +{ + return diagnoseRaw(severity, UnownedStringSlice(message)); +} + +void DiagnosticSink::diagnoseRaw( + Severity severity, + const UnownedStringSlice& message) +{ + if (severity >= Severity::Error) + { + m_errorCount++; + } + + // Did the client supply a callback for us to use? + if(writer) + { + // If so, pass the error string along to them + writer->write(message.begin(), message.getLength()); + } + else + { + // If the user doesn't have a callback, then just + // collect our diagnostic messages into a buffer + outputBuffer.append(message); + } + + if (severity >= Severity::Fatal) + { + // TODO: figure out a better policy for aborting compilation + throw InvalidOperationException(); + } +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DiagnosticLookup !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +void DiagnosticsLookup::_add(const char* name, Index index) +{ + UnownedStringSlice nameSlice(name); + m_map.Add(nameSlice, index); + + // Add a dashed version (KababCase) + { + m_work.Clear(); + + NameConventionUtil::convert(NameConvention::Camel, nameSlice, CharCase::Lower, NameConvention::Kabab, m_work); + + UnownedStringSlice dashSlice(m_arena.allocateString(m_work.getBuffer(), m_work.getLength()), m_work.getLength()); + + m_map.AddIfNotExists(dashSlice, index); + } +} +void DiagnosticsLookup::addAlias(const char* name, const char* diagnosticName) +{ + const Index index = _findDiagnosticIndex(UnownedStringSlice(diagnosticName)); + SLANG_ASSERT(index >= 0); + if (index >= 0) + { + _add(name, index); + } +} + +Index DiagnosticsLookup::add(const DiagnosticInfo* info) +{ + // Check it's not already added + SLANG_ASSERT(m_diagnostics.indexOf(info) < 0); + + const Index index = m_diagnostics.getCount(); + + _add(info->name, index); + + m_diagnostics.add(info); + return index; +} + +void DiagnosticsLookup::add(const DiagnosticInfo*const* infos, Index infosCount) +{ + for (Index i = 0; i < infosCount; ++i) + { + add(infos[i]); + } +} + +DiagnosticsLookup::DiagnosticsLookup(): + m_arena(kArenaInitialSize) +{ +} + +DiagnosticsLookup::DiagnosticsLookup(const DiagnosticInfo*const* diagnostics, Index diagnosticsCount) : + m_arena(kArenaInitialSize) +{ + m_diagnostics.addRange(diagnostics, diagnosticsCount); + + // TODO: We should eventually have a more formal system for associating individual + // diagnostics, or groups of diagnostics, with user-exposed names for use when + // enabling/disabling warnings (or turning warnings into errors, etc.). + // + // For now we build a map from diagnostic name to it's entry. Two entries are typically + // added - the 'original name' as associated with the diagnostic in lowerCamel, and + // a dashified version. + + for (Index i = 0; i < diagnosticsCount; ++i) + { + const DiagnosticInfo* diagnostic = diagnostics[i]; + _add(diagnostic->name, i); + } +} + +} // namespace Slang diff --git a/source/compiler-core/slang-diagnostic-sink.h b/source/compiler-core/slang-diagnostic-sink.h new file mode 100644 index 000000000..84001e6f5 --- /dev/null +++ b/source/compiler-core/slang-diagnostic-sink.h @@ -0,0 +1,316 @@ +#ifndef SLANG_DIAGNOSTIC_SINK_H +#define SLANG_DIAGNOSTIC_SINK_H + +#include "../core/slang-basic.h" +#include "../core/slang-writer.h" +#include "../core/slang-memory-arena.h" + +#include "slang-source-loc.h" +#include "slang-token.h" + +#include "../../slang.h" + +namespace Slang +{ + +enum class Severity +{ + Note, + Warning, + Error, + Fatal, + Internal, +}; + +// TODO(tfoley): move this into a source file... +inline const char* getSeverityName(Severity severity) +{ + switch (severity) + { + case Severity::Note: return "note"; + case Severity::Warning: return "warning"; + case Severity::Error: return "error"; + case Severity::Fatal: return "fatal error"; + case Severity::Internal: return "internal error"; + default: return "unknown error"; + } +} + +// A structure to be used in static data describing different +// diagnostic messages. +struct DiagnosticInfo +{ + int id; + Severity severity; + char const* name; ///< Unique name + char const* messageFormat; +}; + +class Diagnostic +{ +public: + String Message; + SourceLoc loc; + int ErrorID; + Severity severity; + + Diagnostic() + { + ErrorID = -1; + } + Diagnostic( + const String & msg, + int id, + const SourceLoc & pos, + Severity severity) + : severity(severity) + { + Message = msg; + ErrorID = id; + loc = pos; + } +}; + +class Name; + +void printDiagnosticArg(StringBuilder& sb, char const* str); + +void printDiagnosticArg(StringBuilder& sb, int32_t val); +void printDiagnosticArg(StringBuilder& sb, uint32_t val); + +void printDiagnosticArg(StringBuilder& sb, int64_t val); +void printDiagnosticArg(StringBuilder& sb, uint64_t val); + +void printDiagnosticArg(StringBuilder& sb, double val); + +void printDiagnosticArg(StringBuilder& sb, Slang::String const& str); +void printDiagnosticArg(StringBuilder& sb, Slang::UnownedStringSlice const& str); +void printDiagnosticArg(StringBuilder& sb, Name* name); + +void printDiagnosticArg(StringBuilder& sb, TokenType tokenType); +void printDiagnosticArg(StringBuilder& sb, Token const& token); + +struct IRInst; +void printDiagnosticArg(StringBuilder& sb, IRInst* irObject); + +template +void printDiagnosticArg(StringBuilder& sb, RefPtr ptr) +{ + printDiagnosticArg(sb, ptr.Ptr()); +} + +inline SourceLoc const& getDiagnosticPos(SourceLoc const& pos) { return pos; } + +SourceLoc const& getDiagnosticPos(Token const& token); + + +template +SourceLoc getDiagnosticPos(RefPtr const& ptr) +{ + return getDiagnosticPos(ptr.Ptr()); +} + +struct DiagnosticArg +{ + void* data; + void (*printFunc)(StringBuilder&, void*); + + template + struct Helper + { + static void printFunc(StringBuilder& sb, void* data) { printDiagnosticArg(sb, *(T*)data); } + }; + + template + DiagnosticArg(T const& arg) + : data((void*)&arg) + , printFunc(&Helper::printFunc) + {} +}; + + +class DiagnosticSink +{ +public: + /// Flags to control some aspects of Diagnostic sink behavior + typedef uint32_t Flags; + struct Flag + { + enum Enum: Flags + { + VerbosePath = 0x1, ///< Will display a more verbose path (if available) - such as a canonical or absolute path + SourceLocationLine = 0x2, ///< If set will display the location line if source is available + }; + }; + + /// Used by diagnostic sink to be able to underline tokens. If not defined on the DiagnosticSink, + /// will only display a caret at the SourceLoc + typedef UnownedStringSlice(*SourceLocationLexer)(const UnownedStringSlice& text); + + /// Get the total amount of errors that have taken place on this DiagnosticSink + SLANG_FORCE_INLINE int getErrorCount() { return m_errorCount; } + + void diagnoseDispatch(SourceLoc const& pos, DiagnosticInfo const& info) + { + diagnoseImpl(pos, info, 0, nullptr); + } + + void diagnoseDispatch(SourceLoc const& pos, DiagnosticInfo const& info, DiagnosticArg const& arg0) + { + DiagnosticArg const* args[] = { &arg0 }; + diagnoseImpl(pos, info, 1, args); + } + + void diagnoseDispatch(SourceLoc const& pos, DiagnosticInfo const& info, DiagnosticArg const& arg0, DiagnosticArg const& arg1) + { + DiagnosticArg const* args[] = { &arg0, &arg1 }; + diagnoseImpl(pos, info, 2, args); + } + + void diagnoseDispatch(SourceLoc const& pos, DiagnosticInfo const& info, DiagnosticArg const& arg0, DiagnosticArg const& arg1, DiagnosticArg const& arg2) + { + DiagnosticArg const* args[] = { &arg0, &arg1, &arg2 }; + diagnoseImpl(pos, info, 3, args); + } + + void diagnoseDispatch(SourceLoc const& pos, DiagnosticInfo const& info, DiagnosticArg const& arg0, DiagnosticArg const& arg1, DiagnosticArg const& arg2, DiagnosticArg const& arg3) + { + DiagnosticArg const* args[] = { &arg0, &arg1, &arg2, &arg3 }; + diagnoseImpl(pos, info, 4, args); + } + + template + void diagnose(P const& pos, DiagnosticInfo const& info, Args const&... args ) + { + diagnoseDispatch(getDiagnosticPos(pos), info, args...); + } + + // Add a diagnostic with raw text + // (used when we get errors from a downstream compiler) + void diagnoseRaw(Severity severity, char const* message); + void diagnoseRaw(Severity severity, const UnownedStringSlice& message); + + /// During propagation of an exception for an internal + /// error, note that this source location was involved + void noteInternalErrorLoc(SourceLoc const& loc); + + /// Create a blob containing diagnostics if there were any errors. + /// *note* only works if writer is not set, the blob is created from outputBuffer + SlangResult getBlobIfNeeded(ISlangBlob** outBlob); + + /// Get the source manager used + SourceManager* getSourceManager() const { return m_sourceManager; } + /// Set the source manager used for lookup of source locs + void setSourceManager(SourceManager* inSourceManager) { m_sourceManager = inSourceManager; } + + /// Get the flags + Flags getFlags() const { return m_flags; } + /// Set a flag + void setFlag(Flag::Enum flag) { m_flags |= Flags(flag); } + /// Reset a flag + void resetFlag(Flag::Enum flag) { m_flags &= ~Flags(flag); } + /// Test if flag is set + bool isFlagSet(Flag::Enum flag) { return (m_flags & Flags(flag)) != 0; } + + /// Get the (optional) diagnostic sink lexer. This is used to + /// improve quality of highlighting a locations token. If not set, will just have a single + /// character caret at location + SourceLocationLexer getSourceLocationLexer() const { return m_sourceLocationLexer; } + + /// Ctor + DiagnosticSink(SourceManager* sourceManager, SourceLocationLexer sourceLocationLexer) + : m_sourceManager(sourceManager), + m_sourceLocationLexer(sourceLocationLexer) + { + // If we have a source location lexer, we'll by default enable source location output + if (sourceLocationLexer) + { + setFlag(Flag::SourceLocationLine); + } + } + + // Public members + + /// The outputBuffer will contain any diagnostics *iff* the writer is *not* set + StringBuilder outputBuffer; + /// If a writer is set output will *not* be written to the outputBuffer + ISlangWriter* writer = nullptr; + +protected: + void diagnoseImpl(SourceLoc const& pos, DiagnosticInfo const& info, int argCount, DiagnosticArg const* const* args); + + int m_errorCount = 0; + int m_internalErrorLocsNoted = 0; + + Flags m_flags = 0; + + // The source manager to use when mapping source locations to file+line info + SourceManager* m_sourceManager = nullptr; + + SourceLocationLexer m_sourceLocationLexer; +}; + + /// An `ISlangWriter` that writes directly to a diagnostic sink. +class DiagnosticSinkWriter : public AppendBufferWriter +{ +public: + typedef AppendBufferWriter Super; + + DiagnosticSinkWriter(DiagnosticSink* sink) + : Super(WriterFlag::IsStatic) + , m_sink(sink) + {} + + // ISlangWriter + SLANG_NO_THROW virtual SlangResult SLANG_MCALL write(const char* chars, size_t numChars) SLANG_OVERRIDE + { + m_sink->diagnoseRaw(Severity::Note, UnownedStringSlice(chars, chars+numChars)); + return SLANG_OK; + } + +private: + DiagnosticSink* m_sink = nullptr; +}; + +class DiagnosticsLookup : public RefObject +{ +public: + static const Index kArenaInitialSize = 2048; + + const DiagnosticInfo* findDiagostic(const UnownedStringSlice& slice) const + { + const Index* indexPtr = m_map.TryGetValue(slice); + return indexPtr ? m_diagnostics[*indexPtr] : nullptr; + } + Index _findDiagnosticIndex(const UnownedStringSlice& slice) const + { + const Index* indexPtr = m_map.TryGetValue(slice); + return indexPtr ? *indexPtr : 0; + } + + /// info must stay in scope + Index add(const DiagnosticInfo* info); + void add(const DiagnosticInfo*const* infos, Index infosCount); + + void addAlias(const char* name, const char* diagnosticName); + + /// Get the diagnostics held in this lookup + const List& getDiagnostics() const { return m_diagnostics; } + + /// NOTE! diagnostics must stay in scope for lifetime of lookup + DiagnosticsLookup(const DiagnosticInfo*const* diagnostics, Index diagnosticsCount); + DiagnosticsLookup(); + +protected: + void _add(const char* name, Index index); + + List m_diagnostics; + + StringBuilder m_work; + Dictionary m_map; + MemoryArena m_arena; +}; + +} + +#endif diff --git a/source/compiler-core/slang-downstream-compiler.cpp b/source/compiler-core/slang-downstream-compiler.cpp new file mode 100644 index 000000000..696376b49 --- /dev/null +++ b/source/compiler-core/slang-downstream-compiler.cpp @@ -0,0 +1,716 @@ +// slang-downstream-compiler.cpp +#include "slang-downstream-compiler.h" + +#include "../core/slang-common.h" +#include "../../slang-com-helper.h" +#include "../core/slang-string-util.h" + +#include "../core/slang-type-text-util.h" + +#include "../core/slang-io.h" +#include "../core/slang-shared-library.h" +#include "../core/slang-blob.h" + +#ifdef SLANG_VC +# include "windows/slang-win-visual-studio-util.h" +#endif + +#include "slang-visual-studio-compiler-util.h" +#include "slang-gcc-compiler-util.h" +#include "slang-nvrtc-compiler.h" + +namespace Slang +{ + +static DownstreamCompiler::Infos _calcInfos() +{ + typedef DownstreamCompiler::Info Info; + typedef DownstreamCompiler::SourceLanguageFlag SourceLanguageFlag; + typedef DownstreamCompiler::SourceLanguageFlags SourceLanguageFlags; + + DownstreamCompiler::Infos infos; + + infos.infos[int(SLANG_PASS_THROUGH_CLANG)] = Info(SourceLanguageFlag::CPP | SourceLanguageFlag::C); + infos.infos[int(SLANG_PASS_THROUGH_VISUAL_STUDIO)] = Info(SourceLanguageFlag::CPP | SourceLanguageFlag::C); + infos.infos[int(SLANG_PASS_THROUGH_GCC)] = Info(SourceLanguageFlag::CPP | SourceLanguageFlag::C); + + infos.infos[int(SLANG_PASS_THROUGH_NVRTC)] = Info(SourceLanguageFlag::CUDA); + + infos.infos[int(SLANG_PASS_THROUGH_DXC)] = Info(SourceLanguageFlag::HLSL); + infos.infos[int(SLANG_PASS_THROUGH_FXC)] = Info(SourceLanguageFlag::HLSL); + infos.infos[int(SLANG_PASS_THROUGH_GLSLANG)] = Info(SourceLanguageFlag::GLSL); + + return infos; +} + +/* static */DownstreamCompiler::Infos DownstreamCompiler::s_infos = _calcInfos(); + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DownstreamCompiler::Desc !!!!!!!!!!!!!!!!!!!!!!*/ + +void DownstreamCompiler::Desc::appendAsText(StringBuilder& out) const +{ + out << TypeTextUtil::getPassThroughAsHumanText(type); + + // Append the version if there is a version + if (majorVersion || minorVersion) + { + out << " "; + out << majorVersion; + out << "."; + out << minorVersion; + } +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DownstreamDiagnostic !!!!!!!!!!!!!!!!!!!!!!!!*/ + +/* static */UnownedStringSlice DownstreamDiagnostic::getSeverityText(Severity severity) +{ + switch (severity) + { + default: return UnownedStringSlice::fromLiteral("Unknown"); + case Severity::Info: return UnownedStringSlice::fromLiteral("Info"); + case Severity::Warning: return UnownedStringSlice::fromLiteral("Warning"); + case Severity::Error: return UnownedStringSlice::fromLiteral("Error"); + } +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DownstreamCompiler !!!!!!!!!!!!!!!!!!!!!!!!!!!!!*/ + + +/* static */bool DownstreamCompiler::canCompile(SlangPassThrough compiler, SlangSourceLanguage sourceLanguage) +{ + const auto& info = getInfo(compiler); + return (info.sourceLanguageFlags & (SourceLanguageFlags(1) << int(sourceLanguage))) != 0; +} + +/* static */SlangCompileTarget DownstreamCompiler::getCompileTarget(SlangSourceLanguage sourceLanguage) +{ + switch (sourceLanguage) + { + case SLANG_SOURCE_LANGUAGE_HLSL: return SLANG_HLSL; + case SLANG_SOURCE_LANGUAGE_GLSL: return SLANG_GLSL; + case SLANG_SOURCE_LANGUAGE_C: return SLANG_C_SOURCE; + case SLANG_SOURCE_LANGUAGE_CPP: return SLANG_CPP_SOURCE; + case SLANG_SOURCE_LANGUAGE_CUDA: return SLANG_CUDA_SOURCE; + default: return SLANG_TARGET_UNKNOWN; + } +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DownstreamDiagnostics !!!!!!!!!!!!!!!!!!!!!!*/ + +Index DownstreamDiagnostics::getCountBySeverity(Diagnostic::Severity severity) const +{ + Index count = 0; + for (const auto& msg : diagnostics) + { + count += Index(msg.severity == severity); + } + return count; +} + +Int DownstreamDiagnostics::countByStage(Diagnostic::Stage stage, Index counts[Int(Diagnostic::Severity::CountOf)]) const +{ + Int count = 0; + ::memset(counts, 0, sizeof(Index) * Int(Diagnostic::Severity::CountOf)); + for (const auto& diagnostic : diagnostics) + { + if (diagnostic.stage == stage) + { + count++; + counts[Index(diagnostic.severity)]++; + } + } + return count++; +} + +static void _appendCounts(const Index counts[Int(DownstreamDiagnostic::Severity::CountOf)], StringBuilder& out) +{ + typedef DownstreamDiagnostic::Severity Severity; + + for (Index i = 0; i < Int(Severity::CountOf); i++) + { + if (counts[i] > 0) + { + out << DownstreamDiagnostic::getSeverityText(Severity(i)) << "(" << counts[i] << ") "; + } + } +} + +static void _appendSimplified(const Index counts[Int(DownstreamDiagnostic::Severity::CountOf)], StringBuilder& out) +{ + typedef DownstreamDiagnostic::Severity Severity; + for (Index i = 0; i < Int(Severity::CountOf); i++) + { + if (counts[i] > 0) + { + out << DownstreamDiagnostic::getSeverityText(Severity(i)) << " "; + } + } +} + +void DownstreamDiagnostics::appendSummary(StringBuilder& out) const +{ + Index counts[Int(Diagnostic::Severity::CountOf)]; + if (countByStage(Diagnostic::Stage::Compile, counts) > 0) + { + out << "Compile: "; + _appendCounts(counts, out); + out << "\n"; + } + if (countByStage(Diagnostic::Stage::Link, counts) > 0) + { + out << "Link: "; + _appendCounts(counts, out); + out << "\n"; + } +} + +void DownstreamDiagnostics::appendSimplifiedSummary(StringBuilder& out) const +{ + Index counts[Int(Diagnostic::Severity::CountOf)]; + if (countByStage(Diagnostic::Stage::Compile, counts) > 0) + { + out << "Compile: "; + _appendSimplified(counts, out); + out << "\n"; + } + if (countByStage(Diagnostic::Stage::Link, counts) > 0) + { + out << "Link: "; + _appendSimplified(counts, out); + out << "\n"; + } +} + +void DownstreamDiagnostics::removeBySeverity(Diagnostic::Severity severity) +{ + Index count = diagnostics.getCount(); + for (Index i = 0; i < count; ++i) + { + if (diagnostics[i].severity == severity) + { + diagnostics.removeAt(i); + i--; + count--; + } + } +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! CommandLineDownstreamCompileResult !!!!!!!!!!!!!!!!!!!!!!*/ + +SlangResult CommandLineDownstreamCompileResult::getHostCallableSharedLibrary(ComPtr& outLibrary) +{ + if (m_hostCallableSharedLibrary) + { + outLibrary = m_hostCallableSharedLibrary; + return SLANG_OK; + } + + // Okay we want to load + // Try loading the shared library + SharedLibrary::Handle handle; + if (SLANG_FAILED(SharedLibrary::loadWithPlatformPath(m_moduleFilePath.getBuffer(), handle))) + { + return SLANG_FAIL; + } + // The shared library needs to keep temp files in scope + RefPtr sharedLib(new TemporarySharedLibrary(handle, m_moduleFilePath)); + sharedLib->m_temporaryFileSet = m_temporaryFiles; + + m_hostCallableSharedLibrary = sharedLib; + outLibrary = m_hostCallableSharedLibrary; + return SLANG_OK; +} + +SlangResult CommandLineDownstreamCompileResult::getBinary(ComPtr& outBlob) +{ + if (m_binaryBlob) + { + outBlob = m_binaryBlob; + return SLANG_OK; + } + + // Read the binary + try + { + // Read the contents of the binary + List contents = File::readAllBytes(m_moduleFilePath); + + m_binaryBlob = new ScopeRefObjectBlob(ListBlob::moveCreate(contents), m_temporaryFiles); + outBlob = m_binaryBlob; + return SLANG_OK; + } + catch (const Slang::IOException&) + { + return SLANG_FAIL; + } +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! CommandLineDownstreamCompiler !!!!!!!!!!!!!!!!!!!!!!*/ + +static bool _isContentsInFile(const DownstreamCompiler::CompileOptions& options) +{ + if (options.sourceContentsPath.getLength() <= 0) + { + return false; + } + + // We can see if we can load it + if (File::exists(options.sourceContentsPath)) + { + // Here we look for the file on the regular file system (as opposed to using the + // ISlangFileSystem. This is unfortunate but necessary - because when we call out + // to the compiler all it is able to (currently) see are files on the file system. + // + // Note that it could be coincidence that the filesystem has a file that's identical in + // contents/name. That being the case though, any includes wouldn't work for a generated + // file either from some specialized ISlangFileSystem, so this is probably as good as it gets + // until we can integrate directly to a C/C++ compiler through say a shared library where we can control + // file system access. + try + { + String readContents = File::readAllText(options.sourceContentsPath); + // We should see if they are the same + return options.sourceContents == readContents.getUnownedSlice(); + } + catch (const Slang::IOException&) + { + } + } + return false; +} + +SlangResult CommandLineDownstreamCompiler::compile(const CompileOptions& inOptions, RefPtr& out) +{ + // Copy the command line options + CommandLine cmdLine(m_cmdLine); + + CompileOptions options(inOptions); + + // Find all the files that will be produced + RefPtr productFileSet(new TemporaryFileSet); + + if (options.modulePath.getLength() == 0 || options.sourceContents.getLength() != 0) + { + String modulePath = options.modulePath; + + // If there is no module path, generate one. + if (modulePath.getLength() == 0) + { + SLANG_RETURN_ON_FAIL(File::generateTemporary(UnownedStringSlice::fromLiteral("slang-generated"), modulePath)); + options.modulePath = modulePath; + } + + if (_isContentsInFile(options)) + { + options.sourceFiles.add(options.sourceContentsPath); + } + else + { + String compileSourcePath = modulePath; + + // NOTE: Strictly speaking producing filenames by modifying the generateTemporary path that may introduce a temp filename clash, but in practice is extraordinary unlikely + compileSourcePath.append("-src"); + + // Make the temporary filename have the appropriate extension. + if (options.sourceLanguage == SLANG_SOURCE_LANGUAGE_C) + { + compileSourcePath.append(".c"); + } + else + { + compileSourcePath.append(".cpp"); + } + + // Write it out + try + { + productFileSet->add(compileSourcePath); + + File::writeAllText(compileSourcePath, options.sourceContents); + } + catch (...) + { + return SLANG_FAIL; + } + + // Add it as a source file + options.sourceFiles.add(compileSourcePath); + } + + // There is no source contents + options.sourceContents = String(); + options.sourceContentsPath = String(); + } + + // Append command line args to the end of cmdLine using the target specific function for the specified options + SLANG_RETURN_ON_FAIL(calcArgs(options, cmdLine)); + + String moduleFilePath; + + { + StringBuilder builder; + SLANG_RETURN_ON_FAIL(calcModuleFilePath(options, builder)); + moduleFilePath = builder.ProduceString(); + } + + { + List paths; + SLANG_RETURN_ON_FAIL(calcCompileProducts(options, DownstreamCompiler::ProductFlag::All, paths)); + productFileSet->add(paths); + } + + ExecuteResult exeRes; + +#if 0 + // Test + { + String line = ProcessUtil::getCommandLineString(cmdLine); + printf("%s", line.getBuffer()); + } +#endif + + SLANG_RETURN_ON_FAIL(ProcessUtil::execute(cmdLine, exeRes)); + +#if 0 + { + printf("stdout=\"%s\"\nstderr=\"%s\"\nret=%d\n", exeRes.standardOutput.getBuffer(), exeRes.standardError.getBuffer(), int(exeRes.resultCode)); + } +#endif + + DownstreamDiagnostics diagnostics; + SLANG_RETURN_ON_FAIL(parseOutput(exeRes, diagnostics)); + + + out = new CommandLineDownstreamCompileResult(diagnostics, moduleFilePath, productFileSet); + + return SLANG_OK; +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!! DownstreamCompiler::Desc !!!!!!!!!!!!!!!!!!!!!!*/ + +static DownstreamCompiler::Desc _calcCompiledWithDesc() +{ + DownstreamCompiler::Desc desc; + +#if SLANG_VC + desc = WinVisualStudioUtil::getDesc(WinVisualStudioUtil::getCompiledVersion()); +#elif SLANG_CLANG + desc.type = SLANG_PASS_THROUGH_CLANG; + desc.majorVersion = Int(__clang_major__); + desc.minorVersion = Int(__clang_minor__); +#elif SLANG_GCC + desc.type = SLANG_PASS_THROUGH_GCC; + desc.majorVersion = Int(__GNUC__); + desc.minorVersion = Int(__GNUC_MINOR__); +#else + // TODO(JS): Hmmm None is not quite the same as unknown. It works for now, but we might want to have a distinct enum for unknown. + desc.type = SLANG_PASS_THROUGH_NONE; +#endif + + return desc; +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!! DownstreamCompilerUtil !!!!!!!!!!!!!!!!!!!!!!*/ + +const DownstreamCompiler::Desc& DownstreamCompilerUtil::getCompiledWithDesc() +{ + static DownstreamCompiler::Desc s_desc = _calcCompiledWithDesc(); + return s_desc; +} + +/* static */DownstreamCompiler* DownstreamCompilerUtil::findCompiler(const DownstreamCompilerSet* set, MatchType matchType, const DownstreamCompiler::Desc& desc) +{ + List compilers; + set->getCompilers(compilers); + return findCompiler(compilers, matchType, desc); +} + +/* static */DownstreamCompiler* DownstreamCompilerUtil::findCompiler(const List& compilers, MatchType matchType, const DownstreamCompiler::Desc& desc) +{ + if (compilers.getCount() <= 0) + { + return nullptr; + } + + Int bestIndex = -1; + + const SlangPassThrough compilerType = desc.type; + + Int maxVersionValue = 0; + Int minVersionDiff = 0x7fffffff; + + Int descVersionValue = desc.getVersionValue(); + + // If we don't have version set, then anything 0 or above is good enough, and just take newest + if (descVersionValue == 0) + { + maxVersionValue = -1; + matchType = MatchType::Newest; + } + + for (Index i = 0; i < compilers.getCount(); ++i) + { + DownstreamCompiler* compiler = compilers[i]; + auto compilerDesc = compiler->getDesc(); + + if (compilerType == compilerDesc.type) + { + const Int versionValue = compilerDesc.getVersionValue(); + switch (matchType) + { + case MatchType::MinGreaterEqual: + { + auto diff = descVersionValue - versionValue; + if (diff >= 0 && diff < minVersionDiff) + { + bestIndex = i; + minVersionDiff = diff; + } + break; + } + case MatchType::MinAbsolute: + { + auto diff = descVersionValue - versionValue; + diff = (diff >= 0) ? diff : -diff; + if (diff < minVersionDiff) + { + bestIndex = i; + minVersionDiff = diff; + } + break; + } + case MatchType::Newest: + { + if (versionValue > maxVersionValue) + { + maxVersionValue = versionValue; + bestIndex = i; + } + break; + } + } + } + } + + return (bestIndex >= 0) ? compilers[bestIndex] : nullptr; +} + +/* static */DownstreamCompiler* DownstreamCompilerUtil::findClosestCompiler(const List& compilers, const DownstreamCompiler::Desc& desc) +{ + DownstreamCompiler* compiler; + + compiler = findCompiler(compilers, MatchType::MinGreaterEqual, desc); + if (compiler) + { + return compiler; + } + compiler = findCompiler(compilers, MatchType::MinAbsolute, desc); + if (compiler) + { + return compiler; + } + + // If we are gcc, we can try clang and vice versa + if (desc.type == SLANG_PASS_THROUGH_GCC || desc.type == SLANG_PASS_THROUGH_CLANG) + { + DownstreamCompiler::Desc compatible = desc; + compatible.type = (compatible.type == SLANG_PASS_THROUGH_CLANG) ? SLANG_PASS_THROUGH_GCC : SLANG_PASS_THROUGH_CLANG; + + compiler = findCompiler(compilers, MatchType::MinGreaterEqual, compatible); + if (compiler) + { + return compiler; + } + compiler = findCompiler(compilers, MatchType::MinAbsolute, compatible); + if (compiler) + { + return compiler; + } + } + + return nullptr; +} + +/* static */DownstreamCompiler* DownstreamCompilerUtil::findClosestCompiler(const DownstreamCompilerSet* set, const DownstreamCompiler::Desc& desc) +{ + DownstreamCompiler* compiler = set->getCompiler(desc); + if (compiler) + { + return compiler; + } + List compilers; + set->getCompilers(compilers); + return findClosestCompiler(compilers, desc); +} + +/* static */void DownstreamCompilerUtil::updateDefault(DownstreamCompilerSet* set, SlangSourceLanguage sourceLanguage) +{ + DownstreamCompiler* compiler = nullptr; + + switch (sourceLanguage) + { + case SLANG_SOURCE_LANGUAGE_CPP: + case SLANG_SOURCE_LANGUAGE_C: + { + compiler = findClosestCompiler(set, getCompiledWithDesc()); + break; + } + case SLANG_SOURCE_LANGUAGE_CUDA: + { + DownstreamCompiler::Desc desc; + desc.type = SLANG_PASS_THROUGH_NVRTC; + compiler = findCompiler(set, MatchType::Newest, desc); + break; + } + default: break; + } + + set->setDefaultCompiler(sourceLanguage, compiler); +} + +/* static */void DownstreamCompilerUtil::updateDefaults(DownstreamCompilerSet* set) +{ + for (Index i = 0; i < Index(SLANG_SOURCE_LANGUAGE_COUNT_OF); ++i) + { + updateDefault(set, SlangSourceLanguage(i)); + } +} + +static SlangResult _locateDXCCompilers(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set) +{ + // First try dxil, so it's loaded from the same path if it's there + ComPtr dxil; + DefaultSharedLibraryLoader::load(loader, path, "dxil", dxil.writeRef()); + + ComPtr sharedLibrary; + if (SLANG_SUCCEEDED(DefaultSharedLibraryLoader::load(loader, path, "dxcompiler", sharedLibrary.writeRef()))) + { + // Can we determine the version? + DownstreamCompiler::Desc desc(SLANG_PASS_THROUGH_DXC); + RefPtr compiler(new SharedLibraryDownstreamCompiler(desc, sharedLibrary)); + + set->addCompiler(compiler); + } + return SLANG_OK; +} + +static SlangResult _locateFXCCompilers(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set) +{ + ComPtr sharedLibrary; + if (SLANG_SUCCEEDED(DefaultSharedLibraryLoader::load(loader, path, "d3dcompiler_47", sharedLibrary.writeRef()))) + { + // Can we determine the version? + DownstreamCompiler::Desc desc(SLANG_PASS_THROUGH_FXC); + RefPtr compiler(new SharedLibraryDownstreamCompiler(desc, sharedLibrary)); + set->addCompiler(compiler); + } + return SLANG_OK; +} + +static SlangResult _locateGlslangCompilers(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set) +{ +#if SLANG_UNIX_FAMILY + // On unix systems we need to ensure pthread is loaded first. + ComPtr pthreadLibrary; + DefaultSharedLibraryLoader::load(loader, path, "pthread", pthreadLibrary.writeRef()); +#endif + ComPtr sharedLibrary; + if (SLANG_SUCCEEDED(DefaultSharedLibraryLoader::load(loader, path, "slang-glslang", sharedLibrary.writeRef()))) + { + // Can we determine the version? + DownstreamCompiler::Desc desc(SLANG_PASS_THROUGH_GLSLANG); + RefPtr compiler(new SharedLibraryDownstreamCompiler(desc, sharedLibrary)); + set->addCompiler(compiler); + } + return SLANG_OK; +} + +/* static */void DownstreamCompilerUtil::setDefaultLocators(DownstreamCompilerLocatorFunc outFuncs[int(SLANG_PASS_THROUGH_COUNT_OF)]) +{ + outFuncs[int(SLANG_PASS_THROUGH_VISUAL_STUDIO)] = &VisualStudioCompilerUtil::locateCompilers; + outFuncs[int(SLANG_PASS_THROUGH_CLANG)] = &GCCDownstreamCompilerUtil::locateClangCompilers; + outFuncs[int(SLANG_PASS_THROUGH_GCC)] = &GCCDownstreamCompilerUtil::locateGCCCompilers; + outFuncs[int(SLANG_PASS_THROUGH_NVRTC)] = &NVRTCDownstreamCompilerUtil::locateCompilers; + outFuncs[int(SLANG_PASS_THROUGH_DXC)] = &_locateDXCCompilers; + outFuncs[int(SLANG_PASS_THROUGH_FXC)] = &_locateFXCCompilers; + outFuncs[int(SLANG_PASS_THROUGH_GLSLANG)] = &_locateGlslangCompilers; +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DownstreamCompilerSet !!!!!!!!!!!!!!!!!!!!!!*/ + +void DownstreamCompilerSet::getCompilerDescs(List& outCompilerDescs) const +{ + outCompilerDescs.clear(); + for (DownstreamCompiler* compiler : m_compilers) + { + outCompilerDescs.add(compiler->getDesc()); + } +} + +Index DownstreamCompilerSet::_findIndex(const DownstreamCompiler::Desc& desc) const +{ + const Index count = m_compilers.getCount(); + for (Index i = 0; i < count; ++i) + { + if (m_compilers[i]->getDesc() == desc) + { + return i; + } + } + return -1; +} + +DownstreamCompiler* DownstreamCompilerSet::getCompiler(const DownstreamCompiler::Desc& compilerDesc) const +{ + const Index index = _findIndex(compilerDesc); + return index >= 0 ? m_compilers[index] : nullptr; +} + +void DownstreamCompilerSet::getCompilers(List& outCompilers) const +{ + outCompilers.clear(); + outCompilers.addRange((DownstreamCompiler*const*)m_compilers.begin(), m_compilers.getCount()); +} + +bool DownstreamCompilerSet::hasCompiler(SlangPassThrough compilerType) const +{ + for (DownstreamCompiler* compiler : m_compilers) + { + const auto& desc = compiler->getDesc(); + if (desc.type == compilerType) + { + return true; + } + } + return false; +} + + +void DownstreamCompilerSet::remove(SlangPassThrough compilerType) +{ + for (Index i = 0; i < m_compilers.getCount(); ++i) + { + DownstreamCompiler* compiler = m_compilers[i]; + if (compiler->getDesc().type == compilerType) + { + m_compilers.fastRemoveAt(i); + i--; + } + } +} + +void DownstreamCompilerSet::addCompiler(DownstreamCompiler* compiler) +{ + const Index index = _findIndex(compiler->getDesc()); + if (index >= 0) + { + m_compilers[index] = compiler; + } + else + { + m_compilers.add(compiler); + } +} + +} diff --git a/source/compiler-core/slang-downstream-compiler.h b/source/compiler-core/slang-downstream-compiler.h new file mode 100644 index 000000000..35acc820e --- /dev/null +++ b/source/compiler-core/slang-downstream-compiler.h @@ -0,0 +1,501 @@ +#ifndef SLANG_DOWNSTREAM_COMPILER_H +#define SLANG_DOWNSTREAM_COMPILER_H + +#include "../core/slang-common.h" +#include "../core/slang-string.h" + +#include "../core/slang-process-util.h" + +#include "../core/slang-platform.h" +#include "../core/slang-semantic-version.h" + +#include "../core/slang-io.h" + +#include "../../slang-com-ptr.h" + +namespace Slang +{ + +struct DownstreamDiagnostic +{ + typedef DownstreamDiagnostic ThisType; + + enum class Severity + { + Unknown, + Info, + Warning, + Error, + CountOf, + }; + enum class Stage + { + Compile, + Link, + }; + + void reset() + { + severity = Severity::Unknown; + stage = Stage::Compile; + fileLine = 0; + } + + bool operator==(const ThisType& rhs) const + { + return severity == rhs.severity && + stage == rhs.stage && + text == rhs.text && + code == rhs.code && + filePath == rhs.filePath && + fileLine == rhs.fileLine; + } + bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } + + static UnownedStringSlice getSeverityText(Severity severity); + + Severity severity; ///< The severity of error + Stage stage; ///< The stage the error came from + String text; ///< The text of the error + String code; ///< The compiler specific error code + String filePath; ///< The path the error originated from + Int fileLine; ///< The line number the error came from +}; + +struct DownstreamDiagnostics +{ + typedef DownstreamDiagnostic Diagnostic; + + /// Reset to an initial empty state + void reset() { diagnostics.clear(); rawDiagnostics = String(); result = SLANG_OK; } + + /// Get the number of diagnostics by severity + Index getCountBySeverity(Diagnostic::Severity severity) const; + /// True if there are any diagnostics of severity + bool has(Diagnostic::Severity severity) const { return getCountBySeverity(severity) > 0; } + + /// Stores in outCounts, the amount of diagnostics for the stage of each severity + Int countByStage(Diagnostic::Stage stage, Index outCounts[Int(Diagnostic::Severity::CountOf)]) const; + + /// Append a summary to out + void appendSummary(StringBuilder& out) const; + /// Appends a summary that just identifies if there is an error of a type (not a count) + void appendSimplifiedSummary(StringBuilder& out) const; + + /// Remove all diagnostics of the type + void removeBySeverity(Diagnostic::Severity severity); + + String rawDiagnostics; + + SlangResult result; + List diagnostics; +}; + +class DownstreamCompileResult : public RefObject +{ +public: + + virtual SlangResult getHostCallableSharedLibrary(ComPtr& outLibrary) = 0; + virtual SlangResult getBinary(ComPtr& outBlob) = 0; + + const DownstreamDiagnostics& getDiagnostics() const { return m_diagnostics; } + + /// Ctor + DownstreamCompileResult(const DownstreamDiagnostics& diagnostics): + m_diagnostics(diagnostics) + {} + +protected: + DownstreamDiagnostics m_diagnostics; +}; + + +class BlobDownstreamCompileResult : public DownstreamCompileResult +{ +public: + typedef DownstreamCompileResult Super; + + virtual SlangResult getHostCallableSharedLibrary(ComPtr& outLibrary) SLANG_OVERRIDE { SLANG_UNUSED(outLibrary); return SLANG_FAIL; } + virtual SlangResult getBinary(ComPtr& outBlob) SLANG_OVERRIDE { outBlob = m_blob; return m_blob ? SLANG_OK : SLANG_FAIL; } + + BlobDownstreamCompileResult(const DownstreamDiagnostics& diags, ISlangBlob* blob): + Super(diags), + m_blob(blob) + { + + } +protected: + ComPtr m_blob; +}; + +class DownstreamCompiler: public RefObject +{ +public: + typedef RefObject Super; + + typedef DownstreamCompileResult CompileResult; + + typedef uint32_t SourceLanguageFlags; + struct SourceLanguageFlag + { + enum Enum : SourceLanguageFlags + { + Unknown = SourceLanguageFlags(1) << SLANG_SOURCE_LANGUAGE_UNKNOWN, + Slang = SourceLanguageFlags(1) << SLANG_SOURCE_LANGUAGE_SLANG, + HLSL = SourceLanguageFlags(1) << SLANG_SOURCE_LANGUAGE_HLSL, + GLSL = SourceLanguageFlags(1) << SLANG_SOURCE_LANGUAGE_GLSL, + C = SourceLanguageFlags(1) << SLANG_SOURCE_LANGUAGE_C, + CPP = SourceLanguageFlags(1) << SLANG_SOURCE_LANGUAGE_CPP, + CUDA = SourceLanguageFlags(1) << SLANG_SOURCE_LANGUAGE_CUDA, + }; + }; + + struct Info + { + Info():sourceLanguageFlags(0) {} + + Info(SourceLanguageFlags inSourceLanguageFlags): + sourceLanguageFlags(inSourceLanguageFlags) + {} + SourceLanguageFlags sourceLanguageFlags; + }; + struct Infos + { + Info infos[int(SLANG_PASS_THROUGH_COUNT_OF)]; + }; + + struct Desc + { + typedef Desc ThisType; + + HashCode getHashCode() const { return combineHash(HashCode(type), combineHash(HashCode(majorVersion), HashCode(minorVersion))); } + bool operator==(const ThisType& rhs) const { return type == rhs.type && majorVersion == rhs.majorVersion && minorVersion == rhs.minorVersion; } + bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } + + /// Get the version as a value + Int getVersionValue() const { return majorVersion * 100 + minorVersion; } + + void appendAsText(StringBuilder& out) const; + /// true if has a version set + bool hasVersion() const { return majorVersion || minorVersion; } + + /// Ctor + explicit Desc(SlangPassThrough inType = SLANG_PASS_THROUGH_NONE, Int inMajorVersion = 0, Int inMinorVersion = 0):type(inType), majorVersion(inMajorVersion), minorVersion(inMinorVersion) {} + + SlangPassThrough type; ///< The type of the compiler + Int majorVersion; ///< Major version (interpretation is type specific) + Int minorVersion; ///< Minor version + }; + + enum class OptimizationLevel + { + None, ///< Don't optimize at all. + Default, ///< Default optimization level: balance code quality and compilation time. + High, ///< Optimize aggressively. + Maximal, ///< Include optimizations that may take a very long time, or may involve severe space-vs-speed tradeoffs + }; + + enum class DebugInfoType + { + None, ///< Don't emit debug information at all. + Minimal, ///< Emit as little debug information as possible, while still supporting stack traces. + Standard, ///< Emit whatever is the standard level of debug information for each target. + Maximal, ///< Emit as much debug information as possible for each target. + }; + enum class FloatingPointMode + { + Default, + Fast, + Precise, + }; + + enum TargetType + { + Executable, ///< Produce an executable + SharedLibrary, ///< Produce a shared library object/dll + Object, ///< Produce an object file + }; + + enum PipelineType + { + Unknown, + Compute, + Rasterization, + RayTracing, + }; + + struct Define + { + String nameWithSig; ///< If macro takes parameters include in brackets + String value; + }; + + struct CapabilityVersion + { + enum class Kind + { + CUDASM, ///< What the version is for + }; + Kind kind; + SemanticVersion version; + }; + + struct CompileOptions + { + typedef uint32_t Flags; + struct Flag + { + enum Enum : Flags + { + EnableExceptionHandling = 0x01, + Verbose = 0x02, + EnableSecurityChecks = 0x04, + }; + }; + + OptimizationLevel optimizationLevel = OptimizationLevel::Default; + DebugInfoType debugInfoType = DebugInfoType::Standard; + TargetType targetType = TargetType::Executable; + SlangSourceLanguage sourceLanguage = SLANG_SOURCE_LANGUAGE_CPP; + FloatingPointMode floatingPointMode = FloatingPointMode::Default; + PipelineType pipelineType = PipelineType::Unknown; + + Flags flags = Flag::EnableExceptionHandling; + + PlatformKind platform = PlatformKind::Unknown; + + /// The path/name of the output module. Should not have the extension, as that will be added for each of the target types. + /// If not set a module path will be internally generated internally on a command line based compiler + String modulePath; + + List defines; + + /// The contents of the source to compile. This can be empty is sourceFiles is set. + /// If the compiler is a commandLine file this source will be written to a temporary file. + String sourceContents; + /// 'Path' that the contents originated from. NOTE! This is for reporting only and doesn't have to exist on file system + String sourceContentsPath; + + /// The names/paths of source to compile. This can be empty if sourceContents is set. + List sourceFiles; + + List includePaths; + List libraryPaths; + + List requiredCapabilityVersions; + }; + + typedef uint32_t ProductFlags; + struct ProductFlag + { + enum Enum : ProductFlags + { + Debug = 0x1, ///< Used by debugger during execution + Execution = 0x2, ///< Required for execution + Compile = 0x4, ///< A product *required* for compilation + Miscellaneous = 0x8, ///< Anything else + }; + enum Mask : ProductFlags + { + All = 0xf, ///< All the flags + }; + }; + + enum class Product + { + DebugRun, + Run, + CompileTemporary, + All, + }; + + /// Get the desc of this compiler + const Desc& getDesc() const { return m_desc; } + /// Compile using the specified options. The result is in resOut + virtual SlangResult compile(const CompileOptions& options, RefPtr& outResult) = 0; + /// Some downstream compilers are backed by a shared library. This allows access to the shared library to access internal functions. + virtual ISlangSharedLibrary* getSharedLibrary() { return nullptr; } + + /// Get info for a compiler type + static const Info& getInfo(SlangPassThrough compiler) { return s_infos.infos[int(compiler)]; } + /// True if this compiler can compile the specified language + static bool canCompile(SlangPassThrough compiler, SlangSourceLanguage sourceLanguage); + + /// Given a source language return as the equivalent compile target + static SlangCompileTarget getCompileTarget(SlangSourceLanguage sourceLanguage); + +protected: + static Infos s_infos; + + DownstreamCompiler(const Desc& desc) : + m_desc(desc) + {} + DownstreamCompiler() {} + + Desc m_desc; +}; + +class CommandLineDownstreamCompileResult : public DownstreamCompileResult +{ +public: + typedef DownstreamCompileResult Super; + + virtual SlangResult getHostCallableSharedLibrary(ComPtr& outLibrary) SLANG_OVERRIDE; + virtual SlangResult getBinary(ComPtr& outBlob) SLANG_OVERRIDE; + + CommandLineDownstreamCompileResult(const DownstreamDiagnostics& diagnostics, const String& moduleFilePath, TemporaryFileSet* temporaryFileSet) : + Super(diagnostics), + m_moduleFilePath(moduleFilePath), + m_temporaryFiles(temporaryFileSet) + { + } + + RefPtr m_temporaryFiles; + +protected: + + String m_moduleFilePath; + DownstreamCompiler::CompileOptions m_options; + ComPtr m_binaryBlob; + /// Cache of the shared library if appropriate + ComPtr m_hostCallableSharedLibrary; +}; + +class CommandLineDownstreamCompiler : public DownstreamCompiler +{ +public: + typedef DownstreamCompiler Super; + + // DownstreamCompiler + virtual SlangResult compile(const CompileOptions& options, RefPtr& outResult) SLANG_OVERRIDE; + + // Functions to be implemented for a specific CommandLine + + /// Given the compilation options and the module name, determines the actual file name used for output + virtual SlangResult calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath) = 0; + /// Given options determines the paths to products produced (including the 'moduleFilePath'). + /// Note that does *not* guarentee all products were or should be produced. Just aims to include all that could + /// be produced, such that can be removed on completion. + virtual SlangResult calcCompileProducts(const CompileOptions& options, ProductFlags flags, List& outPaths) = 0; + + virtual SlangResult calcArgs(const CompileOptions& options, CommandLine& cmdLine) = 0; + virtual SlangResult parseOutput(const ExecuteResult& exeResult, DownstreamDiagnostics& output) = 0; + + CommandLineDownstreamCompiler(const Desc& desc, const String& exeName) : + Super(desc) + { + m_cmdLine.setExecutableFilename(exeName); + } + + CommandLineDownstreamCompiler(const Desc& desc, const CommandLine& cmdLine) : + Super(desc), + m_cmdLine(cmdLine) + {} + + CommandLineDownstreamCompiler(const Desc& desc):Super(desc) {} + + CommandLine m_cmdLine; +}; + +class SharedLibraryDownstreamCompiler: public DownstreamCompiler +{ +public: + typedef DownstreamCompiler Super; + + // DownstreamCompiler + virtual SlangResult compile(const CompileOptions& options, RefPtr& outResult) SLANG_OVERRIDE { SLANG_UNUSED(options); SLANG_UNUSED(outResult); return SLANG_E_NOT_IMPLEMENTED; } + virtual ISlangSharedLibrary* getSharedLibrary() SLANG_OVERRIDE { return m_library; } + + SharedLibraryDownstreamCompiler(const Desc& desc, ISlangSharedLibrary* library): + Super(desc), + m_library(library) + { + } +protected: + ComPtr m_library; +}; + +class DownstreamCompilerSet : public RefObject +{ +public: + typedef RefObject Super; + + /// Find all the available compilers + void getCompilerDescs(List& outCompilerDescs) const; + /// Returns list of all compilers + void getCompilers(List& outCompilers) const; + + /// Get a compiler + DownstreamCompiler* getCompiler(const DownstreamCompiler::Desc& compilerDesc) const; + + /// Will replace if there is one with same desc + void addCompiler(DownstreamCompiler* compiler); + + /// Get a default compiler + DownstreamCompiler* getDefaultCompiler(SlangSourceLanguage sourceLanguage) const { return m_defaultCompilers[int(sourceLanguage)]; } + /// Set the default compiler + void setDefaultCompiler(SlangSourceLanguage sourceLanguage, DownstreamCompiler* compiler) { m_defaultCompilers[int(sourceLanguage)] = compiler; } + + /// True if has a compiler of the specified type + bool hasCompiler(SlangPassThrough compilerType) const; + + void remove(SlangPassThrough compilerType); + + void clear() { m_compilers.clear(); } + +protected: + + Index _findIndex(const DownstreamCompiler::Desc& desc) const; + + RefPtr m_defaultCompilers[int(SLANG_SOURCE_LANGUAGE_COUNT_OF)]; + // This could be a dictionary/map - but doing a linear search is going to be fine and it makes + // somethings easier. + List> m_compilers; +}; + +typedef SlangResult (*DownstreamCompilerLocatorFunc)(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set); + +/* Only purpose of having base-class here is to make all the DownstreamCompiler types available directly in derived Utils */ +struct DownstreamCompilerBaseUtil +{ + typedef DownstreamCompiler::CompileOptions CompileOptions; + typedef DownstreamCompiler::OptimizationLevel OptimizationLevel; + typedef DownstreamCompiler::TargetType TargetType; + typedef DownstreamCompiler::DebugInfoType DebugInfoType; + + typedef DownstreamDiagnostics::Diagnostic Diagnostic; + + typedef DownstreamCompiler::FloatingPointMode FloatingPointMode; + typedef DownstreamCompiler::ProductFlag ProductFlag; + typedef DownstreamCompiler::ProductFlags ProductFlags; +}; + +struct DownstreamCompilerUtil: public DownstreamCompilerBaseUtil +{ + enum class MatchType + { + MinGreaterEqual, + MinAbsolute, + Newest, + }; + + /// Find a compiler + static DownstreamCompiler* findCompiler(const DownstreamCompilerSet* set, MatchType matchType, const DownstreamCompiler::Desc& desc); + static DownstreamCompiler* findCompiler(const List& compilers, MatchType matchType, const DownstreamCompiler::Desc& desc); + + /// Find the compiler closest to the desc + static DownstreamCompiler* findClosestCompiler(const List& compilers, const DownstreamCompiler::Desc& desc); + static DownstreamCompiler* findClosestCompiler(const DownstreamCompilerSet* set, const DownstreamCompiler::Desc& desc); + + /// Get the information on the compiler used to compile this source + static const DownstreamCompiler::Desc& getCompiledWithDesc(); + + static void updateDefault(DownstreamCompilerSet* set, SlangSourceLanguage sourceLanguage); + static void updateDefaults(DownstreamCompilerSet* set); + + static void setDefaultLocators(DownstreamCompilerLocatorFunc outFuncs[int(SLANG_PASS_THROUGH_COUNT_OF)]); +}; + +} + +#endif diff --git a/source/compiler-core/slang-gcc-compiler-util.cpp b/source/compiler-core/slang-gcc-compiler-util.cpp new file mode 100644 index 000000000..a51d5735f --- /dev/null +++ b/source/compiler-core/slang-gcc-compiler-util.cpp @@ -0,0 +1,645 @@ +// slang-gcc-compiler-util.cpp +#include "slang-gcc-compiler-util.h" + +#include "../core/slang-common.h" +#include "../../slang-com-helper.h" +#include "../core/slang-string-util.h" + +#include "../core/slang-io.h" +#include "../core/slang-shared-library.h" + +namespace Slang +{ + +/* static */SlangResult GCCDownstreamCompilerUtil::parseVersion(const UnownedStringSlice& text, const UnownedStringSlice& prefix, DownstreamCompiler::Desc& outDesc) +{ + List lines; + StringUtil::calcLines(text, lines); + + for (auto line : lines) + { + if (line.startsWith(prefix)) + { + const UnownedStringSlice remainingSlice = UnownedStringSlice(line.begin() + prefix.getLength(), line.end()).trim(); + const Index versionEndIndex = remainingSlice.indexOf(' '); + if (versionEndIndex < 0) + { + return SLANG_FAIL; + } + + const UnownedStringSlice versionSlice(remainingSlice.begin(), remainingSlice.begin() + versionEndIndex); + + // Version is in format 0.0.0 + List split; + StringUtil::split(versionSlice, '.', split); + List digits; + + for (auto v : split) + { + Int version; + SLANG_RETURN_ON_FAIL(StringUtil::parseInt(v, version)); + digits.add(version); + } + + if (digits.getCount() < 2) + { + return SLANG_FAIL; + } + + outDesc.majorVersion = digits[0]; + outDesc.minorVersion = digits[1]; + return SLANG_OK; + } + } + + return SLANG_FAIL; +} + +SlangResult GCCDownstreamCompilerUtil::calcVersion(const String& exeName, DownstreamCompiler::Desc& outDesc) +{ + CommandLine cmdLine; + cmdLine.setExecutableFilename(exeName); + cmdLine.addArg("-v"); + + ExecuteResult exeRes; + SLANG_RETURN_ON_FAIL(ProcessUtil::execute(cmdLine, exeRes)); + + const UnownedStringSlice prefixes[] = + { + UnownedStringSlice::fromLiteral("clang version"), + UnownedStringSlice::fromLiteral("gcc version"), + UnownedStringSlice::fromLiteral("Apple LLVM version"), + }; + const SlangPassThrough types[] = + { + SLANG_PASS_THROUGH_CLANG, + SLANG_PASS_THROUGH_GCC, + SLANG_PASS_THROUGH_CLANG, + }; + + SLANG_COMPILE_TIME_ASSERT(SLANG_COUNT_OF(prefixes) == SLANG_COUNT_OF(types)); + + for (Index i = 0; i < SLANG_COUNT_OF(prefixes); ++i) + { + // Set the type + outDesc.type = types[i]; + // Extract the version + if (SLANG_SUCCEEDED(parseVersion(exeRes.standardError.getUnownedSlice(), prefixes[i], outDesc))) + { + return SLANG_OK; + } + } + return SLANG_FAIL; +} + +static SlangResult _parseSeverity(const UnownedStringSlice& in, DownstreamDiagnostic::Severity& outSeverity) +{ + typedef DownstreamDiagnostic::Severity Severity; + + if (in == "error" || in == "fatal error") + { + outSeverity = Severity::Error; + } + else if (in == "warning") + { + outSeverity = Severity::Warning; + } + else if (in == "info" || in == "note") + { + outSeverity = Severity::Info; + } + else + { + return SLANG_FAIL; + } + return SLANG_OK; +} + +namespace { // anonymous + +enum class LineParseResult +{ + Single, ///< It's a single line + Start, ///< Line was the start of a message + Continuation, ///< Not totally clear, add to previous line if nothing else hit + Ignore, ///< Ignore the line +}; + +} // anonymous + +static SlangResult _parseGCCFamilyLine(const UnownedStringSlice& line, LineParseResult& outLineParseResult, DownstreamDiagnostic& outDiagnostic) +{ + typedef DownstreamDiagnostic Diagnostic; + typedef Diagnostic::Severity Severity; + + // Set to default case + outLineParseResult = LineParseResult::Ignore; + + /* example error output from different scenarios */ + + /* + tests/cpp-compiler/c-compile-error.c: In function 'int main(int, char**)': + tests/cpp-compiler/c-compile-error.c:8:13: error: 'b' was not declared in this scope + int a = b + c; + ^ + tests/cpp-compiler/c-compile-error.c:8:17: error: 'c' was not declared in this scope + int a = b + c; + ^ + */ + + /* /tmp/ccS0JCWe.o:c-compile-link-error.c:(.rdata$.refptr.thing[.refptr.thing]+0x0): undefined reference to `thing' + collect2: error: ld returned 1 exit status*/ + + /* + clang: warning: treating 'c' input as 'c++' when in C++ mode, this behavior is deprecated [-Wdeprecated] + Undefined symbols for architecture x86_64: + "_thing", referenced from: + _main in c-compile-link-error-a83ace.o + ld: symbol(s) not found for architecture x86_64 + clang: error: linker command failed with exit code 1 (use -v to see invocation) */ + + /* /tmp/c-compile-link-error-ccf151.o: In function `main': + c-compile-link-error.c:(.text+0x19): undefined reference to `thing' + clang: error: linker command failed with exit code 1 (use -v to see invocation) + */ + + /* /tmp/c-compile-link-error-301c8c.o: In function `main': + /home/travis/build/shader-slang/slang/tests/cpp-compiler/c-compile-link-error.c:10: undefined reference to `thing' + clang-7: error: linker command failed with exit code 1 (use -v to see invocation)*/ + + /* /path/slang-cpp-prelude.h:4:10: fatal error: ../slang.h: No such file or directory + #include "../slang.h" + ^~~~~~~~~~~~ + compilation terminated.*/ + + outDiagnostic.stage = Diagnostic::Stage::Compile; + + List split; + StringUtil::split(line, ':', split); + + // On windows we can have paths that are a: etc... if we detect this we can combine 0 - 1 to be 1. + if (split.getCount() > 1 && split[0].getLength() == 1) + { + const char c = split[0][0]; + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) + { + // We'll assume it's a path + UnownedStringSlice path(split[0].begin(), split[1].end()); + split.removeAt(0); + split[0] = path; + } + } + + if (split.getCount() == 2) + { + const auto split0 = split[0].trim(); + if (split0 == UnownedStringSlice::fromLiteral("ld")) + { + // We'll ignore for now + outDiagnostic.stage = Diagnostic::Stage::Link; + outDiagnostic.severity = Severity::Info; + outDiagnostic.text = split[1].trim(); + outLineParseResult = LineParseResult::Start; + return SLANG_OK; + } + + if (SLANG_SUCCEEDED(_parseSeverity(split0, outDiagnostic.severity))) + { + // Command line errors can be just contain 'error:' etc. Can be seen on apple/clang + outDiagnostic.stage = Diagnostic::Stage::Compile; + outDiagnostic.text = split[1].trim(); + outLineParseResult = LineParseResult::Single; + return SLANG_OK; + } + + outLineParseResult = LineParseResult::Ignore; + return SLANG_OK; + } + else if (split.getCount() == 3) + { + const auto split0 = split[0].trim(); + const auto split1 = split[1].trim(); + const auto text = split[2].trim(); + + // Check for special handling for clang (Can be Clang or clang apparently) + if (split0.startsWith(UnownedStringSlice::fromLiteral("clang")) || + split0.startsWith(UnownedStringSlice::fromLiteral("Clang")) ) + { + // Extract the type + SLANG_RETURN_ON_FAIL(_parseSeverity(split[1].trim(), outDiagnostic.severity)); + + if (text.startsWith("linker command failed")) + { + outDiagnostic.stage = Diagnostic::Stage::Link; + } + + outDiagnostic.text = text; + outLineParseResult = LineParseResult::Start; + return SLANG_OK; + } + else if (split1.startsWith("(.text")) + { + // This is a little weak... but looks like it's a link error + outDiagnostic.filePath = split[0]; + outDiagnostic.severity = Severity::Error; + outDiagnostic.stage = Diagnostic::Stage::Link; + outDiagnostic.text = text; + outLineParseResult = LineParseResult::Single; + return SLANG_OK; + } + else if (text.startsWith("ld returned")) + { + outDiagnostic.stage = DownstreamDiagnostic::Stage::Link; + SLANG_RETURN_ON_FAIL(_parseSeverity(split[1].trim(), outDiagnostic.severity)); + outDiagnostic.text = line; + outLineParseResult = LineParseResult::Single; + return SLANG_OK; + } + else if (text == "") + { + // This is probably a prelude line, we'll just ignore it + outLineParseResult = LineParseResult::Ignore; + return SLANG_OK; + } + } + else if (split.getCount() == 4) + { + // Probably a link error, give the source line + String ext = Path::getPathExt(split[0]); + + // Maybe a bit fragile -> but probably okay for now + if (ext != "o" && ext != "obj") + { + outLineParseResult = LineParseResult::Ignore; + return SLANG_OK; + } + else + { + outDiagnostic.filePath = split[1]; + outDiagnostic.fileLine = 0; + outDiagnostic.severity = Diagnostic::Severity::Error; + outDiagnostic.stage = Diagnostic::Stage::Link; + outDiagnostic.text = split[3]; + + outLineParseResult = LineParseResult::Start; + return SLANG_OK; + } + } + else if (split.getCount() >= 5) + { + // Probably a regular error line + SLANG_RETURN_ON_FAIL(_parseSeverity(split[3].trim(), outDiagnostic.severity)); + + outDiagnostic.filePath = split[0]; + SLANG_RETURN_ON_FAIL(StringUtil::parseInt(split[1], outDiagnostic.fileLine)); + + // Everything from 4 to the end is the error + outDiagnostic.text = UnownedStringSlice(split[4].begin(), split.getLast().end()); + + outLineParseResult = LineParseResult::Start; + return SLANG_OK; + } + + // Assume it's a continuation + outLineParseResult = LineParseResult::Continuation; + return SLANG_OK; +} + +/* static */SlangResult GCCDownstreamCompilerUtil::parseOutput(const ExecuteResult& exeRes, DownstreamDiagnostics& outOutput) +{ + LineParseResult prevLineResult = LineParseResult::Ignore; + + outOutput.reset(); + outOutput.rawDiagnostics = exeRes.standardError; + + for (auto line : LineParser(exeRes.standardError.getUnownedSlice())) + { + Diagnostic diagnostic; + diagnostic.reset(); + + LineParseResult lineRes; + + SLANG_RETURN_ON_FAIL(_parseGCCFamilyLine(line, lineRes, diagnostic)); + + switch (lineRes) + { + case LineParseResult::Start: + { + // It's start of a new message + outOutput.diagnostics.add(diagnostic); + prevLineResult = LineParseResult::Start; + break; + } + case LineParseResult::Single: + { + // It's a single message, without anything following + outOutput.diagnostics.add(diagnostic); + prevLineResult = LineParseResult::Ignore; + break; + } + case LineParseResult::Continuation: + { + if (prevLineResult == LineParseResult::Start || prevLineResult == LineParseResult::Continuation) + { + if (outOutput.diagnostics.getCount() > 0) + { + // We are now in a continuation, add to the last + auto& text = outOutput.diagnostics.getLast().text; + text.append("\n"); + text.append(line); + } + prevLineResult = LineParseResult::Continuation; + } + break; + } + case LineParseResult::Ignore: + { + prevLineResult = lineRes; + break; + } + default: return SLANG_FAIL; + } + } + + if (outOutput.has(Diagnostic::Severity::Error) || exeRes.resultCode != 0) + { + outOutput.result = SLANG_FAIL; + } + + return SLANG_OK; +} + +/* static */ SlangResult GCCDownstreamCompilerUtil::calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath) +{ + SLANG_ASSERT(options.modulePath.getLength()); + + outPath.Clear(); + + switch (options.targetType) + { + case TargetType::SharedLibrary: + { + outPath << SharedLibrary::calcPlatformPath(options.modulePath.getUnownedSlice()); + return SLANG_OK; + } + case TargetType::Executable: + { + outPath << options.modulePath; + outPath << ProcessUtil::getExecutableSuffix(); + return SLANG_OK; + } + case TargetType::Object: + { +#if __CYGWIN__ + outPath << options.modulePath << ".obj"; +#else + // Will be .o for typical gcc targets + outPath << options.modulePath << ".o"; +#endif + return SLANG_OK; + } + } + + return SLANG_FAIL; +} + +/* static */SlangResult GCCDownstreamCompilerUtil::calcCompileProducts(const CompileOptions& options, ProductFlags flags, List& outPaths) +{ + SLANG_ASSERT(options.modulePath.getLength()); + + outPaths.clear(); + + if (flags & ProductFlag::Execution) + { + StringBuilder builder; + SLANG_RETURN_ON_FAIL(calcModuleFilePath(options, builder)); + outPaths.add(builder); + } + + return SLANG_OK; +} + +/* static */SlangResult GCCDownstreamCompilerUtil::calcArgs(const CompileOptions& options, CommandLine& cmdLine) +{ + SLANG_ASSERT(options.sourceContents.getLength() == 0); + SLANG_ASSERT(options.modulePath.getLength()); + + PlatformKind platformKind = (options.platform == PlatformKind::Unknown) ? PlatformUtil::getPlatformKind() : options.platform; + + if (options.sourceLanguage == SLANG_SOURCE_LANGUAGE_CPP) + { + cmdLine.addArg("-fvisibility=hidden"); + + // Need C++14 for partial specialization + cmdLine.addArg("-std=c++14"); + } + + // TODO(JS): Here we always set -m32 on x86. It could be argued it is only necessary when creating a shared library + // but if we create an object file, we don't know what to choose because we don't know what final usage is. + // It could also be argued that the platformKind could define the actual desired target - but as it stands + // we only have a target of 'Linux' (as opposed to Win32/64). Really it implies we need an arch enumeration too. + // + // For now we just make X86 binaries try and produce x86 compatible binaries as fixes the immediate problems. +#if SLANG_PROCESSOR_X86 + /* Used to specify the processor more broadly. For a x86 binary we need to make sure we build x86 builds + even when on an x64 system. + -m32 + -m64*/ + cmdLine.addArg("-m32"); +#endif + + switch (options.optimizationLevel) + { + case OptimizationLevel::None: + { + // No optimization + cmdLine.addArg("-O0"); + break; + } + case OptimizationLevel::Default: + { + cmdLine.addArg("-Os"); + break; + } + case OptimizationLevel::High: + { + cmdLine.addArg("-O2"); + break; + } + case OptimizationLevel::Maximal: + { + cmdLine.addArg("-O4"); + break; + } + default: break; + } + + if (options.debugInfoType != DebugInfoType::None) + { + cmdLine.addArg("-g"); + } + + if (options.flags & CompileOptions::Flag::Verbose) + { + cmdLine.addArg("-v"); + } + + switch (options.floatingPointMode) + { + case FloatingPointMode::Default: break; + case FloatingPointMode::Precise: + { + //cmdLine.addArg("-fno-unsafe-math-optimizations"); + break; + } + case FloatingPointMode::Fast: + { + // We could enable SSE with -mfpmath=sse + // But that would only make sense on a x64/x86 type processor and only if that feature is present (it is on all x64) + cmdLine.addArg("-ffast-math"); + break; + } + } + + StringBuilder moduleFilePath; + calcModuleFilePath(options, moduleFilePath); + + cmdLine.addArg("-o"); + cmdLine.addArg(moduleFilePath); + + switch (options.targetType) + { + case TargetType::SharedLibrary: + { + // Shared library + cmdLine.addArg("-shared"); + + if (PlatformUtil::isFamily(PlatformFamily::Unix, platformKind)) + { + // Position independent + cmdLine.addArg("-fPIC"); + } + break; + } + case TargetType::Executable: + { + break; + } + case TargetType::Object: + { + // Don't link, just produce object file + cmdLine.addArg("-c"); + break; + } + default: break; + } + + // Add defines + for (const auto& define : options.defines) + { + StringBuilder builder; + + builder << "-D"; + builder << define.nameWithSig; + if (define.value.getLength()) + { + builder << "=" << define.value; + } + + cmdLine.addArg(builder); + } + + // Add includes + for (const auto& include : options.includePaths) + { + cmdLine.addArg("-I"); + cmdLine.addArg(include); + } + + // Link options + if (0) // && options.targetType != TargetType::Object) + { + //linkOptions << "-Wl,"; + //cmdLine.addArg(linkOptions); + } + + if (options.targetType == TargetType::SharedLibrary) + { + if (!PlatformUtil::isFamily(PlatformFamily::Apple, platformKind)) + { + // On MacOS, this linker option is not supported. That's ok though in + // so far as on MacOS it does report any unfound symbols without the option. + + // Linker flag to report any undefined symbols as a link error + cmdLine.addArg("-Wl,--no-undefined"); + } + } + + // Files to compile + for (const auto& sourceFile : options.sourceFiles) + { + cmdLine.addArg(sourceFile); + } + + for (const auto& libPath : options.libraryPaths) + { + // Note that any escaping of the path is handled in the ProcessUtil:: + cmdLine.addArg("-L"); + cmdLine.addArg(libPath); + cmdLine.addArg("-F"); + cmdLine.addArg(libPath); + } + + if (options.sourceLanguage == SLANG_SOURCE_LANGUAGE_CPP && !PlatformUtil::isFamily(PlatformFamily::Windows, platformKind)) + { + // Make STD libs available + cmdLine.addArg("-lstdc++"); + // Make maths lib available + cmdLine.addArg("-lm"); + } + + return SLANG_OK; +} + +/* static */SlangResult GCCDownstreamCompilerUtil::createCompiler(const String& path, const String& inExeName, RefPtr& outCompiler) +{ + String exeName(inExeName); + if (path.getLength() > 0) + { + exeName = Path::combine(path, inExeName); + } + + DownstreamCompiler::Desc desc; + SLANG_RETURN_ON_FAIL(GCCDownstreamCompilerUtil::calcVersion(exeName, desc)); + + RefPtr compiler(new GCCDownstreamCompiler(desc)); + compiler->m_cmdLine.setExecutableFilename(exeName); + + outCompiler = compiler; + return SLANG_OK; +} + +/* static */SlangResult GCCDownstreamCompilerUtil::locateGCCCompilers(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set) +{ + SLANG_UNUSED(loader); + RefPtr compiler; + if (SLANG_SUCCEEDED(createCompiler(path, "g++", compiler))) + { + set->addCompiler(compiler); + } + return SLANG_OK; +} + +/* static */SlangResult GCCDownstreamCompilerUtil::locateClangCompilers(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set) +{ + SLANG_UNUSED(loader); + + RefPtr compiler; + if (SLANG_SUCCEEDED(createCompiler(path, "clang", compiler))) + { + set->addCompiler(compiler); + } + return SLANG_OK; +} + +} diff --git a/source/compiler-core/slang-gcc-compiler-util.h b/source/compiler-core/slang-gcc-compiler-util.h new file mode 100644 index 000000000..b97144e35 --- /dev/null +++ b/source/compiler-core/slang-gcc-compiler-util.h @@ -0,0 +1,56 @@ +#ifndef SLANG_GCC_COMPILER_UTIL_H +#define SLANG_GCC_COMPILER_UTIL_H + +#include "slang-downstream-compiler.h" + +namespace Slang +{ + +/* Utility for processing input and output of gcc-like compilers, including clang */ +struct GCCDownstreamCompilerUtil : public DownstreamCompilerBaseUtil +{ + /// Extracts version number into desc from text (assumes gcc/clang -v layout with a line with version) + static SlangResult parseVersion(const UnownedStringSlice& text, const UnownedStringSlice& prefix, DownstreamCompiler::Desc& outDesc); + + /// Runs the exeName, and extracts the version info into outDesc + static SlangResult calcVersion(const String& exeName, DownstreamCompiler::Desc& outDesc); + + /// Calculate gcc family compilers (including clang) cmdLine arguments from options + static SlangResult calcArgs(const CompileOptions& options, CommandLine& cmdLine); + + /// Parse ExecuteResult into Output + static SlangResult parseOutput(const ExecuteResult& exeRes, DownstreamDiagnostics& outOutput); + + /// Calculate the output module filename + static SlangResult calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath); + + /// Given options, calculate paths to products produced for a compilation + static SlangResult calcCompileProducts(const CompileOptions& options, ProductFlags flags, List& outPaths); + + /// Given a path and an exe name, detects if compiler is present, and if so adds to compiler set. + static SlangResult createCompiler(const String& path, const String& inExeName, RefPtr& outCompiler); + + static SlangResult locateGCCCompilers(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set); + + static SlangResult locateClangCompilers(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set); + +}; + +class GCCDownstreamCompiler : public CommandLineDownstreamCompiler +{ +public: + typedef CommandLineDownstreamCompiler Super; + typedef GCCDownstreamCompilerUtil Util; + + // CommandLineCPPCompiler impl - just forwards to the Util + virtual SlangResult calcArgs(const CompileOptions& options, CommandLine& cmdLine) SLANG_OVERRIDE { return Util::calcArgs(options, cmdLine); } + virtual SlangResult parseOutput(const ExecuteResult& exeResult, DownstreamDiagnostics& output) SLANG_OVERRIDE { return Util::parseOutput(exeResult, output); } + virtual SlangResult calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath) SLANG_OVERRIDE { return Util::calcModuleFilePath(options, outPath); } + virtual SlangResult calcCompileProducts(const CompileOptions& options, ProductFlags flags, List& outPaths) SLANG_OVERRIDE { return Util::calcCompileProducts(options, flags, outPaths); } + + GCCDownstreamCompiler(const Desc& desc):Super(desc) {} +}; + +} + +#endif diff --git a/source/compiler-core/slang-lexer-diagnostic-defs.h b/source/compiler-core/slang-lexer-diagnostic-defs.h new file mode 100644 index 000000000..666ab057f --- /dev/null +++ b/source/compiler-core/slang-lexer-diagnostic-defs.h @@ -0,0 +1,33 @@ +// The file is meant to be included multiple times, to produce different +// pieces of declaration/definition code related to diagnostic messages +// +// Each diagnostic is declared here with: +// +// DIAGNOSTIC(id, severity, name, messageFormat) +// +// Where `id` is the unique diagnostic ID, `severity` is the default +// severity (from the `Severity` enum), `name` is a name used to refer +// to this diagnostic from code, and `messageFormat` is the default +// (non-localized) message for the diagnostic, with placeholders +// for any arguments. + +#ifndef DIAGNOSTIC +#error Need to #define DIAGNOSTIC(...) before including +#define DIAGNOSTIC(id, severity, name, messageFormat) /* */ +#endif + +// +// 1xxxx - Lexical analysis +// + +DIAGNOSTIC(10000, Error, illegalCharacterPrint, "illegal character '$0'") +DIAGNOSTIC(10000, Error, illegalCharacterHex, "illegal character (0x$0)") +DIAGNOSTIC(10001, Error, illegalCharacterLiteral, "illegal character literal") + +DIAGNOSTIC(10002, Warning, octalLiteral, "'0' prefix indicates octal literal") +DIAGNOSTIC(10003, Error, invalidDigitForBase, "invalid digit for base-$1 literal: '$0'") + +DIAGNOSTIC(10004, Error, endOfFileInLiteral, "end of file in literal") +DIAGNOSTIC(10005, Error, newlineInLiteral, "newline in literal") + +#undef DIAGNOSTIC diff --git a/source/compiler-core/slang-lexer.cpp b/source/compiler-core/slang-lexer.cpp new file mode 100644 index 000000000..83b8e0eec --- /dev/null +++ b/source/compiler-core/slang-lexer.cpp @@ -0,0 +1,1400 @@ +// slang-lexer.cpp +#include "slang-lexer.h" + +// This file implements the lexer/scanner, which is responsible for taking a raw stream of +// input bytes and turning it into semantically useful tokens. +// + +#include "slang-name.h" +#include "slang-source-loc.h" + +#include "slang-core-diagnostics.h" + +namespace Slang +{ + Token TokenReader::getEndOfFileToken() + { + return Token(TokenType::EndOfFile, UnownedStringSlice::fromLiteral(""), SourceLoc()); + } + + const Token* TokenList::begin() const + { + SLANG_ASSERT(m_tokens.getCount()); + return &m_tokens[0]; + } + + const Token* TokenList::end() const + { + SLANG_ASSERT(m_tokens.getCount()); + SLANG_ASSERT(m_tokens[m_tokens.getCount() - 1].type == TokenType::EndOfFile); + return &m_tokens[m_tokens.getCount() - 1]; + } + + TokenSpan::TokenSpan() + : m_begin(nullptr) + , m_end (nullptr) + {} + + TokenReader::TokenReader() + : m_cursor(nullptr) + , m_end (nullptr) + {} + + + Token& TokenReader::peekToken() + { + return m_nextToken; + } + + TokenType TokenReader::peekTokenType() const + { + return m_nextToken.type; + } + + SourceLoc TokenReader::peekLoc() const + { + return m_nextToken.loc; + } + + Token TokenReader::advanceToken() + { + if (!m_cursor) + return getEndOfFileToken(); + + Token token = m_nextToken; + if (m_cursor < m_end) + { + m_cursor++; + m_nextToken = *m_cursor; + } + else + m_nextToken.type = TokenType::EndOfFile; + return token; + } + + // Lexer + + void Lexer::initialize( + SourceView* sourceView, + DiagnosticSink* sink, + NamePool* namePool, + MemoryArena* memoryArena, + OptionFlags optionFlags) + { + m_sourceView = sourceView; + m_sink = sink; + m_namePool = namePool; + m_memoryArena = memoryArena; + + auto content = sourceView->getContent(); + + m_begin = content.begin(); + m_cursor = content.begin(); + m_end = content.end(); + + // Set the start location + m_startLoc = sourceView->getRange().begin; + + m_tokenFlags = TokenFlag::AtStartOfLine | TokenFlag::AfterWhitespace; + m_lexerFlags = 0; + m_optionFlags = optionFlags; + } + + Lexer::~Lexer() + { + } + + enum { kEOF = -1 }; + + // Get the next input byte, without any handling of + // escaped newlines, non-ASCII code points, source locations, etc. + static int _peekRaw(Lexer* lexer) + { + // If we are at the end of the input, return a designated end-of-file value + if(lexer->m_cursor == lexer->m_end) + return kEOF; + + // Otherwise, just look at the next byte + return *lexer->m_cursor; + } + + // Read one input byte without any special handling (similar to `peekRaw`) + static int _advanceRaw(Lexer* lexer) + { + // The logic here is basically the same as for `peekRaw()`, + // escape we advance `cursor` if we aren't at the end. + + if (lexer->m_cursor == lexer->m_end) + return kEOF; + + return *lexer->m_cursor++; + } + + // When the cursor is already at the first byte of an end-of-line sequence, + // consume one or two bytes that compose the sequence. + // + // Basically, a newline is one of: + // + // "\n" + // "\r" + // "\r\n" + // "\n\r" + // + // We always look for the longest match possible. + // + static void _handleNewLineInner(Lexer* lexer, int c) + { + SLANG_ASSERT(c == '\n' || c == '\r'); + + int d = _peekRaw(lexer); + if( (c ^ d) == ('\n' ^ '\r') ) + { + _advanceRaw(lexer); + } + } + + // Look ahead one code point, dealing with complications like + // escaped newlines. + static int _peek(Lexer* lexer) + { + // Look at the next raw byte, and decide what to do + int c = _peekRaw(lexer); + + if(c == '\\') + { + // We might have a backslash-escaped newline. + // Look at the next byte (if any) to see. + // + // Note(tfoley): We are assuming a null-terminated input here, + // so that we can safely look at the next byte without issue. + int d = lexer->m_cursor[1]; + switch (d) + { + case '\r': case '\n': + { + // The newline was escaped, so return the code point after *that* + + int e = lexer->m_cursor[2]; + if ((d ^ e) == ('\r' ^ '\n')) + return lexer->m_cursor[3]; + return e; + } + + default: + break; + } + } + // TODO: handle UTF-8 encoding for non-ASCII code points here + + // Default case is to just hand along the byte we read as an ASCII code point. + return c; + } + + // Get the next code point from the input, and advance the cursor. + static int _advance(Lexer* lexer) + { + // We are going to loop, but only as a way of handling + // escaped line endings. + for (;;) + { + // If we are at the end of the input, then the task is easy. + if (lexer->m_cursor == lexer->m_end) + return kEOF; + + // Look at the next raw byte, and decide what to do + int c = *lexer->m_cursor++; + + if (c == '\\') + { + // We might have a backslash-escaped newline. + // Look at the next byte (if any) to see. + // + // Note(tfoley): We are assuming a null-terminated input here, + // so that we can safely look at the next byte without issue. + int d = *lexer->m_cursor; + switch (d) + { + case '\r': case '\n': + // handle the end-of-line for our source location tracking + lexer->m_cursor++; + _handleNewLineInner(lexer, d); + + lexer->m_tokenFlags |= TokenFlag::ScrubbingNeeded; + + // Now try again, looking at the character after the + // escaped newline. + continue; + + default: + break; + } + } + + // TODO: Need to handle non-ASCII code points. + + // Default case is to return the raw byte we saw. + return c; + } + } + + static void _handleNewLine(Lexer* lexer) + { + int c = _advance(lexer); + _handleNewLineInner(lexer, c); + } + + static void _lexLineComment(Lexer* lexer) + { + for(;;) + { + switch(_peek(lexer)) + { + case '\n': case '\r': case kEOF: + return; + + default: + _advance(lexer); + continue; + } + } + } + + static void _lexBlockComment(Lexer* lexer) + { + for(;;) + { + switch(_peek(lexer)) + { + case kEOF: + // TODO(tfoley) diagnostic! + return; + + case '\n': case '\r': + _handleNewLine(lexer); + continue; + + case '*': + _advance(lexer); + switch( _peek(lexer) ) + { + case '/': + _advance(lexer); + return; + + default: + continue; + } + + default: + _advance(lexer); + continue; + } + } + } + + static void _lexHorizontalSpace(Lexer* lexer) + { + for(;;) + { + switch(_peek(lexer)) + { + case ' ': case '\t': + _advance(lexer); + continue; + + default: + return; + } + } + } + + static void _lexIdentifier(Lexer* lexer) + { + for(;;) + { + int c = _peek(lexer); + if(('a' <= c ) && (c <= 'z') + || ('A' <= c) && (c <= 'Z') + || ('0' <= c) && (c <= '9') + || (c == '_')) + { + _advance(lexer); + continue; + } + + return; + } + } + + static SourceLoc _getSourceLoc(Lexer* lexer) + { + return lexer->m_startLoc + (lexer->m_cursor - lexer->m_begin); + } + + static void _lexDigits(Lexer* lexer, int base) + { + for(;;) + { + int c = _peek(lexer); + + int digitVal = 0; + switch(c) + { + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + digitVal = c - '0'; + break; + + case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': + if(base <= 10) return; + digitVal = 10 + c - 'a'; + break; + + case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': + if(base <= 10) return; + digitVal = 10 + c - 'A'; + break; + + default: + // Not more digits! + return; + } + + if(digitVal >= base) + { + char buffer[] = { (char) c, 0 }; + lexer->m_sink->diagnose(_getSourceLoc(lexer), LexerDiagnostics::invalidDigitForBase, buffer, base); + } + + _advance(lexer); + } + } + + static TokenType _maybeLexNumberSuffix(Lexer* lexer, TokenType tokenType) + { + // Be liberal in what we accept here, so that figuring out + // the semantics of a numeric suffix is left up to the parser + // and semantic checking logic. + // + for( ;;) + { + int c = _peek(lexer); + + // Accept any alphanumeric character, plus underscores. + if(('a' <= c ) && (c <= 'z') + || ('A' <= c) && (c <= 'Z') + || ('0' <= c) && (c <= '9') + || (c == '_')) + { + _advance(lexer); + continue; + } + + // Stop at the first character that isn't + // alphanumeric. + return tokenType; + } + } + + static bool _isNumberExponent(int c, int base) + { + switch( c ) + { + default: + return false; + + case 'e': case 'E': + if(base != 10) return false; + break; + + case 'p': case 'P': + if(base != 16) return false; + break; + } + + return true; + } + + static bool _maybeLexNumberExponent(Lexer* lexer, int base) + { + if(!_isNumberExponent(_peek(lexer), base)) + return false; + + // we saw an exponent marker + _advance(lexer); + + // Now start to read the exponent + switch( _peek(lexer) ) + { + case '+': case '-': + _advance(lexer); + break; + } + + // TODO(tfoley): it would be an error to not see digits here... + + _lexDigits(lexer, 10); + + return true; + } + + static TokenType _lexNumberAfterDecimalPoint(Lexer* lexer, int base) + { + _lexDigits(lexer, base); + _maybeLexNumberExponent(lexer, base); + + return _maybeLexNumberSuffix(lexer, TokenType::FloatingPointLiteral); + } + + static TokenType _lexNumber(Lexer* lexer, int base) + { + // TODO(tfoley): Need to consider whehter to allow any kind of digit separator character. + + TokenType tokenType = TokenType::IntegerLiteral; + + // At the start of things, we just concern ourselves with digits + _lexDigits(lexer, base); + + if( _peek(lexer) == '.' ) + { + tokenType = TokenType::FloatingPointLiteral; + + _advance(lexer); + _lexDigits(lexer, base); + } + + if( _maybeLexNumberExponent(lexer, base)) + { + tokenType = TokenType::FloatingPointLiteral; + } + + _maybeLexNumberSuffix(lexer, tokenType); + return tokenType; + } + + static int _maybeReadDigit(char const** ioCursor, int base) + { + auto& cursor = *ioCursor; + + for(;;) + { + int c = *cursor; + switch(c) + { + default: + return -1; + + // TODO: need to decide on digit separator characters + case '_': + cursor++; + continue; + + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + cursor++; + return c - '0'; + + case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': + if(base > 10) + { + cursor++; + return 10 + c - 'a'; + } + return -1; + + case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': + if(base > 10) + { + cursor++; + return 10 + c - 'A'; + } + return -1; + } + } + } + + static int _readOptionalBase(char const** ioCursor) + { + auto& cursor = *ioCursor; + if( *cursor == '0' ) + { + cursor++; + switch(*cursor) + { + case 'x': case 'X': + cursor++; + return 16; + + case 'b': case 'B': + cursor++; + return 2; + + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + return 8; + + default: + return 10; + } + } + + return 10; + } + + + + IntegerLiteralValue getIntegerLiteralValue(Token const& token, UnownedStringSlice* outSuffix) + { + IntegerLiteralValue value = 0; + + const UnownedStringSlice content = token.getContent(); + + char const* cursor = content.begin(); + char const* end = content.end(); + + int base = _readOptionalBase(&cursor); + + for( ;;) + { + int digit = _maybeReadDigit(&cursor, base); + if(digit < 0) + break; + + value = value*base + digit; + } + + if(outSuffix) + { + *outSuffix = UnownedStringSlice(cursor, end); + } + + return value; + } + + FloatingPointLiteralValue getFloatingPointLiteralValue(Token const& token, UnownedStringSlice* outSuffix) + { + FloatingPointLiteralValue value = 0; + + const UnownedStringSlice content = token.getContent(); + + char const* cursor = content.begin(); + char const* end = content.end(); + + int radix = _readOptionalBase(&cursor); + + bool seenDot = false; + FloatingPointLiteralValue divisor = 1; + for( ;;) + { + if(*cursor == '.') + { + cursor++; + seenDot = true; + continue; + } + + int digit = _maybeReadDigit(&cursor, radix); + if(digit < 0) + break; + + value = value*radix + digit; + + if(seenDot) + { + divisor *= radix; + } + } + + // Now read optional exponent + if(_isNumberExponent(*cursor, radix)) + { + cursor++; + + bool exponentIsNegative = false; + switch(*cursor) + { + default: + break; + + case '-': + exponentIsNegative = true; + cursor++; + break; + + case '+': + cursor++; + break; + } + + int exponentRadix = 10; + int exponent = 0; + + for(;;) + { + int digit = _maybeReadDigit(&cursor, exponentRadix); + if(digit < 0) + break; + + exponent = exponent*exponentRadix + digit; + } + + FloatingPointLiteralValue exponentBase = 10; + if(radix == 16) + { + exponentBase = 2; + } + + FloatingPointLiteralValue exponentValue = pow(exponentBase, exponent); + + if( exponentIsNegative ) + { + divisor *= exponentValue; + } + else + { + value *= exponentValue; + } + } + + value /= divisor; + + if(outSuffix) + { + *outSuffix = UnownedStringSlice(cursor, end); + } + + return value; + } + + static void _lexStringLiteralBody(Lexer* lexer, char quote) + { + for(;;) + { + int c = _peek(lexer); + if(c == quote) + { + _advance(lexer); + return; + } + + switch(c) + { + case kEOF: + lexer->m_sink->diagnose(_getSourceLoc(lexer), LexerDiagnostics::endOfFileInLiteral); + return; + + case '\n': case '\r': + lexer->m_sink->diagnose(_getSourceLoc(lexer), LexerDiagnostics::newlineInLiteral); + return; + + case '\\': + // Need to handle various escape sequence cases + _advance(lexer); + switch(_peek(lexer)) + { + case '\'': + case '\"': + case '\\': + case '?': + case 'a': + case 'b': + case 'f': + case 'n': + case 'r': + case 't': + case 'v': + _advance(lexer); + break; + + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': + // octal escape: up to 3 characters + _advance(lexer); + for(int ii = 0; ii < 3; ++ii) + { + int d = _peek(lexer); + if(('0' <= d) && (d <= '7')) + { + _advance(lexer); + continue; + } + else + { + break; + } + } + break; + + case 'x': + // hexadecimal escape: any number of characters + _advance(lexer); + for(;;) + { + int d = _peek(lexer); + if(('0' <= d) && (d <= '9') + || ('a' <= d) && (d <= 'f') + || ('A' <= d) && (d <= 'F')) + { + _advance(lexer); + continue; + } + else + { + break; + } + } + break; + + // TODO: Unicode escape sequences + + } + break; + + default: + _advance(lexer); + continue; + } + } + } + + String getStringLiteralTokenValue(Token const& token) + { + SLANG_ASSERT(token.type == TokenType::StringLiteral + || token.type == TokenType::CharLiteral); + + const UnownedStringSlice content = token.getContent(); + + char const* cursor = content.begin(); + char const* end = content.end(); + SLANG_UNREFERENCED_VARIABLE(end); + + auto quote = *cursor++; + SLANG_ASSERT(quote == '\'' || quote == '"'); + + StringBuilder valueBuilder; + for(;;) + { + SLANG_ASSERT(cursor != end); + + auto c = *cursor++; + + // If we see a closing quote, then we are at the end of the string literal + if(c == quote) + { + SLANG_ASSERT(cursor == end); + return valueBuilder.ProduceString(); + } + + // Characters that don't being escape sequences are easy; + // just append them to the buffer and move on. + if(c != '\\') + { + valueBuilder.Append(c); + continue; + } + + // Now we look at another character to figure out the kind of + // escape sequence we are dealing with: + + char d = *cursor++; + + switch(d) + { + // Simple characters that just needed to be escaped + case '\'': + case '\"': + case '\\': + case '?': + valueBuilder.Append(d); + continue; + + // Traditional escape sequences for special characters + case 'a': valueBuilder.Append('\a'); continue; + case 'b': valueBuilder.Append('\b'); continue; + case 'f': valueBuilder.Append('\f'); continue; + case 'n': valueBuilder.Append('\n'); continue; + case 'r': valueBuilder.Append('\r'); continue; + case 't': valueBuilder.Append('\t'); continue; + case 'v': valueBuilder.Append('\v'); continue; + + // Octal escape: up to 3 characterws + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': + { + cursor--; + int value = 0; + for(int ii = 0; ii < 3; ++ii) + { + d = *cursor; + if(('0' <= d) && (d <= '7')) + { + value = value*8 + (d - '0'); + + cursor++; + continue; + } + else + { + break; + } + } + + // TODO: add support for appending an arbitrary code point? + valueBuilder.Append((char) value); + } + continue; + + // Hexadecimal escape: any number of characters + case 'x': + { + cursor--; + int value = 0; + for(;;) + { + d = *cursor++; + int digitValue = 0; + if(('0' <= d) && (d <= '9')) + { + digitValue = d - '0'; + } + else if( ('a' <= d) && (d <= 'f') ) + { + digitValue = d - 'a'; + } + else if( ('A' <= d) && (d <= 'F') ) + { + digitValue = d - 'A'; + } + else + { + cursor--; + break; + } + + value = value*16 + digitValue; + } + + // TODO: add support for appending an arbitrary code point? + valueBuilder.Append((char) value); + } + continue; + + // TODO: Unicode escape sequences + + } + } + } + + String getFileNameTokenValue(Token const& token) + { + const UnownedStringSlice content = token.getContent(); + + // A file name usually doesn't process escape sequences + // (this is import on Windows, where `\\` is a valid + // path separator character). + + // Just trim off the first and last characters to remove the quotes + // (whether they were `""` or `<>`. + return String(content.begin() + 1, content.end() - 1); + } + + + + static TokenType _lexTokenImpl(Lexer* lexer, LexerFlags effectiveFlags) + { + if(effectiveFlags & kLexerFlag_ExpectDirectiveMessage) + { + for(;;) + { + switch(_peek(lexer)) + { + default: + _advance(lexer); + continue; + + case kEOF: case '\r': case '\n': + break; + } + break; + } + return TokenType::DirectiveMessage; + } + + switch(_peek(lexer)) + { + default: + break; + + case kEOF: + if((effectiveFlags & kLexerFlag_InDirective) != 0) + return TokenType::EndOfDirective; + return TokenType::EndOfFile; + + case '\r': case '\n': + if((effectiveFlags & kLexerFlag_InDirective) != 0) + return TokenType::EndOfDirective; + _handleNewLine(lexer); + return TokenType::NewLine; + + case ' ': case '\t': + _lexHorizontalSpace(lexer); + return TokenType::WhiteSpace; + + case '.': + _advance(lexer); + switch(_peek(lexer)) + { + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + return _lexNumberAfterDecimalPoint(lexer, 10); + + // TODO(tfoley): handle ellipsis (`...`) + + default: + return TokenType::Dot; + } + + case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + return _lexNumber(lexer, 10); + + case '0': + { + auto loc = _getSourceLoc(lexer); + _advance(lexer); + switch(_peek(lexer)) + { + default: + return _maybeLexNumberSuffix(lexer, TokenType::IntegerLiteral); + + case '.': + _advance(lexer); + return _lexNumberAfterDecimalPoint(lexer, 10); + + case 'x': case 'X': + _advance(lexer); + return _lexNumber(lexer, 16); + + case 'b': case 'B': + _advance(lexer); + return _lexNumber(lexer, 2); + + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + lexer->m_sink->diagnose(loc, LexerDiagnostics::octalLiteral); + return _lexNumber(lexer, 8); + } + } + + case 'a': case 'b': case 'c': case 'd': case 'e': + case 'f': case 'g': case 'h': case 'i': case 'j': + case 'k': case 'l': case 'm': case 'n': case 'o': + case 'p': case 'q': case 'r': case 's': case 't': + case 'u': case 'v': case 'w': case 'x': case 'y': + case 'z': + case 'A': case 'B': case 'C': case 'D': case 'E': + case 'F': case 'G': case 'H': case 'I': case 'J': + case 'K': case 'L': case 'M': case 'N': case 'O': + case 'P': case 'Q': case 'R': case 'S': case 'T': + case 'U': case 'V': case 'W': case 'X': case 'Y': + case 'Z': + case '_': + _lexIdentifier(lexer); + return TokenType::Identifier; + + case '\"': + _advance(lexer); + _lexStringLiteralBody(lexer, '\"'); + return TokenType::StringLiteral; + + case '\'': + _advance(lexer); + _lexStringLiteralBody(lexer, '\''); + return TokenType::CharLiteral; + + case '+': + _advance(lexer); + switch(_peek(lexer)) + { + case '+': _advance(lexer); return TokenType::OpInc; + case '=': _advance(lexer); return TokenType::OpAddAssign; + default: + return TokenType::OpAdd; + } + + case '-': + _advance(lexer); + switch(_peek(lexer)) + { + case '-': _advance(lexer); return TokenType::OpDec; + case '=': _advance(lexer); return TokenType::OpSubAssign; + case '>': _advance(lexer); return TokenType::RightArrow; + default: + return TokenType::OpSub; + } + + case '*': + _advance(lexer); + switch(_peek(lexer)) + { + case '=': _advance(lexer); return TokenType::OpMulAssign; + default: + return TokenType::OpMul; + } + + case '/': + _advance(lexer); + switch(_peek(lexer)) + { + case '=': _advance(lexer); return TokenType::OpDivAssign; + case '/': _advance(lexer); _lexLineComment(lexer); return TokenType::LineComment; + case '*': _advance(lexer); _lexBlockComment(lexer); return TokenType::BlockComment; + default: + return TokenType::OpDiv; + } + + case '%': + _advance(lexer); + switch(_peek(lexer)) + { + case '=': _advance(lexer); return TokenType::OpModAssign; + default: + return TokenType::OpMod; + } + + case '|': + _advance(lexer); + switch(_peek(lexer)) + { + case '|': _advance(lexer); return TokenType::OpOr; + case '=': _advance(lexer); return TokenType::OpOrAssign; + default: + return TokenType::OpBitOr; + } + + case '&': + _advance(lexer); + switch(_peek(lexer)) + { + case '&': _advance(lexer); return TokenType::OpAnd; + case '=': _advance(lexer); return TokenType::OpAndAssign; + default: + return TokenType::OpBitAnd; + } + + case '^': + _advance(lexer); + switch(_peek(lexer)) + { + case '=': _advance(lexer); return TokenType::OpXorAssign; + default: + return TokenType::OpBitXor; + } + + case '>': + _advance(lexer); + switch(_peek(lexer)) + { + case '>': + _advance(lexer); + switch(_peek(lexer)) + { + case '=': _advance(lexer); return TokenType::OpShrAssign; + default: return TokenType::OpRsh; + } + case '=': _advance(lexer); return TokenType::OpGeq; + default: + return TokenType::OpGreater; + } + + case '<': + _advance(lexer); + switch(_peek(lexer)) + { + case '<': + _advance(lexer); + switch(_peek(lexer)) + { + case '=': _advance(lexer); return TokenType::OpShlAssign; + default: return TokenType::OpLsh; + } + case '=': _advance(lexer); return TokenType::OpLeq; + default: + return TokenType::OpLess; + } + + case '=': + _advance(lexer); + switch(_peek(lexer)) + { + case '=': _advance(lexer); return TokenType::OpEql; + default: + return TokenType::OpAssign; + } + + case '!': + _advance(lexer); + switch(_peek(lexer)) + { + case '=': _advance(lexer); return TokenType::OpNeq; + default: + return TokenType::OpNot; + } + + case '#': + _advance(lexer); + switch(_peek(lexer)) + { + case '#': _advance(lexer); return TokenType::PoundPound; + default: + return TokenType::Pound; + } + + case '~': _advance(lexer); return TokenType::OpBitNot; + + case ':': + { + _advance(lexer); + if (_peek(lexer) == ':') + { + _advance(lexer); + return TokenType::Scope; + } + return TokenType::Colon; + } + case ';': _advance(lexer); return TokenType::Semicolon; + case ',': _advance(lexer); return TokenType::Comma; + + case '{': _advance(lexer); return TokenType::LBrace; + case '}': _advance(lexer); return TokenType::RBrace; + case '[': _advance(lexer); return TokenType::LBracket; + case ']': _advance(lexer); return TokenType::RBracket; + case '(': _advance(lexer); return TokenType::LParent; + case ')': _advance(lexer); return TokenType::RParent; + + case '?': _advance(lexer); return TokenType::QuestionMark; + case '@': _advance(lexer); return TokenType::At; + case '$': _advance(lexer); return TokenType::Dollar; + + } + + // TODO(tfoley): If we ever wanted to support proper Unicode + // in identifiers, etc., then this would be the right place + // to perform a more expensive dispatch based on the actual + // code point (and not just the first byte). + + { + // If none of the above cases matched, then we have an + // unexpected/invalid character. + + auto loc = _getSourceLoc(lexer); + int c = _advance(lexer); + if(!(effectiveFlags & kLexerFlag_IgnoreInvalid)) + { + auto sink = lexer->m_sink; + if(c >= 0x20 && c <= 0x7E) + { + char buffer[] = { (char) c, 0 }; + sink->diagnose(loc, LexerDiagnostics::illegalCharacterPrint, buffer); + } + else + { + // Fallback: print as hexadecimal + sink->diagnose(loc, LexerDiagnostics::illegalCharacterHex, String((unsigned char)c, 16)); + } + } + + return TokenType::Invalid; + } + } + + Token Lexer::lexToken(LexerFlags extraFlags) + { + auto& flags = m_tokenFlags; + for(;;) + { + Token token; + token.loc = _getSourceLoc(this); + + char const* textBegin = m_cursor; + + auto tokenType = _lexTokenImpl(this, m_lexerFlags | extraFlags); + + // The low-level lexer produces tokens for things we want + // to ignore, such as white space, so we skip them here. + switch(tokenType) + { + case TokenType::Invalid: + flags = 0; + continue; + + case TokenType::NewLine: + flags = TokenFlag::AtStartOfLine | TokenFlag::AfterWhitespace; + continue; + + case TokenType::WhiteSpace: + { + flags |= TokenFlag::AfterWhitespace; + continue; + } + case TokenType::BlockComment: + case TokenType::LineComment: + { + flags |= TokenFlag::AfterWhitespace; + if (m_optionFlags & OptionFlag::TokenizeComments) + { + // We don't break here, and use the normal token adding logic + // because we want the behavior to be identical (in terms of flags etc) + // as if TokenizeComments is not enabled + char const* textEnd = m_cursor; + + token.type = tokenType; + token.flags = m_tokenFlags; + token.setContent(UnownedStringSlice(textBegin, textEnd)); + + return token; + } + + continue; + } + + // We don't want to skip the end-of-file token, but we *do* + // want to make sure it has appropriate flags to make our life easier + case TokenType::EndOfFile: + flags |= TokenFlag::AtStartOfLine | TokenFlag::AfterWhitespace; + break; + + // We will also do some book-keeping around preprocessor directives here: + // + // If we see a `#` at the start of a line, then we are entering a + // preprocessor directive. + case TokenType::Pound: + if((flags & TokenFlag::AtStartOfLine) != 0) + m_lexerFlags |= kLexerFlag_InDirective; + break; + // + // And if we saw an end-of-line during a directive, then we are + // now leaving that directive. + // + case TokenType::EndOfDirective: + m_lexerFlags &= ~kLexerFlag_InDirective; + break; + + default: + break; + } + + token.type = tokenType; + + char const* textEnd = m_cursor; + + // Note(tfoley): `StringBuilder::Append()` seems to crash when appending zero bytes + if(textEnd != textBegin) + { + // "scrubbing" token value here to remove escaped newlines... + // + // Only perform this work if we encountered an escaped newline + // while lexing this token (e.g., keep a flag on the lexer), or + // do it on-demand when the actual value of the token is needed. + if (m_tokenFlags & TokenFlag::ScrubbingNeeded) + { + // Allocate space that will always be more than enough for stripped contents + char* startDst = (char*)m_memoryArena->allocateUnaligned(textEnd - textBegin); + char* dst = startDst; + + auto tt = textBegin; + while (tt != textEnd) + { + char c = *tt++; + if (c == '\\') + { + char d = *tt; + switch (d) + { + case '\r': case '\n': + { + tt++; + char e = *tt; + if ((d ^ e) == ('\r' ^ '\n')) + { + tt++; + } + } + continue; + + default: + break; + } + } + *dst++ = c; + } + token.setContent(UnownedStringSlice(startDst, dst)); + } + else + { + token.setContent(UnownedStringSlice(textBegin, textEnd)); + } + } + + token.flags = flags; + + m_tokenFlags = 0; + + if (tokenType == TokenType::Identifier) + { + token.setName(m_namePool->getName(token.getContent())); + } + + return token; + } + } + + TokenList Lexer::lexAllTokens() + { + TokenList tokenList; + for(;;) + { + Token token = lexToken(); + tokenList.add(token); + + if(token.type == TokenType::EndOfFile) + return tokenList; + } + } + + /* static */UnownedStringSlice Lexer::sourceLocationLexer(const UnownedStringSlice& in) + { + Lexer lexer; + + SourceManager sourceManager; + sourceManager.initialize(nullptr, nullptr); + + auto sourceFile = sourceManager.createSourceFileWithString(PathInfo::makeUnknown(), in); + auto sourceView = sourceManager.createSourceView(sourceFile, nullptr, SourceLoc::fromRaw(0)); + + DiagnosticSink sink(&sourceManager, nullptr); + + MemoryArena arena; + + RootNamePool rootNamePool; + NamePool namePool; + namePool.setRootNamePool(&rootNamePool); + + lexer.initialize(sourceView, &sink, &namePool, &arena); + + Token tok = lexer.lexToken(); + + if (tok.type == TokenType::Invalid) + { + return UnownedStringSlice(); + } + + const int offset = sourceView->getRange().getOffset(tok.loc); + + SLANG_ASSERT(offset >= 0 && offset <= in.getLength()); + SLANG_ASSERT(Index(offset + tok.charsCount) <= in.getLength()); + + return UnownedStringSlice(in.begin() + offset, in.begin() + offset + tok.charsCount); + } + +} diff --git a/source/compiler-core/slang-lexer.h b/source/compiler-core/slang-lexer.h new file mode 100644 index 000000000..f9a698568 --- /dev/null +++ b/source/compiler-core/slang-lexer.h @@ -0,0 +1,164 @@ +#ifndef SLANG_LEXER_H +#define SLANG_LEXER_H + +#include "../core/slang-basic.h" +#include "slang-diagnostic-sink.h" + +namespace Slang +{ + struct NamePool; + + // + + struct TokenList + { + const Token* begin() const; + const Token* end() const; + + SLANG_FORCE_INLINE void add(const Token& token) { m_tokens.add(token); } + + List m_tokens; + }; + + struct TokenSpan + { + TokenSpan(); + TokenSpan( + TokenList const& tokenList) + : m_begin(tokenList.begin()) + , m_end (tokenList.end ()) + {} + + const Token* begin() const { return m_begin; } + const Token* end () const { return m_end ; } + + int getCount() { return (int)(m_end - m_begin); } + + const Token* m_begin; + const Token* m_end; + }; + + struct TokenReader + { + Token m_nextToken; + TokenReader(); + explicit TokenReader(TokenSpan const& tokens) + : m_cursor(tokens.begin()) + , m_end (tokens.end ()) + , m_nextToken(tokens.begin() ? *tokens.begin() : getEndOfFileToken()) + {} + explicit TokenReader(TokenList const& tokens) + : m_cursor(tokens.begin()) + , m_end (tokens.end ()) + , m_nextToken(tokens.begin() ? *tokens.begin() : getEndOfFileToken()) + {} + struct ParsingCursor + { + Token nextToken; + const Token* tokenReaderCursor = nullptr; + }; + ParsingCursor getCursor() + { + ParsingCursor rs; + rs.nextToken = m_nextToken; + rs.tokenReaderCursor = m_cursor; + return rs; + } + void setCursor(ParsingCursor cursor) + { + m_cursor = cursor.tokenReaderCursor; + m_nextToken = cursor.nextToken; + } + bool isAtCursor(const ParsingCursor& cursor) const + { + return cursor.tokenReaderCursor == m_cursor; + } + bool isAtEnd() const { return m_cursor == m_end; } + Token& peekToken(); + TokenType peekTokenType() const; + SourceLoc peekLoc() const; + + Token advanceToken(); + + int getCount() { return (int)(m_end - m_cursor); } + + const Token* m_cursor; + const Token* m_end; + static Token getEndOfFileToken(); + }; + + typedef unsigned int LexerFlags; + enum + { + kLexerFlag_InDirective = 1 << 0, ///< Turn end-of-line and end-of-file into end-of-directive + kLexerFlag_ExpectFileName = 1 << 1, ///< Support `<>` style strings for file paths + kLexerFlag_IgnoreInvalid = 1 << 2, ///< Suppress errors about invalid/unsupported characters + kLexerFlag_ExpectDirectiveMessage = 1 << 3, ///< Don't lexer ordinary tokens, and instead consume rest of line as a string + }; + + struct Lexer + { + typedef uint32_t OptionFlags; + struct OptionFlag + { + enum Enum : OptionFlags + { + TokenizeComments = 1 << 0, ///< If set comments will be output to the token stream + }; + }; + + void initialize( + SourceView* sourceView, + DiagnosticSink* sink, + NamePool* namePool, + MemoryArena* memoryArena, + OptionFlags optionFlags = 0); + + ~Lexer(); + + /// Runs the lexer to try and extract a single token, which is returned. + /// This can be used by the DiagnosticSink to be able to display more appropriate + /// information when displaying a source location - such as underscoring the + /// token at that location. + /// + /// NOTE! This function is relatively slow, and is designed for use around this specific + /// purpose. It does not return a token or a token type, because that information is + /// not needed by the DiagnosticSink. + static UnownedStringSlice sourceLocationLexer(const UnownedStringSlice& in); + + Token lexToken(LexerFlags extraFlags = 0); + + TokenList lexAllTokens(); + + SourceView* m_sourceView; + DiagnosticSink* m_sink; + NamePool* m_namePool; + + char const* m_cursor; + + char const* m_begin; + char const* m_end; + + /// The starting sourceLoc (same as first location of SourceView) + SourceLoc m_startLoc; + + TokenFlags m_tokenFlags; + LexerFlags m_lexerFlags; + OptionFlags m_optionFlags; + + MemoryArena* m_memoryArena; + }; + + + // Helper routines for extracting values from tokens + String getStringLiteralTokenValue(Token const& token); + String getFileNameTokenValue(Token const& token); + + typedef int64_t IntegerLiteralValue; + typedef double FloatingPointLiteralValue; + + IntegerLiteralValue getIntegerLiteralValue(Token const& token, UnownedStringSlice* outSuffix = 0); + FloatingPointLiteralValue getFloatingPointLiteralValue(Token const& token, UnownedStringSlice* outSuffix = 0); +} + +#endif diff --git a/source/compiler-core/slang-misc-diagnostic-defs.h b/source/compiler-core/slang-misc-diagnostic-defs.h new file mode 100644 index 000000000..a1fd01475 --- /dev/null +++ b/source/compiler-core/slang-misc-diagnostic-defs.h @@ -0,0 +1,27 @@ +// + +// The file is meant to be included multiple times, to produce different +// pieces of declaration/definition code related to diagnostic messages +// +// Each diagnostic is declared here with: +// +// DIAGNOSTIC(id, severity, name, messageFormat) +// +// Where `id` is the unique diagnostic ID, `severity` is the default +// severity (from the `Severity` enum), `name` is a name used to refer +// to this diagnostic from code, and `messageFormat` is the default +// (non-localized) message for the diagnostic, with placeholders +// for any arguments. + +#ifndef DIAGNOSTIC +#error Need to #define DIAGNOSTIC(...) before including +#define DIAGNOSTIC(id, severity, name, messageFormat) /* */ +#endif + +// +// -1 - Notes that decorate another diagnostic. +// + +DIAGNOSTIC(-1, Note, seeTokenPasteLocation, "see token pasted location") + +#undef DIAGNOSTIC diff --git a/source/compiler-core/slang-name-convention-util.cpp b/source/compiler-core/slang-name-convention-util.cpp new file mode 100644 index 000000000..930e00e18 --- /dev/null +++ b/source/compiler-core/slang-name-convention-util.cpp @@ -0,0 +1,213 @@ + +#include "slang-name-convention-util.h" + +#include "../core/slang-char-util.h" +#include "../core/slang-string-util.h" + +namespace Slang +{ + +/* static */NameConvention NameConventionUtil::getConvention(const UnownedStringSlice& slice) +{ + for (const char c : slice) + { + switch (c) + { + case '-': return NameConvention::Kabab; + case '_': return NameConvention::Snake; + default: break; + } + } + return NameConvention::Camel; +} + +/* static */void NameConventionUtil::split(NameConvention convention, const UnownedStringSlice& slice, List& out) +{ + switch (convention) + { + case NameConvention::Kabab: + { + StringUtil::split(slice, '-', out); + break; + } + case NameConvention::Snake: + { + StringUtil::split(slice, '_', out); + break; + } + case NameConvention::Camel: + { + typedef CharUtil::Flags CharFlags; + typedef CharUtil::Flag CharFlag; + + CharFlags prevFlags = 0; + const char*const end = slice.end(); + + const char* start = slice.begin(); + for (const char* cur = start; cur < end; ++cur) + { + const char c = *cur; + const CharUtil::Flags flags = CharUtil::getFlags(c); + + if (flags & CharFlag::Upper) + { + if (prevFlags & CharFlag::Lower) + { + // If we go from lower to upper, we have a transition + out.add(UnownedStringSlice(start, cur)); + start = cur; + } + else if ((prevFlags & CharFlag::Upper) && cur + 1 < end) + { + // This works with capital or uncapitalized acronyms, but if we have two capitalized acronyms following each other - it can't split. + // + // For example + // "IAABBSystem" -> "IAABB", "System" + // + // If it only accepted lower case acronyms the logic could be changed such that the following could be produced + // "IAabbSystem" -> "I", "Aabb", "System" + // + // Since Slang source largely goes with upper case acronyms, we work with the heuristic here.. + + if (CharUtil::isLower(cur[1])) + { + out.add(UnownedStringSlice(start, cur)); + start = cur; + } + } + } + + prevFlags = flags; + } + + // Add any end section + if (start < end) + { + out.add(UnownedStringSlice(start, end)); + } + break; + } + } +} + +void NameConventionUtil::split(const UnownedStringSlice& slice, List& out) +{ + split(getConvention(slice), slice, out); +} + +/* static */void NameConventionUtil::join(const UnownedStringSlice* slices, Index slicesCount, CharCase charCase, char joinChar, StringBuilder& out) +{ + if (slicesCount <= 0) + { + return; + } + + Index totalSize = slicesCount - 1; + for (Index i = 0; i < slicesCount; ++i) + { + totalSize += slices[i].getLength(); + } + + char*const dstStart = out.prepareForAppend(totalSize); + char* dst = dstStart; + + for (Index i = 0; i < slicesCount; ++i) + { + const UnownedStringSlice& slice = slices[i]; + const Index count = slice.getLength(); + const char*const src = slice.begin(); + + if (i > 0) + { + *dst++ = joinChar; + } + + switch (charCase) + { + case CharCase::Upper: + { + for (Index j = 0; j < count; ++j) + { + dst[j] = CharUtil::toUpper(src[j]); + } + break; + } + case CharCase::Lower: + { + for (Index j = 0; j < count; ++j) + { + dst[j] = CharUtil::toLower(src[j]); + } + break; + } + } + + dst += count; + } + + SLANG_ASSERT(dstStart + totalSize == dst); + out.appendInPlace(dstStart, totalSize); +} + +/* static */void NameConventionUtil::join(const UnownedStringSlice* slices, Index slicesCount, CharCase charCase, NameConvention convention, StringBuilder& out) +{ + switch (convention) + { + case NameConvention::Kabab: return join(slices, slicesCount, charCase, '-', out); + case NameConvention::Snake: return join(slices, slicesCount, charCase, '_', out); + case NameConvention::Camel: + { + Index totalSize = 0; + + for (Index i = 0; i < slicesCount; ++i) + { + totalSize += slices[i].getLength(); + } + + char*const dstStart = out.prepareForAppend(totalSize); + char* dst = dstStart; + + for (Index i = 0; i < slicesCount; ++i) + { + const UnownedStringSlice& slice = slices[i]; + Index count = slice.getLength(); + const char* src = slice.begin(); + + Int j = 0; + + if (count > 0 && !(i == 0 && charCase == CharCase::Lower)) + { + // Capitalize first letter of each word, unless on first word and 'lower' + dst[j] = CharUtil::toUpper(src[j]); + j++; + } + + for (; j < count; ++j) + { + dst[j] = CharUtil::toLower(src[j]); + } + + dst += count; + } + break; + } + } +} + +/* static */void NameConventionUtil::convert(NameConvention fromConvention, const UnownedStringSlice& slice, CharCase charCase, NameConvention toConvention, StringBuilder& out) +{ + // Split into slices + List slices; + split(fromConvention, slice, slices); + + // Join the slices in the toConvention + join(slices.getBuffer(), slices.getCount(), charCase, toConvention, out); +} + +/* static */void NameConventionUtil::convert(const UnownedStringSlice& slice, CharCase charCase, NameConvention toConvention, StringBuilder& out) +{ + convert(getConvention(slice), slice, charCase, toConvention, out); +} + +} + diff --git a/source/compiler-core/slang-name-convention-util.h b/source/compiler-core/slang-name-convention-util.h new file mode 100644 index 000000000..d84ffd3b7 --- /dev/null +++ b/source/compiler-core/slang-name-convention-util.h @@ -0,0 +1,54 @@ +#ifndef SLANG_COMPILER_CORE_NAME_CONVENTION_UTIL_H +#define SLANG_COMPILER_CORE_NAME_CONVENTION_UTIL_H + +#include "../core/slang-string.h" +#include "../core/slang-list.h" + +namespace Slang +{ + +enum class NameConvention +{ + Kabab, /// Words are separated with -. WORDS-ARE-SEPARATED + Snake, /// Words are separated with _. WORDS_ARE_SEPARATED + Camel, /// Words start with a capital. (Upper will make first words character capitalized, aka PascalCase) +}; + +enum class CharCase +{ + Upper, + Lower, +}; + +/* This utility is to enable easy conversion and interpretation of names that use standard conventions, typically in programming +languages. The conventions are largely how to represent multiple words together. + +Split is used to split up a name into it's constituent 'words' based on a convention. +Join is used to combine words based on a convention/character case + +Convert uses split and join to allow easy conversion between conventions. +*/ +struct NameConventionUtil +{ + /// Given a slice tries to determine the convention used. + /// If no separators are found, will assume Camel + static NameConvention getConvention(const UnownedStringSlice& slice); + + /// Given a slice and a naming convention, split into it's constituent parts. If convention isn't specified, will infer from slice using getConvention. + static void split(NameConvention convention, const UnownedStringSlice& slice, List& out); + static void split(const UnownedStringSlice& slice, List& out); + + /// Given slices, join together with the specified convention into out + static void join(const UnownedStringSlice* slices, Index slicesCount, CharCase charCase, NameConvention convention, StringBuilder& out); + + /// Join with a join char, and potentially changing case of input slices + static void join(const UnownedStringSlice* slices, Index slicesCount, CharCase charCase, char joinChar, StringBuilder& out); + + /// Convert from one convention to another. If fromConvention isn't specified, will infer from slice using getConvention. + static void convert(NameConvention fromConvention, const UnownedStringSlice& slice, CharCase charCase, NameConvention toConvention, StringBuilder& out); + static void convert(const UnownedStringSlice& slice, CharCase charCase, NameConvention toConvention, StringBuilder& out); +}; + +} + +#endif // SLANG_CORE_NAME_CONVENTION_UTIL_H diff --git a/source/compiler-core/slang-name.cpp b/source/compiler-core/slang-name.cpp new file mode 100644 index 000000000..b6035982b --- /dev/null +++ b/source/compiler-core/slang-name.cpp @@ -0,0 +1,42 @@ +// slang-name.cpp +#include "slang-name.h" + +namespace Slang { + +String getText(Name* name) +{ + if (!name) return String(); + return name->text; +} + +UnownedStringSlice getUnownedStringSliceText(Name* name) +{ + return name ? name->text.getUnownedSlice() : UnownedStringSlice(); +} + +const char* getCstr(Name* name) +{ + return name ? name->text.getBuffer() : nullptr; +} + +Name* NamePool::getName(String const& text) +{ + RefPtr name; + if (rootPool->names.TryGetValue(text, name)) + return name; + + name = new Name(); + name->text = text; + rootPool->names.Add(text, name); + return name; +} + +Name* NamePool::tryGetName(String const& text) +{ + RefPtr name; + if (rootPool->names.TryGetValue(text, name)) + return name; + return nullptr; +} + +} // namespace Slang diff --git a/source/compiler-core/slang-name.h b/source/compiler-core/slang-name.h new file mode 100644 index 000000000..cf702686b --- /dev/null +++ b/source/compiler-core/slang-name.h @@ -0,0 +1,89 @@ +// slang-name.h +#ifndef SLANG_NAME_H_INCLUDED +#define SLANG_NAME_H_INCLUDED + +// This file defines the `Name` type, used to represent +// the name of types, variables, etc. in the AST. + +#include "../core/slang-basic.h" + +namespace Slang { + +// The `Name` type is used to represent the name of a type, variable, etc. +// +// The key benefit of using `Name`s instead of raw strings is that `Name`s +// can be compared for equality just by testing pointer equality. Names +// also don't require any memory management; you can just retain an ordinary +// pointer to one and not deal with reference-counting overhead. +// +// In order to provide these benefits, a `Name` can only be created using +// a `NamePool` that owns the allocations for all the names (so they get +// cleaned up when the pool is deleted), and which is responsible for +// ensuring the uniqueness of name objects. +// +class Name : public RefObject +{ +public: + // The raw text of the name. + // + // Note that at some point in the future we might have other categories + // of name than "simple" names, and so this might change to a structured + // ADT instead of a simple string. + String text; +}; + +// Get the textual string representation of a name +// (e.g., so that it can be printed). +String getText(Name* name); + +/// Get the text as unowned string slice +UnownedStringSlice getUnownedStringSliceText(Name* name); + +// Get a name as a C style string, or nullptr if name is nullptr +const char* getCstr(Name* name); + +// A `RootNamePool` is used to store and look up names. +// If two systems need to work together with names, and be sure that they +// get equivalent names for a string like `"Foo"`, then they need to use +// the same root name pool (directly or indirectly). +// +struct RootNamePool +{ + // The mapping from text strings to the corresponding name. + Dictionary > names; +}; + +// A `NamePool` is effectively a way of storing a subset of the +// names that have been created through a `RootNamePool`. +// +// The intention is that eventually we will add the ability to clean +// up a `NamePool`, and remove the names it created from the corresponding +// `RootNamePool` *if* those names are no longer in use. +// +// The goal of such an approach would be to ensure that the memory +// usage of a `Session` can't bloat over time just because of multiple +// `CompileRequest`s being created, used, and then destroyed (each time +// adding just a few more strings to the name mapping). +// +struct NamePool +{ + // Find or create the `Name` that represents the given `text`. + Name* getName(String const& text); + // Try find the `Name` that represents the given `text`. + // If the name does not exist, return nullptr + Name* tryGetName(String const& text); + // Set the parent name pool to use for lookup + void setRootNamePool(RootNamePool* rootNamePool) + { + this->rootPool = rootNamePool; + } + + // + + // The root name pool to use for storage/lookup + RootNamePool* rootPool = nullptr; +}; + +} // namespace Slang + +#endif diff --git a/source/compiler-core/slang-nvrtc-compiler.cpp b/source/compiler-core/slang-nvrtc-compiler.cpp new file mode 100644 index 000000000..b581d21fd --- /dev/null +++ b/source/compiler-core/slang-nvrtc-compiler.cpp @@ -0,0 +1,773 @@ +// slang-nvrtc-compiler.cpp +#include "slang-nvrtc-compiler.h" + +#include "../core/slang-common.h" +#include "../../slang-com-helper.h" + +#include "../core/slang-blob.h" + +#include "../core/slang-string-util.h" +#include "../core/slang-string-slice-pool.h" + +#include "../core/slang-io.h" +#include "../core/slang-shared-library.h" +#include "../core/slang-semantic-version.h" + + +namespace nvrtc +{ + +typedef enum { + NVRTC_SUCCESS = 0, + NVRTC_ERROR_OUT_OF_MEMORY = 1, + NVRTC_ERROR_PROGRAM_CREATION_FAILURE = 2, + NVRTC_ERROR_INVALID_INPUT = 3, + NVRTC_ERROR_INVALID_PROGRAM = 4, + NVRTC_ERROR_INVALID_OPTION = 5, + NVRTC_ERROR_COMPILATION = 6, + NVRTC_ERROR_BUILTIN_OPERATION_FAILURE = 7, + NVRTC_ERROR_NO_NAME_EXPRESSIONS_AFTER_COMPILATION = 8, + NVRTC_ERROR_NO_LOWERED_NAMES_BEFORE_COMPILATION = 9, + NVRTC_ERROR_NAME_EXPRESSION_NOT_VALID = 10, + NVRTC_ERROR_INTERNAL_ERROR = 11 +} nvrtcResult; + +typedef struct _nvrtcProgram *nvrtcProgram; + +#define SLANG_NVRTC_FUNCS(x) \ + x(const char*, nvrtcGetErrorString, (nvrtcResult result)) \ + x(nvrtcResult, nvrtcVersion, (int *major, int *minor)) \ + x(nvrtcResult, nvrtcCreateProgram, (nvrtcProgram *prog, const char *src, const char *name, int numHeaders, const char * const *headers, const char * const *includeNames)) \ + x(nvrtcResult, nvrtcDestroyProgram, (nvrtcProgram *prog)) \ + x(nvrtcResult, nvrtcCompileProgram, (nvrtcProgram prog, int numOptions, const char * const *options)) \ + x(nvrtcResult, nvrtcGetPTXSize, (nvrtcProgram prog, size_t *ptxSizeRet)) \ + x(nvrtcResult, nvrtcGetPTX, (nvrtcProgram prog, char *ptx)) \ + x(nvrtcResult, nvrtcGetProgramLogSize, (nvrtcProgram prog, size_t *logSizeRet)) \ + x(nvrtcResult, nvrtcGetProgramLog, (nvrtcProgram prog, char *log))\ + x(nvrtcResult, nvrtcAddNameExpression, (nvrtcProgram prog, const char * const name_expression)) \ + x(nvrtcResult, nvrtcGetLoweredName, (nvrtcProgram prog, const char *const name_expression, const char** lowered_name)) + +} // namespace nvrtc + +namespace Slang +{ +using namespace nvrtc; + +static SlangResult _asResult(nvrtcResult res) +{ + switch (res) + { + case NVRTC_SUCCESS: + { + return SLANG_OK; + } + case NVRTC_ERROR_OUT_OF_MEMORY: + { + return SLANG_E_OUT_OF_MEMORY; + } + case NVRTC_ERROR_PROGRAM_CREATION_FAILURE: + case NVRTC_ERROR_INVALID_INPUT: + case NVRTC_ERROR_INVALID_PROGRAM: + { + return SLANG_FAIL; + } + case NVRTC_ERROR_INVALID_OPTION: + { + return SLANG_E_INVALID_ARG; + } + case NVRTC_ERROR_COMPILATION: + case NVRTC_ERROR_BUILTIN_OPERATION_FAILURE: + case NVRTC_ERROR_NO_NAME_EXPRESSIONS_AFTER_COMPILATION: + case NVRTC_ERROR_NO_LOWERED_NAMES_BEFORE_COMPILATION: + case NVRTC_ERROR_NAME_EXPRESSION_NOT_VALID: + { + return SLANG_FAIL; + } + case NVRTC_ERROR_INTERNAL_ERROR: + { + return SLANG_E_INTERNAL_FAIL; + } + default: return SLANG_FAIL; + } +} + +class NVRTCDownstreamCompiler : public DownstreamCompiler +{ +public: + typedef DownstreamCompiler Super; + + // DownstreamCompiler + virtual SlangResult compile(const CompileOptions& options, RefPtr& outResult) SLANG_OVERRIDE; + virtual ISlangSharedLibrary* getSharedLibrary() SLANG_OVERRIDE { return m_sharedLibrary; } + + /// Must be called before use + SlangResult init(ISlangSharedLibrary* library); + + NVRTCDownstreamCompiler() {} + +protected: + + struct ScopeProgram + { + ScopeProgram(NVRTCDownstreamCompiler* compiler, nvrtcProgram program): + m_compiler(compiler), + m_program(program) + { + } + ~ScopeProgram() + { + m_compiler->m_nvrtcDestroyProgram(&m_program); + } + NVRTCDownstreamCompiler* m_compiler; + nvrtcProgram m_program; + }; + + +#define SLANG_NVTRC_MEMBER_FUNCS(ret, name, params) \ + ret (*m_##name) params; + + SLANG_NVRTC_FUNCS(SLANG_NVTRC_MEMBER_FUNCS); + + ComPtr m_sharedLibrary; +}; + +#define SLANG_NVRTC_RETURN_ON_FAIL(x) { nvrtcResult _res = x; if (_res != NVRTC_SUCCESS) return _asResult(_res); } + +SlangResult NVRTCDownstreamCompiler::init(ISlangSharedLibrary* library) +{ +#define SLANG_NVTRC_GET_FUNC(ret, name, params) \ + m_##name = (ret (*) params)library->findFuncByName(#name); \ + if (m_##name == nullptr) return SLANG_FAIL; + + SLANG_NVRTC_FUNCS(SLANG_NVTRC_GET_FUNC) + + m_sharedLibrary = library; + + m_desc.type = SLANG_PASS_THROUGH_NVRTC; + + int major, minor; + m_nvrtcVersion(&major, &minor); + m_desc.majorVersion = major; + m_desc.minorVersion = minor; + + return SLANG_OK; +} + +static SlangResult _parseLocation(const UnownedStringSlice& in, DownstreamDiagnostic& outDiagnostic) +{ + const Index startIndex = in.indexOf('('); + + if (startIndex >= 0) + { + outDiagnostic.filePath = UnownedStringSlice(in.begin(), in.begin() + startIndex); + UnownedStringSlice remaining(in.begin() + startIndex + 1, in.end()); + const Int endIndex = remaining.indexOf(')'); + + UnownedStringSlice lineText = UnownedStringSlice(remaining.begin(), remaining.begin() + endIndex); + + Int line; + SLANG_RETURN_ON_FAIL(StringUtil::parseInt(lineText, line)); + outDiagnostic.fileLine = line; + } + else + { + outDiagnostic.fileLine = 0; + outDiagnostic.filePath = in; + } + return SLANG_OK; +} + +static bool _isDriveLetter(char c) +{ + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); +} + +static bool _hasDriveLetter(const UnownedStringSlice& line) +{ + return line.getLength() > 2 && line[1] == ':' && _isDriveLetter(line[0]); +} + +static SlangResult _parseNVRTCLine(const UnownedStringSlice& line, DownstreamDiagnostic& outDiagnostic) +{ + typedef DownstreamDiagnostic Diagnostic; + typedef Diagnostic::Severity Severity; + + outDiagnostic.stage = Diagnostic::Stage::Compile; + + List split; + if (_hasDriveLetter(line)) + { + // The drive letter has :, which confuses things, so skip that and then fix up first entry + UnownedStringSlice lineWithoutDrive(line.begin() + 2, line.end()); + StringUtil::split(lineWithoutDrive, ':', split); + split[0] = UnownedStringSlice(line.begin(), split[0].end()); + } + else + { + StringUtil::split(line, ':', split); + } + + if (split.getCount() >= 3) + { + // tests/cuda/cuda-compile.cu(7): warning: variable "c" is used before its value is set + + const auto split1 = split[1].trim(); + + if (split1 == "error") + { + outDiagnostic.severity = Severity::Error; + } + else if (split1 == "warning") + { + outDiagnostic.severity = Severity::Warning; + } + outDiagnostic.text = split[2].trim(); + + SLANG_RETURN_ON_FAIL(_parseLocation(split[0], outDiagnostic)); + return SLANG_OK; + } + + return SLANG_E_NOT_FOUND; +} + +SlangResult NVRTCDownstreamCompiler::compile(const CompileOptions& options, RefPtr& outResult) +{ + // This compiler doesn't read files, they should be read externally and stored in sourceContents/sourceContentsPath + if (options.sourceFiles.getCount() > 0) + { + return SLANG_FAIL; + } + + CommandLine cmdLine; + + switch (options.debugInfoType) + { + case DebugInfoType::None: + { + break; + } + default: + { + cmdLine.addArg("--device-debug"); + break; + } + case DebugInfoType::Maximal: + { + cmdLine.addArg("--device-debug"); + cmdLine.addArg("--generate-line-info"); + break; + } + } + + // Don't seem to have such a control, so ignore for now + //switch (options.optimizationLevel) + //{ + // default: break; + //} + + switch (options.floatingPointMode) + { + case FloatingPointMode::Default: break; + case FloatingPointMode::Precise: + { + break; + } + case FloatingPointMode::Fast: + { + cmdLine.addArg("--use_fast_math"); + break; + } + } + + // Add defines + for (const auto& define : options.defines) + { + StringBuilder builder; + builder << "-D"; + builder << define.nameWithSig; + if (define.value.getLength()) + { + builder << "=" << define.value; + } + + cmdLine.addArg(builder); + } + + // Add includes + for (const auto& include : options.includePaths) + { + cmdLine.addArg("-I"); + cmdLine.addArg(include); + } + + // Neither of these options are strictly required, for general use of nvrtc, + // but are enabled to make use withing Slang work more smoothly + { + // Require c++14, as makes initialization construction with {} available and so simplifies code generation + cmdLine.addArg("-std=c++14"); + + // Disable all warnings + // This is arguably too much - but nvrtc does not appear to have a mechanism to switch off individual warnings. + // I tried the -Xcudafe mechanism but that does not appear to work for nvrtc + cmdLine.addArg("-w"); + } + + { + // The lowest supported CUDA architecture version supported + // by NVRTC is `compute_30`. + // + SemanticVersion version(3); + + // Newer releases of NVRTC only support `compute_35` and up + // (with everything before `compute_52` being deprecated). + // + if( m_desc.majorVersion >= 11 ) + { + version = SemanticVersion(3, 5); + } + + // If constructs used in the code to be compield require + // a higher architecture version than the minimum, then + // we will set the version to the highest version listed + // among the requirements. + // + for (const auto& capabilityVersion : options.requiredCapabilityVersions) + { + if (capabilityVersion.kind == DownstreamCompiler::CapabilityVersion::Kind::CUDASM) + { + if (capabilityVersion.version > version) + { + version = capabilityVersion.version; + } + } + } + + StringBuilder builder; + builder << "-arch=compute_"; + builder << version.m_major; + + SLANG_ASSERT(version.m_minor >= 0 && version.m_minor <= 9); + builder << char('0' + version.m_minor); + + cmdLine.addArg(builder); + } + + List headers; + List headerIncludeNames; + + // If compiling for OptiX, we need to add the appropriate search paths to the command line. + // + if(options.pipelineType == PipelineType::RayTracing) + { + // The device-side OptiX API is accessed through a constellation + // of headers provided by the OptiX SDK, so we need to set an + // include path for the compile that makes those visible. + // + // TODO: The OptiX SDK installer doesn't set any kind of environment + // variable to indicate where the SDK was installed, so we seemingly + // need to probe paths instead. The form of the path will differ + // betwene Windows and Unix-y platforms, and we will need some kind + // of approach to probe multiple versiosn and use the latest. + // + // HACK: For now I'm using the fixed path for the most recent SDK + // release on Windows. This means that OptiX cross-compilation will + // only "work" on a subset of platforms, but that doesn't matter + // for now since it doesn't really "work" at all. + // + cmdLine.addArg("-I"); + cmdLine.addArg("C:/ProgramData/NVIDIA Corporation/OptiX SDK 7.0.0/include/"); + + // The OptiX headers in turn `#include ` and expect that + // to work. We could try to also add in an include path from the CUDA + // SDK (which seems to provide a `stddef.h` in the most recent version), + // but using that version doesn't seem to work (and also bakes in a + // requirement that the user have the CUDA SDK installed in addition + // to the OptiX SDK). + // + // Instead, we will rely on the NVRTC feature that lets us set up + // memory buffers to be used as include files by the we compile. + // We will define a dummy `stddef.h` that includes the bare minimum + // lines required to get the OptiX headers to compile without complaint. + // + // TODO: Confirm that the `LP64` definition herei s actually needed. + // + headerIncludeNames.add("stddef.h"); + headers.add("#pragma once\n" "#define LP64\n"); + + // Finally, we want the CUDA prelude to be able to react to whether + // or not OptiX is required (most notably by `#include`ing the appropriate + // header(s)), so we will insert a preprocessor define to indicate + // the requirement. + // + cmdLine.addArg("-DSLANG_CUDA_ENABLE_OPTIX"); + } + + SLANG_ASSERT(headers.getCount() == headerIncludeNames.getCount()); + + nvrtcProgram program = nullptr; + nvrtcResult res = m_nvrtcCreateProgram(&program, options.sourceContents.getBuffer(), options.sourceContentsPath.getBuffer(), + (int) headers.getCount(), + headers.getBuffer(), + headerIncludeNames.getBuffer()); + if (res != NVRTC_SUCCESS) + { + return _asResult(res); + } + ScopeProgram scope(this, program); + + List dstOptions; + dstOptions.setCount(cmdLine.m_args.getCount()); + for (Index i = 0; i < cmdLine.m_args.getCount(); ++i) + { + dstOptions[i] = cmdLine.m_args[i].value.getBuffer(); + } + + res = m_nvrtcCompileProgram(program, int(dstOptions.getCount()), dstOptions.getBuffer()); + + RefPtr blob; + DownstreamDiagnostics diagnostics; + + diagnostics.result = _asResult(res); + + { + String rawDiagnostics; + + size_t logSize = 0; + SLANG_NVRTC_RETURN_ON_FAIL(m_nvrtcGetProgramLogSize(program, &logSize)); + + if (logSize) + { + char* dst = rawDiagnostics.prepareForAppend(Index(logSize)); + SLANG_NVRTC_RETURN_ON_FAIL(m_nvrtcGetProgramLog(program, dst)); + rawDiagnostics.appendInPlace(dst, Index(logSize)); + + diagnostics.rawDiagnostics = rawDiagnostics; + } + + // Parse the diagnostics here + for (auto line : LineParser(diagnostics.rawDiagnostics.getUnownedSlice())) + { + DownstreamDiagnostic diagnostic; + SlangResult lineRes = _parseNVRTCLine(line, diagnostic); + + if (SLANG_SUCCEEDED(lineRes)) + { + diagnostics.diagnostics.add(diagnostic); + } + else if (lineRes != SLANG_E_NOT_FOUND) + { + return lineRes; + } + } + + // if it has a compilation error.. set on output + if (diagnostics.has(DownstreamDiagnostic::Severity::Error)) + { + diagnostics.result = SLANG_FAIL; + } + } + + if (res == nvrtc::NVRTC_SUCCESS) + { + // We should parse the log to set up the diagnostics + size_t ptxSize; + SLANG_NVRTC_RETURN_ON_FAIL(m_nvrtcGetPTXSize(program, &ptxSize)); + + List ptx; + ptx.setCount(Index(ptxSize)); + + SLANG_NVRTC_RETURN_ON_FAIL(m_nvrtcGetPTX(program, (char*)ptx.getBuffer())); + + blob = ListBlob::moveCreate(ptx); + } + + outResult = new BlobDownstreamCompileResult(diagnostics, blob); + + return SLANG_OK; +} + +/* An implementation of Path::Visitor that can be used for finding NVRTC shared library installations. */ +struct NVRTCPathVisitor : Path::Visitor +{ + struct Candidate + { + typedef Candidate ThisType; + + bool operator==(const ThisType& rhs) const { return path == rhs.path && version == rhs.version; } + bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } + + static Candidate make(const String& path, const SemanticVersion& version) + { + Candidate can; + can.version = version; + can.path = path; + return can; + } + String path; + SemanticVersion version; + }; + + Index findVersion(const SemanticVersion& version) const + { + const Index count = m_candidates.getCount(); + for (Index i = 0; i < count; ++i) + { + if (m_candidates[i].version == version) + { + return i; + } + } + return -1; + } + + static bool _orderCandiate(const Candidate& a, const Candidate& b) { return a.version < b.version; } + void sortCandidates() { m_candidates.sort(_orderCandiate); } + + void accept(Path::Type type, const UnownedStringSlice& filename) SLANG_OVERRIDE + { + // Lets make sure it start's with nvrtc64, but not worry about case + if (type == Path::Type::File) + { + // If there is a defined extension, make sure it has it + if (m_postfix.getLength() && filename.getLength() >= m_postfix.getLength()) + { + // We test without case - really for windows + UnownedStringSlice filenamePostfix = filename.tail(filename.getLength() - m_postfix.getLength()); + if (!filenamePostfix.caseInsensitiveEquals(m_postfix.getUnownedSlice())) + { + return; + } + } + + if (filename.getLength() >= m_prefix.getLength() && + filename.subString(0, m_prefix.getLength()).caseInsensitiveEquals(m_prefix.getUnownedSlice())) + { + // Versions are typically (on windows) of the form + // nvrtc64_110_2.dll + // 11 - Major + // 0 Minor + // 2 Patch + Index endIndex = filename.indexOf('.'); + endIndex = (endIndex < 0) ? filename.getLength() : endIndex; + + UnownedStringSlice versionSlice = UnownedStringSlice(filename.begin() + m_prefix.getLength(), filename.begin() + endIndex); + + Int patch = 0; + UnownedStringSlice majorMinorSlice; + { + List slices; + StringUtil::split(versionSlice, '_', slices); + if (slices.getCount() >= 2) + { + // We don't bother checking for error here, if it's not parsable, it will be 0 + StringUtil::parseInt(slices[1], patch); + } + majorMinorSlice = slices[0]; + } + + if (majorMinorSlice.getLength() < 2) + { + // Must be a major and minor + return; + } + + UnownedStringSlice majorSlice = majorMinorSlice.head(majorMinorSlice.getLength() - 1); + UnownedStringSlice minorSlice = majorMinorSlice.subString(majorMinorSlice.getLength() - 1, 1); + + Int major; + Int minor; + + if (SLANG_FAILED(StringUtil::parseInt(majorSlice, major)) || + SLANG_FAILED(StringUtil::parseInt(minorSlice, minor))) + { + return; + } + + const SemanticVersion version = SemanticVersion(int(major), int(minor), int(patch)); + + // We may want to add multiple versions, if they are in different locations - as there may be multiple entries + // in the PATH, and only one works. We'll only know which works by loading +#if 0 + // We already found this version, so let's not add it again + if (findVersion(version) >= 0) + { + return; + } +#endif + + // Strip to make a shared library name + UnownedStringSlice sharedLibraryName = filename.tail(m_prefix.getLength() - m_sharedLibraryStem.getLength()); + sharedLibraryName = filename.head(filename.getLength() - m_postfix.getLength()); + + auto candidate = Candidate::make(Path::combine(m_basePath, sharedLibraryName), version); + + // If we already have this candidate, then skip + if (m_candidates.indexOf(candidate) >= 0) + { + return; + } + + // Add to the list of candidates + m_candidates.add(candidate); + } + } + } + + SlangResult findInDirectory(const String& path) + { + m_basePath = path; + return Path::find(path, nullptr, this); + } + + bool hasCandidates() const { return m_candidates.getCount() > 0; } + + NVRTCPathVisitor(const UnownedStringSlice& sharedLibraryStem): + m_sharedLibraryStem(sharedLibraryStem) + { + // Work out the prefix and postfix of the shader + StringBuilder buf; + SharedLibrary::appendPlatformFileName(sharedLibraryStem, buf); + const Index index = buf.indexOf(sharedLibraryStem); + SLANG_ASSERT(index >= 0); + + m_prefix = buf.getUnownedSlice().head(index + sharedLibraryStem.getLength()); + m_postfix = buf.getUnownedSlice().tail(index + sharedLibraryStem.getLength()); + } + + String m_prefix; + String m_postfix; + String m_basePath; + String m_sharedLibraryStem; + + List m_candidates; +}; + +static SlangResult _findAndLoadNVRTC(ISlangSharedLibraryLoader* loader, ComPtr& outLibrary) +{ +#if SLANG_WINDOWS_FAMILY + // We only need to search 64 bit versions on windows + NVRTCPathVisitor visitor(UnownedStringSlice::fromLiteral("nvrtc64_")); + + // First try the instance path (if supported on platform) + { + StringBuilder instancePath; + if (SLANG_SUCCEEDED(PlatformUtil::getInstancePath(instancePath))) + { + visitor.findInDirectory(instancePath); + } + } + + // If we don't have a candidate try CUDA_PATH + if (!visitor.hasCandidates()) + { + StringBuilder buf; + if (!SLANG_SUCCEEDED(PlatformUtil::getEnvironmentVariable(UnownedStringSlice::fromLiteral("CUDA_PATH"), buf))) + { + // Look for candidates in the directory + visitor.findInDirectory(Path::combine(buf, "bin")); + } + } + + // If we haven't we go searching through PATH + if (!visitor.hasCandidates()) + { + List splitPath; + + StringBuilder buf; + if (SLANG_SUCCEEDED(PlatformUtil::getEnvironmentVariable(UnownedStringSlice::fromLiteral("PATH"), buf))) + { + // Split so we get individual paths + List paths; + StringUtil::split(buf.getUnownedSlice(), ';', paths); + + // We use a pool to make sure we only check each path once + StringSlicePool pool(StringSlicePool::Style::Empty); + + // We are going to search the paths in order + for (const auto path : paths) + { + // PATH can have the same path multiple times. If we have already searched this path, we don't need to again + if (!pool.has(path)) + { + pool.add(path); + + Path::split(path, splitPath); + + // We could search every path, but here we restrict to paths that look like CUDA installations. + // It's a path that contains a CUDA directory and has bin + if (splitPath.indexOf("CUDA") >= 0 && splitPath[splitPath.getCount() - 1].caseInsensitiveEquals(UnownedStringSlice::fromLiteral("bin"))) + { + // Okay lets search it + visitor.findInDirectory(path); + } + } + } + } + } + + // Put into version order with oldest first. + visitor.sortCandidates(); + + // We want to start with the newest version... + for (Index i = visitor.m_candidates.getCount() - 1; i >= 0; --i) + { + const auto& candidate = visitor.m_candidates[i]; + if (SLANG_SUCCEEDED(loader->loadSharedLibrary(candidate.path.getBuffer(), outLibrary.writeRef()))) + { + return SLANG_OK; + } + } +#endif + // This is an official-ish list of versions is here: + // https://developer.nvidia.com/cuda-toolkit-archive + + // Filenames for NVRTC + // https://docs.nvidia.com/cuda/nvrtc/index.html + // + // From this it appears on platforms other than windows the SharedLibrary name + // should be nvrtc which is already tried, so we can give up now. + return SLANG_E_NOT_FOUND; +} + +/* static */SlangResult NVRTCDownstreamCompilerUtil::locateCompilers(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set) +{ + ComPtr library; + + // If the user supplies a path to their preferred version of NVRTC, + // we just use this. + if (path.getLength() != 0) + { + SLANG_RETURN_ON_FAIL(loader->loadSharedLibrary(path.getBuffer(), library.writeRef())); + } + else + { + // As a catch-all for non-Windows platforms, we search for + // a library simply named `nvrtc` (well, `libnvrtc`) which + // is expected to match whatever the user has installed. + // + // On Windows an installation could place the version of nvrtc it uses in the same directory + // as the slang binary, such that it's loaded. + // Using this name also allows a ISlangSharedLibraryLoader to easily identify what is required + // and perhaps load a specific version + if (SLANG_FAILED(loader->loadSharedLibrary("nvrtc", library.writeRef()))) + { + // Try something more sophisticated to locate NVRTC + SLANG_RETURN_ON_FAIL(_findAndLoadNVRTC(loader, library)); + } + } + + SLANG_ASSERT(library); + if (!library) + { + return SLANG_FAIL; + } + + RefPtr compiler(new NVRTCDownstreamCompiler); + SLANG_RETURN_ON_FAIL(compiler->init(library)); + + set->addCompiler(compiler); + return SLANG_OK; +} + + +} diff --git a/source/compiler-core/slang-nvrtc-compiler.h b/source/compiler-core/slang-nvrtc-compiler.h new file mode 100644 index 000000000..48c6d4da6 --- /dev/null +++ b/source/compiler-core/slang-nvrtc-compiler.h @@ -0,0 +1,19 @@ +#ifndef SLANG_NVRTC_COMPILER_UTIL_H +#define SLANG_NVRTC_COMPILER_UTIL_H + +#include "slang-downstream-compiler.h" + +#include "../core/slang-platform.h" + +namespace Slang +{ + + +struct NVRTCDownstreamCompilerUtil +{ + static SlangResult locateCompilers(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set); +}; + +} + +#endif diff --git a/source/compiler-core/slang-source-loc.cpp b/source/compiler-core/slang-source-loc.cpp new file mode 100644 index 000000000..4b589fbf3 --- /dev/null +++ b/source/compiler-core/slang-source-loc.cpp @@ -0,0 +1,689 @@ +// slang-source-loc.cpp +#include "slang-source-loc.h" + +#include "../core/slang-string-util.h" + +namespace Slang { + +/* !!!!!!!!!!!!!!!!!!!!!!!!! SourceView !!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +const String PathInfo::getMostUniqueIdentity() const +{ + switch (type) + { + case Type::Normal: return uniqueIdentity; + case Type::FoundPath: + case Type::FromString: + { + return foundPath; + } + default: return ""; + } +} + +bool PathInfo::operator==(const ThisType& rhs) const +{ + // They must be the same type + if (type != rhs.type) + { + return false; + } + + switch (type) + { + case Type::TokenPaste: + case Type::TypeParse: + case Type::Unknown: + case Type::CommandLine: + { + return true; + } + case Type::Normal: + { + return foundPath == rhs.foundPath && uniqueIdentity == rhs.uniqueIdentity; + } + case Type::FromString: + case Type::FoundPath: + { + // Only have a found path + return foundPath == rhs.foundPath; + } + default: break; + } + + return false; +} + +void PathInfo::appendDisplayName(StringBuilder& out) const +{ + switch (type) + { + case Type::TokenPaste: out << "[Token Paste]"; break; + case Type::TypeParse: out << "[Type Parse]"; break; + case Type::Unknown: out << "[Unknown]"; break; + case Type::CommandLine: out << "[Command Line]"; break; + case Type::Normal: + case Type::FromString: + case Type::FoundPath: + { + + out.appendChar('"'); + StringUtil::appendEscaped(foundPath.getUnownedSlice(), out); + out.appendChar('"'); + break; + } + default: break; + } +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!! SourceView !!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +int SourceView::findEntryIndex(SourceLoc sourceLoc) const +{ + if (!m_range.contains(sourceLoc)) + { + return -1; + } + + const auto rawValue = sourceLoc.getRaw(); + + Index hi = m_entries.getCount(); + // If there are no entries, or it is in front of the first entry, then there is no associated entry + if (hi == 0 || + m_entries[0].m_startLoc.getRaw() > sourceLoc.getRaw()) + { + return -1; + } + + Index lo = 0; + while (lo + 1 < hi) + { + const Index mid = (hi + lo) >> 1; + const Entry& midEntry = m_entries[mid]; + SourceLoc::RawValue midValue = midEntry.m_startLoc.getRaw(); + if (midValue <= rawValue) + { + // The location we seek is at or after this entry + lo = mid; + } + else + { + // The location we seek is before this entry + hi = mid; + } + } + + return int(lo); +} + +void SourceView::addLineDirective(SourceLoc directiveLoc, StringSlicePool::Handle pathHandle, int line) +{ + SLANG_ASSERT(pathHandle != StringSlicePool::Handle(0)); + SLANG_ASSERT(m_range.contains(directiveLoc)); + + // Check that the directiveLoc values are always increasing + SLANG_ASSERT(m_entries.getCount() == 0 || (m_entries.getLast().m_startLoc.getRaw() < directiveLoc.getRaw())); + + // Calculate the offset + const int offset = m_range.getOffset(directiveLoc); + + // Get the line index in the original file + const int lineIndex = m_sourceFile->calcLineIndexFromOffset(offset); + + Entry entry; + entry.m_startLoc = directiveLoc; + entry.m_pathHandle = pathHandle; + + // We also need to make sure that any lookups for line numbers will + // get corrected based on this files location. + // We assume the line number coming from the directive is a line number, NOT an index, so the correction needs + 1 + // There is an additional + 1 because we want the NEXT line - ie the line after the #line directive, to the specified value + // Taking both into account means +2 is correct 'fix' + entry.m_lineAdjust = line - (lineIndex + 2); + + m_entries.add(entry); +} + +void SourceView::addLineDirective(SourceLoc directiveLoc, const String& path, int line) +{ + StringSlicePool::Handle pathHandle = getSourceManager()->getStringSlicePool().add(path.getUnownedSlice()); + return addLineDirective(directiveLoc, pathHandle, line); +} + +void SourceView::addDefaultLineDirective(SourceLoc directiveLoc) +{ + SLANG_ASSERT(m_range.contains(directiveLoc)); + // Check that the directiveLoc values are always increasing + SLANG_ASSERT(m_entries.getCount() == 0 || (m_entries.getLast().m_startLoc.getRaw() < directiveLoc.getRaw())); + + // Well if there are no entries, or the last one puts it in default case, then we don't need to add anything + if (m_entries.getCount() == 0 || (m_entries.getCount() && m_entries.getLast().isDefault())) + { + return; + } + + Entry entry; + entry.m_startLoc = directiveLoc; + entry.m_lineAdjust = 0; // No line adjustment... we are going back to default + entry.m_pathHandle = StringSlicePool::Handle(0); // Mark that there is no path, and that this is a 'default' + + SLANG_ASSERT(entry.isDefault()); + + m_entries.add(entry); +} + +HumaneSourceLoc SourceView::getHumaneLoc(SourceLoc loc, SourceLocType type) +{ + const int offset = m_range.getOffset(loc); + + // We need the line index from the original source file + const int lineIndex = m_sourceFile->calcLineIndexFromOffset(offset); + + // TODO: we should really translate the byte index in the line + // to deal with: + // + // - Non-ASCII characters, while might consume multiple bytes + // + // - Tab characters, which should really adjust how we report + // columns (although how are we supposed to know the setting + // that an IDE expects us to use when reporting locations?) + const int columnIndex = m_sourceFile->calcColumnIndex(lineIndex, offset); + + HumaneSourceLoc humaneLoc; + humaneLoc.column = columnIndex + 1; + humaneLoc.line = lineIndex + 1; + + // Make up a default entry + StringSlicePool::Handle pathHandle = StringSlicePool::Handle(0); + + // Only bother looking up the entry information if we want a 'Normal' lookup + const int entryIndex = (type == SourceLocType::Nominal) ? findEntryIndex(loc) : -1; + if (entryIndex >= 0) + { + const Entry& entry = m_entries[entryIndex]; + // Adjust the line + humaneLoc.line += entry.m_lineAdjust; + // Get the pathHandle.. + pathHandle = entry.m_pathHandle; + } + + humaneLoc.pathInfo = _getPathInfoFromHandle(pathHandle); + return humaneLoc; +} + +PathInfo SourceView::_getPathInfo() const +{ + if (m_viewPath.getLength()) + { + PathInfo pathInfo(m_sourceFile->getPathInfo()); + pathInfo.foundPath = m_viewPath; + return pathInfo; + } + else + { + return m_sourceFile->getPathInfo(); + } +} + +PathInfo SourceView::_getPathInfoFromHandle(StringSlicePool::Handle pathHandle) const +{ + // If there is no override path, then just the source files path + if (pathHandle == StringSlicePool::Handle(0)) + { + return _getPathInfo(); + } + else + { + return PathInfo::makePath(getSourceManager()->getStringSlicePool().getSlice(pathHandle)); + } +} + +PathInfo SourceView::getPathInfo(SourceLoc loc, SourceLocType type) +{ + if (type == SourceLocType::Actual) + { + return _getPathInfo(); + } + + const int entryIndex = findEntryIndex(loc); + return _getPathInfoFromHandle((entryIndex >= 0) ? m_entries[entryIndex].m_pathHandle : StringSlicePool::Handle(0)); +} + +/* !!!!!!!!!!!!!!!!!!!!!!! SourceFile !!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +void SourceFile::setLineBreakOffsets(const uint32_t* offsets, UInt numOffsets) +{ + m_lineBreakOffsets.clear(); + m_lineBreakOffsets.addRange(offsets, numOffsets); +} + +const List& SourceFile::getLineBreakOffsets() +{ + // We now have a raw input file that we can search for line breaks. + // We obviously don't want to do a linear scan over and over, so we will + // cache an array of line break locations in the file. + if (m_lineBreakOffsets.getCount() == 0) + { + UnownedStringSlice content(getContent()), line; + char const* contentBegin = content.begin(); + while (StringUtil::extractLine(content, line)) + { + m_lineBreakOffsets.add(uint32_t(line.begin() - contentBegin)); + } + // Note that we do *not* treat the end of the file as a line + // break, because otherwise we would report errors like + // "end of file inside string literal" with a line number + // that points at a line that doesn't exist. + } + + return m_lineBreakOffsets; +} + +SourceFile::OffsetRange SourceFile::getOffsetRangeAtLineIndex(Index lineIndex) +{ + const List& offsets = getLineBreakOffsets(); + const Index count = offsets.getCount(); + + if (lineIndex >= count - 1) + { + // Work out the line start + const uint32_t offsetEnd = uint32_t(getContentSize()); + const uint32_t offsetStart = (lineIndex >= count) ? offsetEnd : offsets[lineIndex]; + // The line is the span from start, to the end of the content + return OffsetRange{ offsetStart, offsetEnd }; + } + else + { + const uint32_t offsetStart = offsets[lineIndex]; + const uint32_t offsetEnd = offsets[lineIndex + 1]; + return OffsetRange { offsetStart, offsetEnd }; + } +} + +UnownedStringSlice SourceFile::getLineAtIndex(Index lineIndex) +{ + const OffsetRange range = getOffsetRangeAtLineIndex(lineIndex); + + if (range.isValid() && hasContent()) + { + const UnownedStringSlice content = getContent(); + SLANG_ASSERT(range.end <= uint32_t(content.getLength())); + + const char*const text = content.begin(); + return UnownedStringSlice(text + range.start, text + range.end); + } + + return UnownedStringSlice(); +} + +UnownedStringSlice SourceFile::getLineContainingOffset(uint32_t offset) +{ + const Index lineIndex = calcLineIndexFromOffset(offset); + return getLineAtIndex(lineIndex); +} + +bool SourceFile::isOffsetOnLine(uint32_t offset, Index lineIndex) +{ + const OffsetRange range = getOffsetRangeAtLineIndex(lineIndex); + return range.isValid() && range.containsInclusive(offset); +} + +int SourceFile::calcLineIndexFromOffset(int offset) +{ + SLANG_ASSERT(UInt(offset) <= getContentSize()); + + // Make sure we have the line break offsets + const auto& lineBreakOffsets = getLineBreakOffsets(); + + // At this point we can assume the `lineBreakOffsets` array has been filled in. + // We will use a binary search to find the line index that contains our + // chosen offset. + Index lo = 0; + Index hi = lineBreakOffsets.getCount(); + + while (lo + 1 < hi) + { + const Index mid = (hi + lo) >> 1; + const uint32_t midOffset = lineBreakOffsets[mid]; + if (midOffset <= uint32_t(offset)) + { + lo = mid; + } + else + { + hi = mid; + } + } + + return int(lo); +} + +int SourceFile::calcColumnIndex(int lineIndex, int offset) +{ + const auto& lineBreakOffsets = getLineBreakOffsets(); + return offset - lineBreakOffsets[lineIndex]; +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!! SourceFile !!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +void SourceFile::setContents(ISlangBlob* blob) +{ + const UInt contentSize = blob->getBufferSize(); + + SLANG_ASSERT(contentSize == m_contentSize); + + char const* contentBegin = (char const*)blob->getBufferPointer(); + char const* contentEnd = contentBegin + contentSize; + + m_contentBlob = blob; + m_content = UnownedStringSlice(contentBegin, contentEnd); +} + +void SourceFile::setContents(const String& content) +{ + ComPtr contentBlob = StringUtil::createStringBlob(content); + setContents(contentBlob); +} + +SourceFile::SourceFile(SourceManager* sourceManager, const PathInfo& pathInfo, size_t contentSize) : + m_sourceManager(sourceManager), + m_pathInfo(pathInfo), + m_contentSize(contentSize) +{ +} + +SourceFile::~SourceFile() +{ +} + +String SourceFile::calcVerbosePath() const +{ + ISlangFileSystemExt* fileSystemExt = getSourceManager()->getFileSystemExt(); + + if (fileSystemExt) + { + String canonicalPath; + ComPtr canonicalPathBlob; + if (SLANG_SUCCEEDED(fileSystemExt->getCanonicalPath(m_pathInfo.foundPath.getBuffer(), canonicalPathBlob.writeRef()))) + { + canonicalPath = StringUtil::getString(canonicalPathBlob); + } + if (canonicalPath.getLength() > 0) + { + return canonicalPath; + } + } + + return m_pathInfo.foundPath; +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!! SourceManager !!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +void SourceManager::initialize( + SourceManager* p, + ISlangFileSystemExt* fileSystemExt) +{ + m_fileSystemExt = fileSystemExt; + + m_parent = p; + + if( p ) + { + // If we have a parent source manager, then we assume that all code at that level + // has already been loaded, and it is safe to start our own source locations + // right after those from the parent. + // + // TODO: more clever allocation in cases where that might not be reasonable + m_startLoc = p->m_nextLoc; + } + else + { + // Location zero is reserved for an invalid location, + // so we need to start reserving locations starting at 1. + m_startLoc = SourceLoc::fromRaw(1); + } + + m_nextLoc = m_startLoc; +} + +SourceManager::~SourceManager() +{ + for (auto item : m_sourceViews) + { + delete item; + } + + for (auto item : m_sourceFiles) + { + delete item; + } +} + +UnownedStringSlice SourceManager::allocateStringSlice(const UnownedStringSlice& slice) +{ + const UInt numChars = slice.getLength(); + + char* dst = (char*)m_memoryArena.allocate(numChars); + ::memcpy(dst, slice.begin(), numChars); + + return UnownedStringSlice(dst, numChars); +} + +SourceRange SourceManager::allocateSourceRange(UInt size) +{ + // TODO: consider using atomics here + + + SourceLoc beginLoc = m_nextLoc; + SourceLoc endLoc = beginLoc + size; + + // We need to be able to represent the location that is *at* the end of + // the input source, so the next available location for a new file + // must be placed one after the end of this one. + + m_nextLoc = endLoc + 1; + + return SourceRange(beginLoc, endLoc); +} + +SourceFile* SourceManager::createSourceFileWithSize(const PathInfo& pathInfo, size_t contentSize) +{ + SourceFile* sourceFile = new SourceFile(this, pathInfo, contentSize); + m_sourceFiles.add(sourceFile); + return sourceFile; +} + +SourceFile* SourceManager::createSourceFileWithString(const PathInfo& pathInfo, const String& contents) +{ + SourceFile* sourceFile = new SourceFile(this, pathInfo, contents.getLength()); + m_sourceFiles.add(sourceFile); + sourceFile->setContents(contents); + return sourceFile; +} + +SourceFile* SourceManager::createSourceFileWithBlob(const PathInfo& pathInfo, ISlangBlob* blob) +{ + SourceFile* sourceFile = new SourceFile(this, pathInfo, blob->getBufferSize()); + m_sourceFiles.add(sourceFile); + sourceFile->setContents(blob); + return sourceFile; +} + +SourceView* SourceManager::createSourceView(SourceFile* sourceFile, const PathInfo* pathInfo, SourceLoc initiatingSourceLoc) +{ + SourceRange range = allocateSourceRange(sourceFile->getContentSize()); + + SourceView* sourceView = nullptr; + if (pathInfo && + (pathInfo->foundPath.getLength() && sourceFile->getPathInfo().foundPath != pathInfo->foundPath)) + { + sourceView = new SourceView(sourceFile, range, &pathInfo->foundPath, initiatingSourceLoc); + } + else + { + sourceView = new SourceView(sourceFile, range, nullptr, initiatingSourceLoc); + } + + m_sourceViews.add(sourceView); + + return sourceView; +} + +SourceView* SourceManager::findSourceView(SourceLoc loc) const +{ + Index hi = m_sourceViews.getCount(); + // It must be in the range of this manager and have associated views for it to possibly be a hit + if (!getSourceRange().contains(loc) || hi == 0) + { + return nullptr; + } + + // If we don't have very many, we may as well just linearly search + if (hi <= 8) + { + for (int i = 0; i < hi; ++i) + { + SourceView* view = m_sourceViews[i]; + if (view->getRange().contains(loc)) + { + return view; + } + } + return nullptr; + } + + const SourceLoc::RawValue rawLoc = loc.getRaw(); + + // Binary chop to see if we can find the associated SourceUnit + Index lo = 0; + while (lo + 1 < hi) + { + Index mid = (hi + lo) >> 1; + + SourceView* midView = m_sourceViews[mid]; + if (midView->getRange().contains(loc)) + { + return midView; + } + + const SourceLoc::RawValue midValue = midView->getRange().begin.getRaw(); + if (midValue <= rawLoc) + { + // The location we seek is at or after this entry + lo = mid; + } + else + { + // The location we seek is before this entry + hi = mid; + } + } + + // Check if low is actually a hit + SourceView* view = m_sourceViews[lo]; + return (view->getRange().contains(loc)) ? view : nullptr; +} + +SourceView* SourceManager::findSourceViewRecursively(SourceLoc loc) const +{ + // Start with this manager + const SourceManager* manager = this; + do + { + SourceView* sourceView = manager->findSourceView(loc); + // If we found a hit we are done + if (sourceView) + { + return sourceView; + } + // Try the parent + manager = manager->m_parent; + } + while (manager); + // Didn't find it + return nullptr; +} + +SourceFile* SourceManager::findSourceFile(const String& uniqueIdentity) const +{ + SourceFile*const* filePtr = m_sourceFileMap.TryGetValue(uniqueIdentity); + return (filePtr) ? *filePtr : nullptr; +} + +SourceFile* SourceManager::findSourceFileRecursively(const String& uniqueIdentity) const +{ + const SourceManager* manager = this; + do + { + SourceFile* sourceFile = manager->findSourceFile(uniqueIdentity); + if (sourceFile) + { + return sourceFile; + } + manager = manager->m_parent; + } while (manager); + return nullptr; +} + +SourceFile* SourceManager::findSourceFileByContentRecursively(const char* text) +{ + const SourceManager* manager = this; + do + { + SourceFile* sourceFile = manager->findSourceFileByContent(text); + if (sourceFile) + { + return sourceFile; + } + manager = manager->m_parent; + } while (manager); + return nullptr; +} + +SourceFile* SourceManager::findSourceFileByContent(const char* text) const +{ + for (SourceFile* sourceFile : getSourceFiles()) + { + auto content = sourceFile->getContent(); + + if (text >= content.begin() && text <= content.end()) + { + return sourceFile; + } + } + return nullptr; +} + +void SourceManager::addSourceFile(const String& uniqueIdentity, SourceFile* sourceFile) +{ + SLANG_ASSERT(!findSourceFileRecursively(uniqueIdentity)); + m_sourceFileMap.Add(uniqueIdentity, sourceFile); +} + +HumaneSourceLoc SourceManager::getHumaneLoc(SourceLoc loc, SourceLocType type) +{ + SourceView* sourceView = findSourceViewRecursively(loc); + if (sourceView) + { + return sourceView->getHumaneLoc(loc, type); + } + else + { + return HumaneSourceLoc(); + } +} + +PathInfo SourceManager::getPathInfo(SourceLoc loc, SourceLocType type) +{ + SourceView* sourceView = findSourceViewRecursively(loc); + if (sourceView) + { + return sourceView->getPathInfo(loc, type); + } + else + { + return PathInfo::makeUnknown(); + } +} + +} // namespace Slang diff --git a/source/compiler-core/slang-source-loc.h b/source/compiler-core/slang-source-loc.h new file mode 100644 index 000000000..54811918f --- /dev/null +++ b/source/compiler-core/slang-source-loc.h @@ -0,0 +1,487 @@ +// slang-source-loc.h +#ifndef SLANG_SOURCE_LOC_H_INCLUDED +#define SLANG_SOURCE_LOC_H_INCLUDED + +#include "../core/slang-basic.h" +#include "../core/slang-memory-arena.h" +#include "../core/slang-string-slice-pool.h" + +#include "../../slang-com-ptr.h" +#include "../../slang.h" + +namespace Slang { + +/** Overview: + +There needs to be a mechanism where we can easily and quickly track a specific locations in any source file used during a compilation. +This is important because that original location is meaningful to the user as it relates to their original source. Thus SourceLoc are +used so we can display meaningful and accurate errors/warnings as well as being able to always map generated code locations back to their origins. + +A 'SourceLoc' along with associated structures (SourceView, SourceFile, SourceMangager) this can pinpoint the location down to the byte across the +compilation. This could be achieved by storing for every token and instruction the file, line and column number came from. The SourceLoc is used in +lots of places - every AST node, every Token from the lexer, every IRInst - so we really want to make it small. So for this reason we actually +encode SourceLoc as a single integer and then use the associated structures when needed to determine what the location actually refers to - +the source file, line and column number, or in effect the byte in the original file. + +Unfortunately there is extra complications. When a source is parsed it's interpretation (in terms of how a piece of source maps to an 'original' file etc) +can be overridden - for example by using #line directives. Moreover a single source file can be parsed multiple times. When it's parsed multiple times the +interpretation of the mapping (#line directives for example) can change. This is the purpose of the SourceView - it holds the interpretation of a source file +for a specific Lex/Parse. + +Another complication is that not all 'source' comes from SourceFiles, a macro expansion, may generate new 'source' we need to handle this, but also be able +to have a SourceLoc map to the expansion unambiguously. This is handled by creating a SourceFile and SourceView that holds only the macro generated +specific information. + +SourceFile - Is the immutable text contents of a file (or perhaps some generated source - say from doing a macro substitution) +SourceView - Tracks a single parse of a SourceFile. Each SourceView defines a range of source locations used. If a SourceFile is parsed twice, two +SourceViews are created, with unique SourceRanges. This is so that it is possible to tell which specific parse a SourceLoc is from - and so know the right +interpretation for that lex/parse. +*/ + +struct PathInfo +{ + typedef PathInfo ThisType; + + /// To be more rigorous about where a path comes from, the type identifies what a paths origin is + enum class Type : uint8_t + { + Unknown, ///< The path is not known + Normal, ///< Normal has both path and uniqueIdentity + FoundPath, ///< Just has a found path (uniqueIdentity is unknown, or even 'unknowable') + FromString, ///< Created from a string (so found path might not be defined and should not be taken as to map to a loaded file) + TokenPaste, ///< No paths, just created to do a macro expansion + TypeParse, ///< No path, just created to do a type parse + CommandLine, ///< A macro constructed from the command line + }; + + /// True if has a canonical path + SLANG_FORCE_INLINE bool hasUniqueIdentity() const { return type == Type::Normal && uniqueIdentity.getLength() > 0; } + /// True if has a regular found path + SLANG_FORCE_INLINE bool hasFoundPath() const { return type == Type::Normal || type == Type::FoundPath || (type == Type::FromString && foundPath.getLength() > 0); } + /// True if has a found path that has originated from a file (as opposed to string or some other origin) + SLANG_FORCE_INLINE bool hasFileFoundPath() const { return (type == Type::Normal || type == Type::FoundPath) && foundPath.getLength() > 0; } + + bool operator==(const ThisType& rhs) const; + bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } + + /// Returns the 'most unique' identity for the path. If has a 'uniqueIdentity' returns that, else the foundPath, else "". + const String getMostUniqueIdentity() const; + + /// Append to out, how to display the path + void appendDisplayName(StringBuilder& out) const; + + // So simplify construction. In normal usage it's safer to use make methods over constructing directly. + static PathInfo makeUnknown() { return PathInfo { Type::Unknown, String(), String() }; } + static PathInfo makeTokenPaste() { return PathInfo{ Type::TokenPaste, "token paste", String()}; } + static PathInfo makeNormal(const String& foundPathIn, const String& uniqueIdentity) { SLANG_ASSERT(uniqueIdentity.getLength() > 0 && foundPathIn.getLength() > 0); return PathInfo { Type::Normal, foundPathIn, uniqueIdentity }; } + static PathInfo makePath(const String& pathIn) { SLANG_ASSERT(pathIn.getLength() > 0); return PathInfo { Type::FoundPath, pathIn, String()}; } + static PathInfo makeTypeParse() { return PathInfo { Type::TypeParse, "type string", String() }; } + static PathInfo makeCommandLine() { return PathInfo { Type::CommandLine, "command line", String() }; } + static PathInfo makeFromString(const String& userPath) { return PathInfo{ Type::FromString, userPath, String() }; } + + Type type; ///< The type of path + String foundPath; ///< The path where the file was found (might contain relative elements) + String uniqueIdentity; ///< The unique identity of the file on the path found +}; + +class SourceLoc +{ +public: + typedef SourceLoc ThisType; + typedef uint32_t RawValue; + +private: + RawValue raw; + +public: + SourceLoc() + : raw(0) + {} + + SourceLoc( + SourceLoc const& loc) + : raw(loc.raw) + {} + + SLANG_FORCE_INLINE bool operator==(const ThisType& rhs) const { return raw == rhs.raw; } + SLANG_FORCE_INLINE bool operator!=(const ThisType& rhs) const { return !(raw == rhs.raw); } + + RawValue getRaw() const { return raw; } + void setRaw(RawValue value) { raw = value; } + + static SourceLoc fromRaw(RawValue value) + { + SourceLoc result; + result.setRaw(value); + return result; + } + + bool isValid() const + { + return raw != 0; + } + SourceLoc& operator=(const ThisType& rhs) = default; +}; + +inline SourceLoc operator+(SourceLoc loc, Int offset) +{ + return SourceLoc::fromRaw(SourceLoc::RawValue(Int(loc.getRaw()) + offset)); +} + +// A range of locations in the input source +struct SourceRange +{ + /// True if the loc is in the range. Range is inclusive on begin to end. + bool contains(SourceLoc loc) const { const auto rawLoc = loc.getRaw(); return rawLoc >= begin.getRaw() && rawLoc <= end.getRaw(); } + /// Get the total size + UInt getSize() const { return UInt(end.getRaw() - begin.getRaw()); } + + /// Get the offset of a loc in this range + int getOffset(SourceLoc loc) const { SLANG_ASSERT(contains(loc)); return int(loc.getRaw() - begin.getRaw()); } + + /// Convert an offset to a loc + SourceLoc getSourceLocFromOffset(uint32_t offset) const { SLANG_ASSERT(offset <= getSize()); return begin + Int(offset); } + + SourceRange() + {} + + SourceRange(SourceLoc loc) + : begin(loc) + , end(loc) + {} + + SourceRange(SourceLoc begin, SourceLoc end) + : begin(begin) + , end(end) + {} + + SourceLoc begin; + SourceLoc end; +}; + + +// Pre-declare +struct SourceManager; + +// A logical or physical storage object for a range of input code +// that has logically contiguous source locations. +class SourceFile +{ +public: + + struct OffsetRange + { + /// We need a value to indicate an invalid range. We can't use 0 as that is valid for an offset range + /// We can't use a negative number, and don't want to make signed so we get the full 32-bits. + /// So we just use the max value as invalid + static const uint32_t kInvalid = 0xffffffff; + + /// True if the range is valid + SLANG_FORCE_INLINE bool isValid() const { return end >= start && start != kInvalid; } + /// True if offset is within range (inclusively) + SLANG_FORCE_INLINE bool containsInclusive(uint32_t offset) const { return offset >= start && offset <= end; } + + /// Get the count + SLANG_FORCE_INLINE uint32_t getCount() const { return end - start; } + + /// Return an invalid range. + static OffsetRange makeInvalid() { return OffsetRange{ kInvalid, kInvalid }; } + + uint32_t start; + uint32_t end; + }; + + /// Returns the line break offsets (in bytes from start of content) + /// Note that this is lazily evaluated - the line breaks are only calculated on the first request + const List& getLineBreakOffsets(); + + /// Returns true if the offset is on the specified line + /// NOTE! If offsets are not fully setup (because we don't have source), will only be correct for lines that have offsets + bool isOffsetOnLine(uint32_t offset, Index lineIndex); + + /// Get the line containing the offset. Requires that content is available, else will return an empty slice. + UnownedStringSlice getLineContainingOffset(uint32_t offset); + + /// Get the line at the specified line index. Requires that content is available, else will return an empty slice. + UnownedStringSlice getLineAtIndex(Index lineIndex); + + /// Get the offset range at the specified line index. Works without content. + OffsetRange getOffsetRangeAtLineIndex(Index lineIndex); + + /// Set the line break offsets + void setLineBreakOffsets(const uint32_t* offsets, UInt numOffsets); + + /// Calculate the line based on the offset + int calcLineIndexFromOffset(int offset); + + /// Calculate the offset for a line + int calcColumnIndex(int line, int offset); + + /// Get the content holding blob + ISlangBlob* getContentBlob() const { return m_contentBlob; } + + /// True if has full set content + bool hasContent() const { return m_contentBlob != nullptr; } + + /// Get the content size + size_t getContentSize() const { return m_contentSize; } + + /// Get the content + const UnownedStringSlice& getContent() const { return m_content; } + + /// Get path info + const PathInfo& getPathInfo() const { return m_pathInfo; } + + /// Set the content as a blob + void setContents(ISlangBlob* blob); + /// Set the content as a string + void setContents(const String& content); + + /// Calculate a display path -> can canonicalize if necessary + String calcVerbosePath() const; + + /// Get the source manager this was created on + SourceManager* getSourceManager() const { return m_sourceManager; } + + /// Ctor + SourceFile(SourceManager* sourceManager, const PathInfo& pathInfo, size_t contentSize); + /// Dtor + ~SourceFile(); + + protected: + + SourceManager* m_sourceManager; ///< The source manager this belongs to + PathInfo m_pathInfo; ///< The path The logical file path to report for locations inside this span. + ComPtr m_contentBlob; ///< A blob that owns the storage for the file contents. If nullptr, there is no contents + UnownedStringSlice m_content; ///< The actual contents of the file. + size_t m_contentSize; ///< The size of the actual contents + + // In order to speed up lookup of line number information, + // we will cache the starting offset of each line break in + // the input file: + List m_lineBreakOffsets; +}; + +enum class SourceLocType +{ + Nominal, ///< The normal interpretation which takes into account #line directives + Actual, ///< Ignores #line directives - and is the location as seen in the actual file +}; + +// A source location in a format a human might like to see +struct HumaneSourceLoc +{ + PathInfo pathInfo = PathInfo::makeUnknown(); + Int line = 0; + Int column = 0; +}; + + +/* A SourceView maps to a single span of SourceLoc range and is equivalent to a single include or more precisely use of a source file. +It is distinct from a SourceFile - because a SourceFile may be included multiple times, with different interpretations (depending +on #defines for example). +*/ +class SourceView +{ + public: + + // Each entry represents some contiguous span of locations that + // all map to the same logical file. + struct Entry + { + /// True if this resets the line numbering. It is distinct from a m_lineAdjust being 0, because it also means the path returns to the default. + bool isDefault() const { return m_pathHandle == StringSlicePool::Handle(0); } + + SourceLoc m_startLoc; ///< Where does this entry begin? + StringSlicePool::Handle m_pathHandle; ///< What is the presumed path for this entry. If 0 it means there is no path. + int32_t m_lineAdjust; ///< Adjustment to apply to source line numbers when printing presumed locations. Relative to the line number in the underlying file. + }; + + /// Given a sourceLoc finds the entry associated with it. If returns -1 then no entry is + /// associated with this location, and therefore the location should be interpreted as an offset + /// into the underlying sourceFile. + int findEntryIndex(SourceLoc sourceLoc) const; + + /// Add a line directive for this view. The directiveLoc must of course be in this SourceView + /// The path handle, must have been constructed on the SourceManager associated with the view + /// NOTE! Directives are assumed to be added IN ORDER during parsing such that every directiveLoc > previous + void addLineDirective(SourceLoc directiveLoc, StringSlicePool::Handle pathHandle, int line); + void addLineDirective(SourceLoc directiveLoc, const String& path, int line); + + /// Removes any corrections on line numbers and reverts to the source files path + void addDefaultLineDirective(SourceLoc directiveLoc); + + /// Get the range that this view applies to + const SourceRange& getRange() const { return m_range; } + /// Get the entries + const List& getEntries() const { return m_entries; } + /// Set the entries list + void setEntries(const Entry* entries, UInt numEntries) { m_entries.clear(); m_entries.addRange(entries, numEntries); } + + /// Get the source file holds the contents this view + SourceFile* getSourceFile() const { return m_sourceFile; } + /// Get the source manager + SourceManager* getSourceManager() const { return m_sourceFile->getSourceManager(); } + + /// Get the associated 'content' (the source text) + const UnownedStringSlice& getContent() const { return m_sourceFile->getContent(); } + + /// Get the size of the content + size_t getContentSize() const { return m_sourceFile->getContentSize(); } + + /// Get the humane location + /// Type determines if the location wanted is the original, or the 'normal' (which modifys behavior based on #line directives) + HumaneSourceLoc getHumaneLoc(SourceLoc loc, SourceLocType type = SourceLocType::Nominal); + + /// Get the path associated with a location + PathInfo getPathInfo(SourceLoc loc, SourceLocType type = SourceLocType::Nominal); + + /// Get the initiating source location - that is the source location that caused the this SourceView to be created + /// Can be SourceLoc(0) if there is no initiating location. + /// For example for a #include - the view's initiating source loc for the view that is the contents of the view + /// will be the location of the #include in the source. + /// For the original source file (ie not an include) - the view will have an initiating source loc of SourceLoc(0) + SourceLoc getInitiatingSourceLoc() const { return m_initiatingSourceLoc; } + + /// Ctor + SourceView(SourceFile* sourceFile, SourceRange range, const String* viewPath, SourceLoc initiatingSourceLoc): + m_range(range), + m_sourceFile(sourceFile), + m_initiatingSourceLoc(initiatingSourceLoc) + { + if (viewPath) + { + m_viewPath = *viewPath; + } + } + + protected: + /// Get the pathInfo from a string handle. If it's 0, it will return the _getPathInfo + PathInfo _getPathInfoFromHandle(StringSlicePool::Handle pathHandle) const; + /// Gets the pathInfo for this view. It may be different from the m_sourceFile's if the path has been + /// overridden by m_viewPath + PathInfo _getPathInfo() const; + + String m_viewPath; ///< Path to this view. If empty the path is the path to the SourceView + + SourceLoc m_initiatingSourceLoc; ///< An optional source loc that defines where this view was initiated from. SourceLoc(0) if not defined. + + SourceRange m_range; ///< The range that this SourceView applies to + SourceFile* m_sourceFile; ///< The source file. Can hold the line breaks + List m_entries; ///< An array entries describing how we should interpret a range, starting from the start location. +}; + +struct SourceManager +{ + // Initialize a source manager, with an optional parent + void initialize(SourceManager* parent, ISlangFileSystemExt* fileSystemExt); + + /// Allocate a range of SourceLoc locations, these can be used to identify a specific location in the source + SourceRange allocateSourceRange(UInt size); + + /// Create a SourceFile defined with the specified path, and content held within a blob + SourceFile* createSourceFileWithSize(const PathInfo& pathInfo, size_t contentSize); + SourceFile* createSourceFileWithString(const PathInfo& pathInfo, const String& contents); + SourceFile* createSourceFileWithBlob(const PathInfo& pathInfo, ISlangBlob* blob); + + /// Get the humane source location + HumaneSourceLoc getHumaneLoc(SourceLoc loc, SourceLocType type = SourceLocType::Nominal); + + /// Get the path associated with a location + PathInfo getPathInfo(SourceLoc loc, SourceLocType type = SourceLocType::Nominal); + + /// Create a new source view from a file + /// @param sourceFile is the source file that contains the source + /// @param pathInfo is path used to read the file from + /// @param initiatingSourceLoc the (optional) location in the source that led the the creation of this view. If there isn't an initiating source location pass SourceLoc(0)s + SourceView* createSourceView(SourceFile* sourceFile, const PathInfo* pathInfo, SourceLoc initiatingSourceLoc); + + /// Find a view by a source file location. + /// If not found in this manager will look in the parent SourceManager + /// Returns nullptr if not found. + SourceView* findSourceViewRecursively(SourceLoc loc) const; + + /// Find the SourceView associated with this manager for a specified location + /// Returns nullptr if not found. + SourceView* findSourceView(SourceLoc loc) const; + + /// Searches this manager, and then the parent to see if can find a match for path. + /// If not found returns nullptr. + SourceFile* findSourceFileRecursively(const String& uniqueIdentity) const; + /// Find if the source file is defined on this manager. + SourceFile* findSourceFile(const String& uniqueIdentity) const; + + /// Searches this manager, and then the parent to see if can find a match + SourceFile* findSourceFileByContentRecursively(const char* text); + /// Find the source file that contains *the memory* text points to. + SourceFile* findSourceFileByContent(const char* text) const; + + /// Get the file system associated with this source manager + ISlangFileSystemExt* getFileSystemExt() const { return m_fileSystemExt; } + /// Get the file system associated with this source manager + void setFileSystemExt(ISlangFileSystemExt* fileSystemExt) { m_fileSystemExt = fileSystemExt; } + + /// Add a source file, uniqueIdentity must be unique for this manager AND any parents + void addSourceFile(const String& uniqueIdentity, SourceFile* sourceFile); + + /// Get the slice pool + StringSlicePool& getStringSlicePool() { return m_slicePool; } + + /// Get the source range for just this manager + /// Caution - the range will change if allocations are made to this manager. + SourceRange getSourceRange() const { return SourceRange(m_startLoc, m_nextLoc); } + + /// Get the parent manager to this manager. Returns nullptr if there isn't any. + SourceManager* getParent() const { return m_parent; } + + /// A memory arena to hold allocations that are in scope for the same time as SourceManager + MemoryArena* getMemoryArena() { return &m_memoryArena; } + + /// Allocate a string slice + UnownedStringSlice allocateStringSlice(const UnownedStringSlice& slice); + + /// Get all of the source files + const List& getSourceFiles() const { return m_sourceFiles; } + + /// Get the source views + const List& getSourceViews() const { return m_sourceViews; } + + SourceManager() : + m_memoryArena(2048), + m_slicePool(StringSlicePool::Style::Default) + {} + ~SourceManager(); + + protected: + + // The first location available to this source manager + // (may not be the first location of all, because we might + // have a parent source manager) + SourceLoc m_startLoc; + + // The "parent" source manager that owns locations ahead of `startLoc` + SourceManager* m_parent = nullptr; + + // The location to be used by the next source file to be loaded + SourceLoc m_nextLoc; + + // All of the SourceViews constructed on this SourceManager. These are held in increasing order of range, so can find by doing a binary chop. + List m_sourceViews; + // All of the SourceFiles constructed on this SourceManager. This owns the SourceFile. + List m_sourceFiles; + + StringSlicePool m_slicePool; + + // Memory arena that can be used for holding data to held in scope as long as the Source is + // Can be used for storing the decoded contents of Token. Content for example. + MemoryArena m_memoryArena; + + // Maps uniqueIdentities to source files + Dictionary m_sourceFileMap; + + ComPtr m_fileSystemExt; +}; + +} // namespace Slang + +#endif diff --git a/source/compiler-core/slang-token-defs.h b/source/compiler-core/slang-token-defs.h new file mode 100644 index 000000000..6cece330e --- /dev/null +++ b/source/compiler-core/slang-token-defs.h @@ -0,0 +1,96 @@ +// slang-token-defs.h + +// This file is meant to be included multiple times, to produce different +// pieces of code related to tokens +// +// Each token is declared here with: +// +// TOKEN(id, desc) +// +// where `id` is the identifier that will be used for the token in +// ordinary code, while `desc` is name we should print when +// referring to this token in diagnostic messages. + + +#ifndef TOKEN +#error Need to define TOKEN(ID, DESC) before including "token-defs.h" +#endif + +TOKEN(Unknown, "") +TOKEN(EndOfFile, "end of file") +TOKEN(EndOfDirective, "end of line") +TOKEN(Invalid, "invalid character") +TOKEN(Identifier, "identifier") +TOKEN(IntegerLiteral, "integer literal") +TOKEN(FloatingPointLiteral, "floating-point literal") +TOKEN(StringLiteral, "string literal") +TOKEN(CharLiteral, "character literal") +TOKEN(WhiteSpace, "whitespace") +TOKEN(NewLine, "newline") +TOKEN(LineComment, "line comment") +TOKEN(BlockComment, "block comment") +TOKEN(DirectiveMessage, "user-defined message") + +#define PUNCTUATION(id, text) \ + TOKEN(id, "'" text "'") + +PUNCTUATION(Semicolon, ";") +PUNCTUATION(Comma, ",") +PUNCTUATION(Dot, ".") + +PUNCTUATION(LBrace, "{") +PUNCTUATION(RBrace, "}") +PUNCTUATION(LBracket, "[") +PUNCTUATION(RBracket, "]") +PUNCTUATION(LParent, "(") +PUNCTUATION(RParent, ")") + +PUNCTUATION(OpAssign, "=") +PUNCTUATION(OpAdd, "+") +PUNCTUATION(OpSub, "-") +PUNCTUATION(OpMul, "*") +PUNCTUATION(OpDiv, "/") +PUNCTUATION(OpMod, "%") +PUNCTUATION(OpNot, "!") +PUNCTUATION(OpBitNot, "~") +PUNCTUATION(OpLsh, "<<") +PUNCTUATION(OpRsh, ">>") +PUNCTUATION(OpEql, "==") +PUNCTUATION(OpNeq, "!=") +PUNCTUATION(OpGreater, ">") +PUNCTUATION(OpLess, "<") +PUNCTUATION(OpGeq, ">=") +PUNCTUATION(OpLeq, "<=") +PUNCTUATION(OpAnd, "&&") +PUNCTUATION(OpOr, "||") +PUNCTUATION(OpBitAnd, "&") +PUNCTUATION(OpBitOr, "|") +PUNCTUATION(OpBitXor, "^") +PUNCTUATION(OpInc, "++") +PUNCTUATION(OpDec, "--") + +PUNCTUATION(OpAddAssign, "+=") +PUNCTUATION(OpSubAssign, "-=") +PUNCTUATION(OpMulAssign, "*=") +PUNCTUATION(OpDivAssign, "/=") +PUNCTUATION(OpModAssign, "%=") +PUNCTUATION(OpShlAssign, "<<=") +PUNCTUATION(OpShrAssign, ">>=") +PUNCTUATION(OpAndAssign, "&=") +PUNCTUATION(OpOrAssign, "|=") +PUNCTUATION(OpXorAssign, "^=") + +PUNCTUATION(QuestionMark, "?") +PUNCTUATION(Colon, ":") +PUNCTUATION(RightArrow, "->") +PUNCTUATION(At, "@") +PUNCTUATION(Dollar, "$") +PUNCTUATION(Pound, "#") +PUNCTUATION(PoundPound, "##") + +PUNCTUATION(Scope, "::") + +#undef PUNCTUATION + +// Un-define the `TOKEN` macro so that client doesn't have to +#undef TOKEN diff --git a/source/compiler-core/slang-token.cpp b/source/compiler-core/slang-token.cpp new file mode 100644 index 000000000..576b08d66 --- /dev/null +++ b/source/compiler-core/slang-token.cpp @@ -0,0 +1,21 @@ +// slang-token.cpp +#include "slang-token.h" + +//#include + +namespace Slang { + +char const* TokenTypeToString(TokenType type) +{ + switch( type ) + { + default: + SLANG_ASSERT(!"unexpected"); + return ""; + +#define TOKEN(NAME, DESC) case TokenType::NAME: return DESC; +#include "slang-token-defs.h" + } +} + +} // namespace Slang diff --git a/source/compiler-core/slang-token.h b/source/compiler-core/slang-token.h new file mode 100644 index 000000000..9697a5c2d --- /dev/null +++ b/source/compiler-core/slang-token.h @@ -0,0 +1,140 @@ +// slang-token.h +#ifndef SLANG_TOKEN_H_INCLUDED +#define SLANG_TOKEN_H_INCLUDED + +#include "../core/slang-basic.h" + +#include "slang-source-loc.h" +#include "slang-name.h" + +namespace Slang { + +class Name; + +enum class TokenType : uint8_t +{ +#define TOKEN(NAME, DESC) NAME, +#include "slang-token-defs.h" +}; + +char const* TokenTypeToString(TokenType type); + +typedef uint8_t TokenFlags; +struct TokenFlag +{ + enum Enum : TokenFlags + { + AtStartOfLine = 1 << 0, + AfterWhitespace = 1 << 1, + SuppressMacroExpansion = 1 << 2, + ScrubbingNeeded = 1 << 3, + Name = 1 << 4, ///< Determines if 'name' is set or 'chars' in the charsNameUnion + }; +}; + +class Token +{ +public: + + TokenType type = TokenType::Unknown; + TokenFlags flags = 0; + + SourceLoc loc; + uint32_t charsCount = 0; ///< Amount of characters. Is set if name or not. + + union CharsNameUnion + { + const char* chars; + Name* name; + }; + + CharsNameUnion charsNameUnion; + + bool hasContent() const { return charsCount > 0; } + Index getContentLength() const { return charsCount; } + + UnownedStringSlice getContent() const; + /// Set content + void setContent(const UnownedStringSlice& content); + + Name* getName() const; + + Name* getNameOrNull() const; + + SourceLoc getLoc() const { return loc; } + + /// Set the name + SLANG_FORCE_INLINE void setName(Name* inName); + + Token() + { + charsNameUnion.chars = nullptr; + } + + Token( + TokenType inType, + const UnownedStringSlice& inContent, + SourceLoc inLoc, + TokenFlags inFlags = 0) + : flags(inFlags) + { + SLANG_ASSERT((inFlags & TokenFlag::Name) == 0); + type = inType; + charsNameUnion.chars = inContent.begin(); + charsCount = uint32_t(inContent.getLength()); + loc = inLoc; + } + Token( + TokenType inType, + Name* name, + SourceLoc inLoc, + TokenFlags inFlags = 0) + { + SLANG_ASSERT(name); + type = inType; + flags = inFlags | TokenFlag::Name; + charsNameUnion.name = name; + charsCount = uint32_t(name->text.getLength()); + loc = inLoc; + } +}; + +// --------------------------------------------------------------------------- +SLANG_FORCE_INLINE UnownedStringSlice Token::getContent() const +{ + return (flags & TokenFlag::Name) ? charsNameUnion.name->text.getUnownedSlice() : UnownedStringSlice(charsNameUnion.chars, charsCount); +} + +// --------------------------------------------------------------------------- +SLANG_FORCE_INLINE Name* Token::getName() const +{ + return getNameOrNull(); +} + +// --------------------------------------------------------------------------- +SLANG_FORCE_INLINE Name* Token::getNameOrNull() const +{ + return (flags & TokenFlag::Name) ? charsNameUnion.name : nullptr; +} + +// --------------------------------------------------------------------------- +SLANG_FORCE_INLINE void Token::setContent(const UnownedStringSlice& content) +{ + flags &= ~TokenFlag::Name; + charsNameUnion.chars = content.begin(); + charsCount = uint32_t(content.getLength()); +} + +// --------------------------------------------------------------------------- +SLANG_FORCE_INLINE void Token::setName(Name* inName) +{ + SLANG_ASSERT(inName); + flags |= TokenFlag::Name; + charsNameUnion.name = inName; + charsCount = uint32_t(inName->text.getLength()); +} + + +} // namespace Slang + +#endif diff --git a/source/compiler-core/slang-visual-studio-compiler-util.cpp b/source/compiler-core/slang-visual-studio-compiler-util.cpp new file mode 100644 index 000000000..fd879e63f --- /dev/null +++ b/source/compiler-core/slang-visual-studio-compiler-util.cpp @@ -0,0 +1,452 @@ +// slang-visual-studio-compiler-util.cpp +#include "slang-visual-studio-compiler-util.h" + +#include "../core/slang-common.h" +#include "../../slang-com-helper.h" +#include "../core/slang-string-util.h" + +// if Visual Studio import the visual studio platform specific header +#if SLANG_VC +# include "windows/slang-win-visual-studio-util.h" +#endif + +#include "../core/slang-io.h" + +namespace Slang +{ + +/* static */ SlangResult VisualStudioCompilerUtil::calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath) +{ + SLANG_ASSERT(options.modulePath.getLength()); + + outPath.Clear(); + + switch (options.targetType) + { + case TargetType::SharedLibrary: + { + outPath << options.modulePath << ".dll"; + return SLANG_OK; + } + case TargetType::Executable: + { + outPath << options.modulePath << ".exe"; + return SLANG_OK; + } + case TargetType::Object: + { + outPath << options.modulePath << ".obj"; + return SLANG_OK; + } + default: break; + } + + return SLANG_FAIL; +} + +/* static */SlangResult VisualStudioCompilerUtil::calcCompileProducts(const CompileOptions& options, ProductFlags flags, List& outPaths) +{ + SLANG_ASSERT(options.modulePath.getLength()); + + outPaths.clear(); + + if (flags & ProductFlag::Execution) + { + StringBuilder builder; + SLANG_RETURN_ON_FAIL(calcModuleFilePath(options, builder)); + outPaths.add(builder); + } + if (flags & ProductFlag::Miscellaneous) + { + outPaths.add(options.modulePath + ".ilk"); + + if (options.targetType == TargetType::SharedLibrary) + { + outPaths.add(options.modulePath + ".exp"); + outPaths.add(options.modulePath + ".lib"); + } + } + if (flags & ProductFlag::Compile) + { + outPaths.add(options.modulePath + ".obj"); + } + if (flags & ProductFlag::Debug) + { + // TODO(JS): Could try and determine based on debug information + outPaths.add(options.modulePath + ".pdb"); + } + + return SLANG_OK; +} + +/* static */SlangResult VisualStudioCompilerUtil::calcArgs(const CompileOptions& options, CommandLine& cmdLine) +{ + SLANG_ASSERT(options.sourceContents.getLength() == 0); + SLANG_ASSERT(options.modulePath.getLength()); + + // https://docs.microsoft.com/en-us/cpp/build/reference/compiler-options-listed-alphabetically?view=vs-2019 + + cmdLine.addArg("/nologo"); + // Generate complete debugging information + cmdLine.addArg("/Zi"); + // Display full path of source files in diagnostics + cmdLine.addArg("/FC"); + + if (options.flags & CompileOptions::Flag::EnableExceptionHandling) + { + if (options.sourceLanguage == SLANG_SOURCE_LANGUAGE_CPP) + { + // https://docs.microsoft.com/en-us/cpp/build/reference/eh-exception-handling-model?view=vs-2019 + // Assumes c functions cannot throw + cmdLine.addArg("/EHsc"); + } + } + + if (options.flags & CompileOptions::Flag::Verbose) + { + // Doesn't appear to be a VS equivalent + } + + if (options.flags & CompileOptions::Flag::EnableSecurityChecks) + { + cmdLine.addArg("/GS"); + } + else + { + cmdLine.addArg("/GS-"); + } + + switch (options.debugInfoType) + { + default: + { + // Multithreaded statically linked runtime library + cmdLine.addArg("/MD"); + break; + } + case DebugInfoType::None: + { + break; + } + case DebugInfoType::Maximal: + { + // Multithreaded statically linked *debug* runtime library + cmdLine.addArg("/MDd"); + break; + } + } + + // /Fd - followed by name of the pdb file + if (options.debugInfoType != DebugInfoType::None) + { + cmdLine.addPrefixPathArg("/Fd", options.modulePath, ".pdb"); + } + + switch (options.optimizationLevel) + { + case OptimizationLevel::None: + { + // No optimization + cmdLine.addArg("/Od"); + break; + } + case OptimizationLevel::Default: + { + break; + } + case OptimizationLevel::High: + { + cmdLine.addArg("/O2"); + break; + } + case OptimizationLevel::Maximal: + { + cmdLine.addArg("/Ox"); + break; + } + default: break; + } + + switch (options.floatingPointMode) + { + case FloatingPointMode::Default: break; + case FloatingPointMode::Precise: + { + // precise is default behavior, VS also has 'strict' + // + // ```/fp:strict has behavior similar to /fp:precise, that is, the compiler preserves the source ordering and rounding properties of floating-point code when + // it generates and optimizes object code for the target machine, and observes the standard when handling special values. In addition, the program may safely + // access or modify the floating-point environment at runtime.``` + + cmdLine.addArg("/fp:precise"); + break; + } + case FloatingPointMode::Fast: + { + cmdLine.addArg("/fp:fast"); + break; + } + } + + switch (options.targetType) + { + case TargetType::SharedLibrary: + { + // Create dynamic link library + if (options.debugInfoType == DebugInfoType::None) + { + cmdLine.addArg("/LDd"); + } + else + { + cmdLine.addArg("/LD"); + } + + cmdLine.addPrefixPathArg("/Fe", options.modulePath, ".dll"); + break; + } + case TargetType::Executable: + { + cmdLine.addPrefixPathArg("/Fe", options.modulePath, ".exe"); + break; + } + default: break; + } + + // Object file specify it's location - needed if we are out + cmdLine.addPrefixPathArg("/Fo", options.modulePath, ".obj"); + + // Add defines + for (const auto& define : options.defines) + { + StringBuilder builder; + builder << "/D"; + builder << define.nameWithSig; + if (define.value.getLength()) + { + builder << "=" << define.value; + } + + cmdLine.addArg(builder); + } + + // Add includes + for (const auto& include : options.includePaths) + { + cmdLine.addArg("/I"); + cmdLine.addArg(include); + } + + // https://docs.microsoft.com/en-us/cpp/build/reference/eh-exception-handling-model?view=vs-2019 + // /Eha - Specifies the model of exception handling. (a, s, c, r are options) + + // Files to compile + for (const auto& sourceFile : options.sourceFiles) + { + cmdLine.addArg(sourceFile); + } + + // Link options (parameters past /link go to linker) + cmdLine.addArg("/link"); + + for (const auto& libPath : options.libraryPaths) + { + // Note that any escaping of the path is handled in the ProcessUtil:: + cmdLine.addPrefixPathArg("/LIBPATH:", libPath); + } + + return SLANG_OK; +} + +static SlangResult _parseSeverity(const UnownedStringSlice& in, DownstreamDiagnostics::Diagnostic::Severity& outSeverity) +{ + typedef DownstreamDiagnostics::Diagnostic::Severity Type; + + if (in == "error" || in == "fatal error") + { + outSeverity = Type::Error; + } + else if (in == "warning") + { + outSeverity = Type::Warning; + } + else if (in == "info") + { + outSeverity = Type::Info; + } + else + { + return SLANG_FAIL; + } + return SLANG_OK; +} + +static SlangResult _parseVisualStudioLine(const UnownedStringSlice& line, DownstreamDiagnostics::Diagnostic& outDiagnostic) +{ + typedef DownstreamDiagnostics::Diagnostic Diagnostic; + + UnownedStringSlice linkPrefix = UnownedStringSlice::fromLiteral("LINK :"); + if (line.startsWith(linkPrefix)) + { + outDiagnostic.stage = Diagnostic::Stage::Link; + outDiagnostic.severity = Diagnostic::Severity::Info; + + outDiagnostic.text = UnownedStringSlice(line.begin() + linkPrefix.getLength(), line.end()); + + return SLANG_OK; + } + + outDiagnostic.stage = Diagnostic::Stage::Compile; + + const char*const start = line.begin(); + const char*const end = line.end(); + + UnownedStringSlice postPath; + // Handle the path and line no + { + const char* cur = start; + + // We have to assume it is a path up to the first : that isn't part of a drive specification + + if ((end - cur > 2) && Path::isDriveSpecification(UnownedStringSlice(start, start + 2))) + { + // Skip drive spec + cur += 2; + } + + // Find the first colon after this + Index colonIndex = UnownedStringSlice(cur, end).indexOf(':'); + if (colonIndex < 0) + { + return SLANG_FAIL; + } + + // Looks like we have a line number + if (cur[colonIndex - 1] == ')') + { + const char* lineNoEnd = cur + colonIndex - 1; + const char* lineNoStart = lineNoEnd; + while (lineNoStart > start && *lineNoStart != '(') + { + lineNoStart--; + } + // Check this appears plausible + if (*lineNoStart != '(' || *lineNoEnd != ')') + { + return SLANG_FAIL; + } + Int numDigits = 0; + Int lineNo = 0; + for (const char* digitCur = lineNoStart + 1; digitCur < lineNoEnd; ++digitCur) + { + char c = *digitCur; + if (c >= '0' && c <= '9') + { + lineNo = lineNo * 10 + (c - '0'); + numDigits++; + } + else + { + return SLANG_FAIL; + } + } + if (numDigits == 0) + { + return SLANG_FAIL; + } + + outDiagnostic.filePath = UnownedStringSlice(start, lineNoStart); + outDiagnostic.fileLine = lineNo; + } + else + { + outDiagnostic.filePath = UnownedStringSlice(start, cur + colonIndex); + outDiagnostic.fileLine = 0; + } + + // Save the remaining text in 'postPath' + postPath = UnownedStringSlice(cur + colonIndex + 1, end); + } + + // Split up the error section + UnownedStringSlice postError; + { + // tests/cpp-compiler/c-compile-link-error.exe : fatal error LNK1120: 1 unresolved externals + + const Index errorColonIndex = postPath.indexOf(':'); + if (errorColonIndex < 0) + { + return SLANG_FAIL; + } + + const UnownedStringSlice errorSection = UnownedStringSlice(postPath.begin(), postPath.begin() + errorColonIndex); + Index errorCodeIndex = errorSection.lastIndexOf(' '); + if (errorCodeIndex < 0) + { + return SLANG_FAIL; + } + + // Extract the code + outDiagnostic.code = UnownedStringSlice(errorSection.begin() + errorCodeIndex + 1, errorSection.end()); + if (outDiagnostic.code.startsWith(UnownedStringSlice::fromLiteral("LNK"))) + { + outDiagnostic.stage = Diagnostic::Stage::Link; + } + + // Extract the bit before the code + SLANG_RETURN_ON_FAIL(_parseSeverity(UnownedStringSlice(errorSection.begin(), errorSection.begin() + errorCodeIndex).trim(), outDiagnostic.severity)); + + // Link codes start with LNK prefix + postError = UnownedStringSlice(postPath.begin() + errorColonIndex + 1, end); + } + + outDiagnostic.text = postError; + + return SLANG_OK; +} + +/* static */SlangResult VisualStudioCompilerUtil::parseOutput(const ExecuteResult& exeRes, DownstreamDiagnostics& outDiagnostics) +{ + outDiagnostics.reset(); + + outDiagnostics.rawDiagnostics = exeRes.standardOutput; + + for (auto line : LineParser(exeRes.standardOutput.getUnownedSlice())) + { +#if 0 + fwrite(line.begin(), 1, line.size(), stdout); + fprintf(stdout, "\n"); +#endif + + Diagnostic diagnostic; + if (SLANG_SUCCEEDED(_parseVisualStudioLine(line, diagnostic))) + { + outDiagnostics.diagnostics.add(diagnostic); + } + } + + // if it has a compilation error.. set on output + if (outDiagnostics.has(Diagnostic::Severity::Error)) + { + outDiagnostics.result = SLANG_FAIL; + } + + return SLANG_OK; +} + +/* static */SlangResult VisualStudioCompilerUtil::locateCompilers(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set) +{ + SLANG_UNUSED(loader); + + // TODO(JS): We don't support fixed path for visual studio just yet + if (path.getLength() == 0) + { +#if SLANG_VC + return WinVisualStudioUtil::find(set); +#endif + } + + return SLANG_OK; +} + +} diff --git a/source/compiler-core/slang-visual-studio-compiler-util.h b/source/compiler-core/slang-visual-studio-compiler-util.h new file mode 100644 index 000000000..018dde212 --- /dev/null +++ b/source/compiler-core/slang-visual-studio-compiler-util.h @@ -0,0 +1,42 @@ +#ifndef SLANG_VISUAL_STUDIO_COMPILER_UTIL_H +#define SLANG_VISUAL_STUDIO_COMPILER_UTIL_H + +#include "slang-downstream-compiler.h" + +namespace Slang +{ + + +struct VisualStudioCompilerUtil : public DownstreamCompilerBaseUtil +{ + /// Calculate Visual Studio family compilers cmdLine arguments from options + static SlangResult calcArgs(const CompileOptions& options, CommandLine& cmdLine); + /// Parse Visual Studio exeRes into CPPCompiler::Output + static SlangResult parseOutput(const ExecuteResult& exeRes, DownstreamDiagnostics& outOutput); + + static SlangResult calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath); + + static SlangResult calcCompileProducts(const CompileOptions& options, ProductFlags flags, List& outPaths); + + static SlangResult locateCompilers(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set); +}; + +class VisualStudioDownstreamCompiler : public CommandLineDownstreamCompiler +{ +public: + typedef CommandLineDownstreamCompiler Super; + typedef VisualStudioCompilerUtil Util; + + // CommandLineDownstreamCompiler impl - just forwards to the Util + virtual SlangResult calcArgs(const CompileOptions& options, CommandLine& cmdLine) SLANG_OVERRIDE { return Util::calcArgs(options, cmdLine); } + virtual SlangResult parseOutput(const ExecuteResult& exeResult, DownstreamDiagnostics& output) SLANG_OVERRIDE { return Util::parseOutput(exeResult, output); } + virtual SlangResult calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath) SLANG_OVERRIDE { return Util::calcModuleFilePath(options, outPath); } + virtual SlangResult calcCompileProducts(const CompileOptions& options, ProductFlags productFlags, List& outPaths) SLANG_OVERRIDE { return Util::calcCompileProducts(options, productFlags, outPaths); } + + VisualStudioDownstreamCompiler(const Desc& desc):Super(desc) {} +}; + + +} + +#endif diff --git a/source/compiler-core/windows/slang-win-visual-studio-util.cpp b/source/compiler-core/windows/slang-win-visual-studio-util.cpp new file mode 100644 index 000000000..0dc8ea764 --- /dev/null +++ b/source/compiler-core/windows/slang-win-visual-studio-util.cpp @@ -0,0 +1,366 @@ +#include "slang-win-visual-studio-util.h" + +#include "../../core/slang-common.h" +#include "../../core/slang-process-util.h" +#include "../../core/slang-string-util.h" + +#include "../slang-visual-studio-compiler-util.h" + +#ifdef _WIN32 +# define WIN32_LEAN_AND_MEAN +# define NOMINMAX +# include +# undef WIN32_LEAN_AND_MEAN +# undef NOMINMAX + +# include + +#endif + +// The method used to invoke VS was originally inspired by some ideas in +// https://github.com/RuntimeCompiledCPlusPlus/RuntimeCompiledCPlusPlus/ + +namespace Slang { + +// Information on VS versioning can be found here +// https://en.wikipedia.org/wiki/Microsoft_Visual_C%2B%2B#Internal_version_numbering + + +namespace { // anonymous + +typedef WinVisualStudioUtil::Version Version; + +struct RegistryInfo +{ + const char* regName; ///< The name of the entry in the registry + const char* pathFix; ///< With the value from the registry how to fix the path +}; + +struct VersionInfo +{ + Version version; ///< The version + const char* name; ///< The name of the registry key +}; + +} // anonymous + +static SlangResult _readRegistryKey(const char* path, const char* keyName, String& outString) +{ + // https://docs.microsoft.com/en-us/windows/desktop/api/winreg/nf-winreg-regopenkeyexa + HKEY key; + LONG ret = RegOpenKeyExA(HKEY_LOCAL_MACHINE, path, 0, KEY_READ | KEY_WOW64_32KEY, &key); + if (ret != ERROR_SUCCESS) + { + return SLANG_FAIL; + } + + char value[MAX_PATH]; + DWORD size = MAX_PATH; + + // https://docs.microsoft.com/en-us/windows/desktop/api/winreg/nf-winreg-regqueryvalueexa + ret = RegQueryValueExA(key, keyName, nullptr, nullptr, (LPBYTE)value, &size); + RegCloseKey(key); + + if (ret != ERROR_SUCCESS) + { + return SLANG_FAIL; + } + + outString = value; + return SLANG_OK; +} + +// Make easier to set up the array +static Version _makeVersion(int main, int dot = 0) { return WinVisualStudioUtil::makeVersion(main, dot); } + +VersionInfo _makeVersionInfo(const char* name, int high, int dot = 0) +{ + VersionInfo info; + info.name = name; + info.version = WinVisualStudioUtil::makeVersion(high, dot); + return info; +} + +static const VersionInfo s_versionInfos[] = +{ + _makeVersionInfo("VS 2005", 8), + _makeVersionInfo("VS 2008", 9), + _makeVersionInfo("VS 2010", 10), + _makeVersionInfo("VS 2012", 11), + _makeVersionInfo("VS 2013", 12), + _makeVersionInfo("VS 2015", 14), + _makeVersionInfo("VS 2017", 15), + _makeVersionInfo("VS 2019", 16), +}; + +// When trying to figure out how this stuff works by running regedit - care is needed, +// because what regedit displays varies on which version of regedit is used. +// In order to use the registry paths used here it's necessary to use Start/Run with +// %systemroot%\syswow64\regedit to view 32 bit keys + +static const RegistryInfo s_regInfos[] = +{ + {"SOFTWARE\\Microsoft\\VisualStudio\\SxS\\VC7", "" }, + {"SOFTWARE\\Microsoft\\VisualStudio\\SxS\\VS7", "VC\\Auxiliary\\Build\\" }, +}; + +static bool _canUseVSWhere(Version version) +{ + // If greater than 15.0 we can use vswhere tool + return (int(version) >= int(_makeVersion(15))); +} + +static int _getRegistryKeyIndex(Version version) +{ + if (int(version) >= int(_makeVersion(15))) + { + return 1; + } + return 0; +} + +/* static */void WinVisualStudioUtil::getVersions(List& outVersions) +{ + const int count = SLANG_COUNT_OF(s_versionInfos); + outVersions.setCount(count); + + Version* dst = outVersions.begin(); + for (int i = 0; i < count; ++i) + { + dst[i] = s_versionInfos[i].version; + } +} + +/* static */WinVisualStudioUtil::Version WinVisualStudioUtil::getCompiledVersion() +{ + // Get the version of visual studio used to compile this source + const uint32_t version = _MSC_VER; + + switch (version) + { + case 1400: return _makeVersion(8); + case 1500: return _makeVersion(9); + case 1600: return _makeVersion(10); + case 1700: return _makeVersion(11); + case 1800: return _makeVersion(12); + case 1900: + { + return _makeVersion(14); + } + case 1911: + case 1912: + case 1913: + case 1914: + case 1915: + case 1916: + { + return _makeVersion(15); + } + case 1920: + { + return _makeVersion(16); + } + default: + { + int lastKnownVersion = 1920; + if (version > lastKnownVersion) + { + // Its an unknown newer version + return Version::Future; + } + break; + } + } + + // Unknown version + return Version::Unknown; +} + +static SlangResult _find(int versionIndex, WinVisualStudioUtil::VersionPath& outPath) +{ + const auto& versionInfo = s_versionInfos[versionIndex]; + + auto version = versionInfo.version; + + outPath.version = version; + outPath.vcvarsPath = String(); + + if (_canUseVSWhere(version)) + { + CommandLine cmd; + + // Lookup directly %ProgramFiles(x86)% path + // https://docs.microsoft.com/en-us/windows/desktop/api/shlobj_core/nf-shlobj_core-shgetfolderpatha + HWND hwnd = GetConsoleWindow(); + + char programFilesPath[_MAX_PATH]; + SHGetFolderPathA(hwnd, CSIDL_PROGRAM_FILESX86, NULL, 0, programFilesPath); + + String vswherePath = programFilesPath; + vswherePath.append("\\Microsoft Visual Studio\\Installer\\vswhere"); + + cmd.setExecutableFilename(vswherePath); + + StringBuilder versionName; + WinVisualStudioUtil::append(version, versionName); + + String args[] = { "-version", versionName, "-requires", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", "-property", "installationPath" }; + cmd.addArgs(args, SLANG_COUNT_OF(args)); + + ExecuteResult exeRes; + if (SLANG_SUCCEEDED(ProcessUtil::execute(cmd, exeRes))) + { + // We need to chopoff CR/LF if there is one + List lines; + StringUtil::calcLines(exeRes.standardOutput.getUnownedSlice(), lines); + + if (lines.getCount()) + { + outPath.vcvarsPath = lines[0]; + outPath.vcvarsPath.append("\\VC\\Auxiliary\\Build\\"); + return SLANG_OK; + } + } + } + + const Int keyIndex = _getRegistryKeyIndex(version); + if (keyIndex >= 0) + { + SLANG_ASSERT(keyIndex < SLANG_COUNT_OF(s_regInfos)); + + // Try reading the key + const auto& keyInfo = s_regInfos[keyIndex]; + + StringBuilder keyName; + WinVisualStudioUtil::append(versionInfo.version, keyName); + + String value; + if (SLANG_SUCCEEDED(_readRegistryKey(keyInfo.regName, keyName.getBuffer(), value))) + { + outPath.vcvarsPath = value; + return SLANG_OK; + } + } + + return SLANG_FAIL; +} + +/* static */SlangResult WinVisualStudioUtil::find(List& outVersionPaths) +{ + outVersionPaths.clear(); + + const int versionCount = SLANG_COUNT_OF(s_versionInfos); + + for (int i = versionCount - 1; i >= 0; --i) + { + VersionPath versionPath; + if (SLANG_SUCCEEDED(_find(i, versionPath))) + { + outVersionPaths.add(versionPath); + } + } + + return SLANG_OK; +} + +/* static */SlangResult WinVisualStudioUtil::find(Version version, VersionPath& outPath) +{ + const int versionCount = SLANG_COUNT_OF(s_versionInfos); + + for (int i = 0; i < versionCount; ++i) + { + const auto& versionInfo = s_versionInfos[i]; + if (versionInfo.version == version) + { + return _find(i, outPath); + } + } + return SLANG_FAIL; +} + +/* static */SlangResult WinVisualStudioUtil::find(DownstreamCompilerSet* set) +{ + const int versionCount = SLANG_COUNT_OF(s_versionInfos); + + for (int i = versionCount - 1; i >= 0; --i) + { + const auto& versionInfo = s_versionInfos[i]; + auto desc = getDesc(versionInfo.version); + + VersionPath versionPath; + if (!set->getCompiler(desc) && SLANG_SUCCEEDED(_find(i, versionPath))) + { + RefPtr compiler = new VisualStudioDownstreamCompiler(desc); + calcExecuteCompilerArgs(versionPath, compiler->m_cmdLine); + set->addCompiler(compiler); + } + } + + return SLANG_OK; +} + +/* static */void WinVisualStudioUtil::calcExecuteCompilerArgs(const VersionPath& versionPath, CommandLine& outCmdLine) +{ + // To invoke cl we need to run the suitable vcvars. In order to run this we have to have MS CommandLine. + // So here we build up a cl command line that is run by first running vcvars, and then executing cl with the parameters as passed to commandLine + + CommandLine cmdLine; + + cmdLine.setExecutableFilename("cmd.exe"); + { + String options[] = { "/q", "/c", "@prompt", "$" }; + cmdLine.addArgs(options, SLANG_COUNT_OF(options)); + } + + cmdLine.addArg("&&"); + + { + StringBuilder path; + path << versionPath.vcvarsPath; + path << "\\vcvarsall.bat"; + cmdLine.addArg(path); + } + +#if SLANG_PTR_IS_32 + cmdLine.addArg("x86"); +#else + cmdLine.addArg("x86_amd64"); +#endif + + cmdLine.addArg("&&"); + cmdLine.addArg("cl"); + + outCmdLine = cmdLine; +} + +/* static */SlangResult WinVisualStudioUtil::executeCompiler(const VersionPath& versionPath, const CommandLine& commandLine, ExecuteResult& outResult) +{ + CommandLine cmdLine; + calcExecuteCompilerArgs(versionPath, cmdLine); + // Append the command line options + cmdLine.addArgs(commandLine.m_args.getBuffer(), commandLine.m_args.getCount()); + return ProcessUtil::execute(cmdLine, outResult); +} + +/* static */void WinVisualStudioUtil::append(Version version, StringBuilder& outBuilder) +{ + switch (version) + { + case Version::Unknown: + { + outBuilder << "unknown"; + } + case Version::Future: + { + outBuilder << "future"; + break; + } + default: + { + outBuilder << (int(version) / 10) << "." << (int(version) % 10); + break; + } + } +} + +} // namespace Slang diff --git a/source/compiler-core/windows/slang-win-visual-studio-util.h b/source/compiler-core/windows/slang-win-visual-studio-util.h new file mode 100644 index 000000000..05a3ea061 --- /dev/null +++ b/source/compiler-core/windows/slang-win-visual-studio-util.h @@ -0,0 +1,68 @@ +#ifndef SLANG_WIN_VISUAL_STUDIO_UTIL_H +#define SLANG_WIN_VISUAL_STUDIO_UTIL_H + +#include "../../core/slang-list.h" +#include "../../core/slang-string.h" + +#include "../../core/slang-process-util.h" + +#include "../slang-downstream-compiler.h" + +namespace Slang { + +struct WinVisualStudioUtil +{ + enum class Version: uint32_t + { + Unknown = 0, ///< This is an unknown (and not later) version + Future = 0xff * 10, ///< This is a version 'from the future' - that isn't specifically known. Will be treated as latest + }; + + struct VersionPath + { + Version version; ///< The visual studio version + String vcvarsPath; ///< The path to vcvars bat files, that need to be executed before executing the compiler + }; + + /// Find all the installations + static SlangResult find(List& outVersionPaths); + + /// Given a version find it's path + static SlangResult find(Version version, VersionPath& outPath); + + /// Find and add to the set (if not already there) + static SlangResult find(DownstreamCompilerSet* set); + + /// Create the cmdLine to start compiler for specified path + static void calcExecuteCompilerArgs(const VersionPath& versionPath, CommandLine& outCmdLine); + + /// Run visual studio on specified path with the parameters specified on the command line. Output placed in outResult. + static SlangResult executeCompiler(const VersionPath& versionPath, const CommandLine& commandLine, ExecuteResult& outResult); + + /// Get all the known version numbers + static void getVersions(List& outVersions); + + /// Gets the msc compiler used to compile this version. Returning Version(0) means unknown + static Version getCompiledVersion(); + + /// Create a version from a high and low indices + static Version makeVersion(int high, int low = 0) { SLANG_ASSERT(low >= 0 && low <= 9); return Version(high * 10 + low); } + + /// Convert a version number into a string + static void append(Version version, StringBuilder& outBuilder); + + /// Get version as desc + static DownstreamCompiler::Desc getDesc(Version version) + { + DownstreamCompiler::Desc desc; + desc.type = SLANG_PASS_THROUGH_VISUAL_STUDIO; + desc.majorVersion = Int(version) / 10; + desc.minorVersion = Int(version) % 10; + return desc; + } + +}; + +} // namespace Slang + +#endif diff --git a/source/core/slang-char-util.cpp b/source/core/slang-char-util.cpp index 298f9b75f..ea9e6dbf2 100644 --- a/source/core/slang-char-util.cpp +++ b/source/core/slang-char-util.cpp @@ -2,10 +2,8 @@ namespace Slang { -static const CharUtil::CharFlagMap _calcCharFlagsMap() +/* static */CharUtil::CharFlagMap CharUtil::makeCharFlagMap() { - typedef CharUtil::Flag Flag; - CharUtil::CharFlagMap map; memset(&map, 0, sizeof(map)); @@ -48,6 +46,11 @@ static const CharUtil::CharFlagMap _calcCharFlagsMap() return map; } -/* static */const CharUtil::CharFlagMap CharUtil::g_charFlagMap = _calcCharFlagsMap(); +/* static */int CharUtil::_ensureLink() +{ + return makeCharFlagMap().flags[0]; +} + +/* static */const CharUtil::CharFlagMap CharUtil::g_charFlagMap = makeCharFlagMap(); } // namespace Slang diff --git a/source/core/slang-char-util.h b/source/core/slang-char-util.h index 2434697d4..8f7f69c90 100644 --- a/source/core/slang-char-util.h +++ b/source/core/slang-char-util.h @@ -41,12 +41,19 @@ struct CharUtil /// Given a character return the upper case equivalent SLANG_FORCE_INLINE static char toUpper(char c) { return (c >= 'a' && c <= 'z') ? (c -'a' + 'A') : c; } - + struct CharFlagMap { Flags flags[0x100]; }; + static CharFlagMap makeCharFlagMap(); + + // HACK! + // JS: Many of the inlined functions of CharUtil just access a global map. That referencing this global is *NOT* enough to + // link correctly with CharUtil on linux for a shared library. Caling this function can force linkage. + static int _ensureLink(); + static const CharFlagMap g_charFlagMap; }; diff --git a/source/core/slang-downstream-compiler.cpp b/source/core/slang-downstream-compiler.cpp deleted file mode 100644 index 1a81fcac8..000000000 --- a/source/core/slang-downstream-compiler.cpp +++ /dev/null @@ -1,716 +0,0 @@ -// slang-downstream-compiler.cpp -#include "slang-downstream-compiler.h" - -#include "slang-common.h" -#include "../../slang-com-helper.h" -#include "slang-string-util.h" - -#include "slang-type-text-util.h" - -#include "slang-io.h" -#include "slang-shared-library.h" -#include "slang-blob.h" - -#ifdef SLANG_VC -# include "windows/slang-win-visual-studio-util.h" -#endif - -#include "slang-visual-studio-compiler-util.h" -#include "slang-gcc-compiler-util.h" -#include "slang-nvrtc-compiler.h" - -namespace Slang -{ - -static DownstreamCompiler::Infos _calcInfos() -{ - typedef DownstreamCompiler::Info Info; - typedef DownstreamCompiler::SourceLanguageFlag SourceLanguageFlag; - typedef DownstreamCompiler::SourceLanguageFlags SourceLanguageFlags; - - DownstreamCompiler::Infos infos; - - infos.infos[int(SLANG_PASS_THROUGH_CLANG)] = Info(SourceLanguageFlag::CPP | SourceLanguageFlag::C); - infos.infos[int(SLANG_PASS_THROUGH_VISUAL_STUDIO)] = Info(SourceLanguageFlag::CPP | SourceLanguageFlag::C); - infos.infos[int(SLANG_PASS_THROUGH_GCC)] = Info(SourceLanguageFlag::CPP | SourceLanguageFlag::C); - - infos.infos[int(SLANG_PASS_THROUGH_NVRTC)] = Info(SourceLanguageFlag::CUDA); - - infos.infos[int(SLANG_PASS_THROUGH_DXC)] = Info(SourceLanguageFlag::HLSL); - infos.infos[int(SLANG_PASS_THROUGH_FXC)] = Info(SourceLanguageFlag::HLSL); - infos.infos[int(SLANG_PASS_THROUGH_GLSLANG)] = Info(SourceLanguageFlag::GLSL); - - return infos; -} - -/* static */DownstreamCompiler::Infos DownstreamCompiler::s_infos = _calcInfos(); - -/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DownstreamCompiler::Desc !!!!!!!!!!!!!!!!!!!!!!*/ - -void DownstreamCompiler::Desc::appendAsText(StringBuilder& out) const -{ - out << TypeTextUtil::getPassThroughAsHumanText(type); - - // Append the version if there is a version - if (majorVersion || minorVersion) - { - out << " "; - out << majorVersion; - out << "."; - out << minorVersion; - } -} - -/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DownstreamDiagnostic !!!!!!!!!!!!!!!!!!!!!!!!*/ - -/* static */UnownedStringSlice DownstreamDiagnostic::getSeverityText(Severity severity) -{ - switch (severity) - { - default: return UnownedStringSlice::fromLiteral("Unknown"); - case Severity::Info: return UnownedStringSlice::fromLiteral("Info"); - case Severity::Warning: return UnownedStringSlice::fromLiteral("Warning"); - case Severity::Error: return UnownedStringSlice::fromLiteral("Error"); - } -} - -/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DownstreamCompiler !!!!!!!!!!!!!!!!!!!!!!!!!!!!!*/ - - -/* static */bool DownstreamCompiler::canCompile(SlangPassThrough compiler, SlangSourceLanguage sourceLanguage) -{ - const auto& info = getInfo(compiler); - return (info.sourceLanguageFlags & (SourceLanguageFlags(1) << int(sourceLanguage))) != 0; -} - -/* static */SlangCompileTarget DownstreamCompiler::getCompileTarget(SlangSourceLanguage sourceLanguage) -{ - switch (sourceLanguage) - { - case SLANG_SOURCE_LANGUAGE_HLSL: return SLANG_HLSL; - case SLANG_SOURCE_LANGUAGE_GLSL: return SLANG_GLSL; - case SLANG_SOURCE_LANGUAGE_C: return SLANG_C_SOURCE; - case SLANG_SOURCE_LANGUAGE_CPP: return SLANG_CPP_SOURCE; - case SLANG_SOURCE_LANGUAGE_CUDA: return SLANG_CUDA_SOURCE; - default: return SLANG_TARGET_UNKNOWN; - } -} - -/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DownstreamDiagnostics !!!!!!!!!!!!!!!!!!!!!!*/ - -Index DownstreamDiagnostics::getCountBySeverity(Diagnostic::Severity severity) const -{ - Index count = 0; - for (const auto& msg : diagnostics) - { - count += Index(msg.severity == severity); - } - return count; -} - -Int DownstreamDiagnostics::countByStage(Diagnostic::Stage stage, Index counts[Int(Diagnostic::Severity::CountOf)]) const -{ - Int count = 0; - ::memset(counts, 0, sizeof(Index) * Int(Diagnostic::Severity::CountOf)); - for (const auto& diagnostic : diagnostics) - { - if (diagnostic.stage == stage) - { - count++; - counts[Index(diagnostic.severity)]++; - } - } - return count++; -} - -static void _appendCounts(const Index counts[Int(DownstreamDiagnostic::Severity::CountOf)], StringBuilder& out) -{ - typedef DownstreamDiagnostic::Severity Severity; - - for (Index i = 0; i < Int(Severity::CountOf); i++) - { - if (counts[i] > 0) - { - out << DownstreamDiagnostic::getSeverityText(Severity(i)) << "(" << counts[i] << ") "; - } - } -} - -static void _appendSimplified(const Index counts[Int(DownstreamDiagnostic::Severity::CountOf)], StringBuilder& out) -{ - typedef DownstreamDiagnostic::Severity Severity; - for (Index i = 0; i < Int(Severity::CountOf); i++) - { - if (counts[i] > 0) - { - out << DownstreamDiagnostic::getSeverityText(Severity(i)) << " "; - } - } -} - -void DownstreamDiagnostics::appendSummary(StringBuilder& out) const -{ - Index counts[Int(Diagnostic::Severity::CountOf)]; - if (countByStage(Diagnostic::Stage::Compile, counts) > 0) - { - out << "Compile: "; - _appendCounts(counts, out); - out << "\n"; - } - if (countByStage(Diagnostic::Stage::Link, counts) > 0) - { - out << "Link: "; - _appendCounts(counts, out); - out << "\n"; - } -} - -void DownstreamDiagnostics::appendSimplifiedSummary(StringBuilder& out) const -{ - Index counts[Int(Diagnostic::Severity::CountOf)]; - if (countByStage(Diagnostic::Stage::Compile, counts) > 0) - { - out << "Compile: "; - _appendSimplified(counts, out); - out << "\n"; - } - if (countByStage(Diagnostic::Stage::Link, counts) > 0) - { - out << "Link: "; - _appendSimplified(counts, out); - out << "\n"; - } -} - -void DownstreamDiagnostics::removeBySeverity(Diagnostic::Severity severity) -{ - Index count = diagnostics.getCount(); - for (Index i = 0; i < count; ++i) - { - if (diagnostics[i].severity == severity) - { - diagnostics.removeAt(i); - i--; - count--; - } - } -} - -/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! CommandLineDownstreamCompileResult !!!!!!!!!!!!!!!!!!!!!!*/ - -SlangResult CommandLineDownstreamCompileResult::getHostCallableSharedLibrary(ComPtr& outLibrary) -{ - if (m_hostCallableSharedLibrary) - { - outLibrary = m_hostCallableSharedLibrary; - return SLANG_OK; - } - - // Okay we want to load - // Try loading the shared library - SharedLibrary::Handle handle; - if (SLANG_FAILED(SharedLibrary::loadWithPlatformPath(m_moduleFilePath.getBuffer(), handle))) - { - return SLANG_FAIL; - } - // The shared library needs to keep temp files in scope - RefPtr sharedLib(new TemporarySharedLibrary(handle, m_moduleFilePath)); - sharedLib->m_temporaryFileSet = m_temporaryFiles; - - m_hostCallableSharedLibrary = sharedLib; - outLibrary = m_hostCallableSharedLibrary; - return SLANG_OK; -} - -SlangResult CommandLineDownstreamCompileResult::getBinary(ComPtr& outBlob) -{ - if (m_binaryBlob) - { - outBlob = m_binaryBlob; - return SLANG_OK; - } - - // Read the binary - try - { - // Read the contents of the binary - List contents = File::readAllBytes(m_moduleFilePath); - - m_binaryBlob = new ScopeRefObjectBlob(ListBlob::moveCreate(contents), m_temporaryFiles); - outBlob = m_binaryBlob; - return SLANG_OK; - } - catch (const Slang::IOException&) - { - return SLANG_FAIL; - } -} - -/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! CommandLineDownstreamCompiler !!!!!!!!!!!!!!!!!!!!!!*/ - -static bool _isContentsInFile(const DownstreamCompiler::CompileOptions& options) -{ - if (options.sourceContentsPath.getLength() <= 0) - { - return false; - } - - // We can see if we can load it - if (File::exists(options.sourceContentsPath)) - { - // Here we look for the file on the regular file system (as opposed to using the - // ISlangFileSystem. This is unfortunate but necessary - because when we call out - // to the compiler all it is able to (currently) see are files on the file system. - // - // Note that it could be coincidence that the filesystem has a file that's identical in - // contents/name. That being the case though, any includes wouldn't work for a generated - // file either from some specialized ISlangFileSystem, so this is probably as good as it gets - // until we can integrate directly to a C/C++ compiler through say a shared library where we can control - // file system access. - try - { - String readContents = File::readAllText(options.sourceContentsPath); - // We should see if they are the same - return options.sourceContents == readContents.getUnownedSlice(); - } - catch (const Slang::IOException&) - { - } - } - return false; -} - -SlangResult CommandLineDownstreamCompiler::compile(const CompileOptions& inOptions, RefPtr& out) -{ - // Copy the command line options - CommandLine cmdLine(m_cmdLine); - - CompileOptions options(inOptions); - - // Find all the files that will be produced - RefPtr productFileSet(new TemporaryFileSet); - - if (options.modulePath.getLength() == 0 || options.sourceContents.getLength() != 0) - { - String modulePath = options.modulePath; - - // If there is no module path, generate one. - if (modulePath.getLength() == 0) - { - SLANG_RETURN_ON_FAIL(File::generateTemporary(UnownedStringSlice::fromLiteral("slang-generated"), modulePath)); - options.modulePath = modulePath; - } - - if (_isContentsInFile(options)) - { - options.sourceFiles.add(options.sourceContentsPath); - } - else - { - String compileSourcePath = modulePath; - - // NOTE: Strictly speaking producing filenames by modifying the generateTemporary path that may introduce a temp filename clash, but in practice is extraordinary unlikely - compileSourcePath.append("-src"); - - // Make the temporary filename have the appropriate extension. - if (options.sourceLanguage == SLANG_SOURCE_LANGUAGE_C) - { - compileSourcePath.append(".c"); - } - else - { - compileSourcePath.append(".cpp"); - } - - // Write it out - try - { - productFileSet->add(compileSourcePath); - - File::writeAllText(compileSourcePath, options.sourceContents); - } - catch (...) - { - return SLANG_FAIL; - } - - // Add it as a source file - options.sourceFiles.add(compileSourcePath); - } - - // There is no source contents - options.sourceContents = String(); - options.sourceContentsPath = String(); - } - - // Append command line args to the end of cmdLine using the target specific function for the specified options - SLANG_RETURN_ON_FAIL(calcArgs(options, cmdLine)); - - String moduleFilePath; - - { - StringBuilder builder; - SLANG_RETURN_ON_FAIL(calcModuleFilePath(options, builder)); - moduleFilePath = builder.ProduceString(); - } - - { - List paths; - SLANG_RETURN_ON_FAIL(calcCompileProducts(options, DownstreamCompiler::ProductFlag::All, paths)); - productFileSet->add(paths); - } - - ExecuteResult exeRes; - -#if 0 - // Test - { - String line = ProcessUtil::getCommandLineString(cmdLine); - printf("%s", line.getBuffer()); - } -#endif - - SLANG_RETURN_ON_FAIL(ProcessUtil::execute(cmdLine, exeRes)); - -#if 0 - { - printf("stdout=\"%s\"\nstderr=\"%s\"\nret=%d\n", exeRes.standardOutput.getBuffer(), exeRes.standardError.getBuffer(), int(exeRes.resultCode)); - } -#endif - - DownstreamDiagnostics diagnostics; - SLANG_RETURN_ON_FAIL(parseOutput(exeRes, diagnostics)); - - - out = new CommandLineDownstreamCompileResult(diagnostics, moduleFilePath, productFileSet); - - return SLANG_OK; -} - -/* !!!!!!!!!!!!!!!!!!!!!!!!! DownstreamCompiler::Desc !!!!!!!!!!!!!!!!!!!!!!*/ - -static DownstreamCompiler::Desc _calcCompiledWithDesc() -{ - DownstreamCompiler::Desc desc; - -#if SLANG_VC - desc = WinVisualStudioUtil::getDesc(WinVisualStudioUtil::getCompiledVersion()); -#elif SLANG_CLANG - desc.type = SLANG_PASS_THROUGH_CLANG; - desc.majorVersion = Int(__clang_major__); - desc.minorVersion = Int(__clang_minor__); -#elif SLANG_GCC - desc.type = SLANG_PASS_THROUGH_GCC; - desc.majorVersion = Int(__GNUC__); - desc.minorVersion = Int(__GNUC_MINOR__); -#else - // TODO(JS): Hmmm None is not quite the same as unknown. It works for now, but we might want to have a distinct enum for unknown. - desc.type = SLANG_PASS_THROUGH_NONE; -#endif - - return desc; -} - -/* !!!!!!!!!!!!!!!!!!!!!!!!! DownstreamCompilerUtil !!!!!!!!!!!!!!!!!!!!!!*/ - -const DownstreamCompiler::Desc& DownstreamCompilerUtil::getCompiledWithDesc() -{ - static DownstreamCompiler::Desc s_desc = _calcCompiledWithDesc(); - return s_desc; -} - -/* static */DownstreamCompiler* DownstreamCompilerUtil::findCompiler(const DownstreamCompilerSet* set, MatchType matchType, const DownstreamCompiler::Desc& desc) -{ - List compilers; - set->getCompilers(compilers); - return findCompiler(compilers, matchType, desc); -} - -/* static */DownstreamCompiler* DownstreamCompilerUtil::findCompiler(const List& compilers, MatchType matchType, const DownstreamCompiler::Desc& desc) -{ - if (compilers.getCount() <= 0) - { - return nullptr; - } - - Int bestIndex = -1; - - const SlangPassThrough compilerType = desc.type; - - Int maxVersionValue = 0; - Int minVersionDiff = 0x7fffffff; - - Int descVersionValue = desc.getVersionValue(); - - // If we don't have version set, then anything 0 or above is good enough, and just take newest - if (descVersionValue == 0) - { - maxVersionValue = -1; - matchType = MatchType::Newest; - } - - for (Index i = 0; i < compilers.getCount(); ++i) - { - DownstreamCompiler* compiler = compilers[i]; - auto compilerDesc = compiler->getDesc(); - - if (compilerType == compilerDesc.type) - { - const Int versionValue = compilerDesc.getVersionValue(); - switch (matchType) - { - case MatchType::MinGreaterEqual: - { - auto diff = descVersionValue - versionValue; - if (diff >= 0 && diff < minVersionDiff) - { - bestIndex = i; - minVersionDiff = diff; - } - break; - } - case MatchType::MinAbsolute: - { - auto diff = descVersionValue - versionValue; - diff = (diff >= 0) ? diff : -diff; - if (diff < minVersionDiff) - { - bestIndex = i; - minVersionDiff = diff; - } - break; - } - case MatchType::Newest: - { - if (versionValue > maxVersionValue) - { - maxVersionValue = versionValue; - bestIndex = i; - } - break; - } - } - } - } - - return (bestIndex >= 0) ? compilers[bestIndex] : nullptr; -} - -/* static */DownstreamCompiler* DownstreamCompilerUtil::findClosestCompiler(const List& compilers, const DownstreamCompiler::Desc& desc) -{ - DownstreamCompiler* compiler; - - compiler = findCompiler(compilers, MatchType::MinGreaterEqual, desc); - if (compiler) - { - return compiler; - } - compiler = findCompiler(compilers, MatchType::MinAbsolute, desc); - if (compiler) - { - return compiler; - } - - // If we are gcc, we can try clang and vice versa - if (desc.type == SLANG_PASS_THROUGH_GCC || desc.type == SLANG_PASS_THROUGH_CLANG) - { - DownstreamCompiler::Desc compatible = desc; - compatible.type = (compatible.type == SLANG_PASS_THROUGH_CLANG) ? SLANG_PASS_THROUGH_GCC : SLANG_PASS_THROUGH_CLANG; - - compiler = findCompiler(compilers, MatchType::MinGreaterEqual, compatible); - if (compiler) - { - return compiler; - } - compiler = findCompiler(compilers, MatchType::MinAbsolute, compatible); - if (compiler) - { - return compiler; - } - } - - return nullptr; -} - -/* static */DownstreamCompiler* DownstreamCompilerUtil::findClosestCompiler(const DownstreamCompilerSet* set, const DownstreamCompiler::Desc& desc) -{ - DownstreamCompiler* compiler = set->getCompiler(desc); - if (compiler) - { - return compiler; - } - List compilers; - set->getCompilers(compilers); - return findClosestCompiler(compilers, desc); -} - -/* static */void DownstreamCompilerUtil::updateDefault(DownstreamCompilerSet* set, SlangSourceLanguage sourceLanguage) -{ - DownstreamCompiler* compiler = nullptr; - - switch (sourceLanguage) - { - case SLANG_SOURCE_LANGUAGE_CPP: - case SLANG_SOURCE_LANGUAGE_C: - { - compiler = findClosestCompiler(set, getCompiledWithDesc()); - break; - } - case SLANG_SOURCE_LANGUAGE_CUDA: - { - DownstreamCompiler::Desc desc; - desc.type = SLANG_PASS_THROUGH_NVRTC; - compiler = findCompiler(set, MatchType::Newest, desc); - break; - } - default: break; - } - - set->setDefaultCompiler(sourceLanguage, compiler); -} - -/* static */void DownstreamCompilerUtil::updateDefaults(DownstreamCompilerSet* set) -{ - for (Index i = 0; i < Index(SLANG_SOURCE_LANGUAGE_COUNT_OF); ++i) - { - updateDefault(set, SlangSourceLanguage(i)); - } -} - -static SlangResult _locateDXCCompilers(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set) -{ - // First try dxil, so it's loaded from the same path if it's there - ComPtr dxil; - DefaultSharedLibraryLoader::load(loader, path, "dxil", dxil.writeRef()); - - ComPtr sharedLibrary; - if (SLANG_SUCCEEDED(DefaultSharedLibraryLoader::load(loader, path, "dxcompiler", sharedLibrary.writeRef()))) - { - // Can we determine the version? - DownstreamCompiler::Desc desc(SLANG_PASS_THROUGH_DXC); - RefPtr compiler(new SharedLibraryDownstreamCompiler(desc, sharedLibrary)); - - set->addCompiler(compiler); - } - return SLANG_OK; -} - -static SlangResult _locateFXCCompilers(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set) -{ - ComPtr sharedLibrary; - if (SLANG_SUCCEEDED(DefaultSharedLibraryLoader::load(loader, path, "d3dcompiler_47", sharedLibrary.writeRef()))) - { - // Can we determine the version? - DownstreamCompiler::Desc desc(SLANG_PASS_THROUGH_FXC); - RefPtr compiler(new SharedLibraryDownstreamCompiler(desc, sharedLibrary)); - set->addCompiler(compiler); - } - return SLANG_OK; -} - -static SlangResult _locateGlslangCompilers(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set) -{ -#if SLANG_UNIX_FAMILY - // On unix systems we need to ensure pthread is loaded first. - ComPtr pthreadLibrary; - DefaultSharedLibraryLoader::load(loader, path, "pthread", pthreadLibrary.writeRef()); -#endif - ComPtr sharedLibrary; - if (SLANG_SUCCEEDED(DefaultSharedLibraryLoader::load(loader, path, "slang-glslang", sharedLibrary.writeRef()))) - { - // Can we determine the version? - DownstreamCompiler::Desc desc(SLANG_PASS_THROUGH_GLSLANG); - RefPtr compiler(new SharedLibraryDownstreamCompiler(desc, sharedLibrary)); - set->addCompiler(compiler); - } - return SLANG_OK; -} - -/* static */void DownstreamCompilerUtil::setDefaultLocators(DownstreamCompilerLocatorFunc outFuncs[int(SLANG_PASS_THROUGH_COUNT_OF)]) -{ - outFuncs[int(SLANG_PASS_THROUGH_VISUAL_STUDIO)] = &VisualStudioCompilerUtil::locateCompilers; - outFuncs[int(SLANG_PASS_THROUGH_CLANG)] = &GCCDownstreamCompilerUtil::locateClangCompilers; - outFuncs[int(SLANG_PASS_THROUGH_GCC)] = &GCCDownstreamCompilerUtil::locateGCCCompilers; - outFuncs[int(SLANG_PASS_THROUGH_NVRTC)] = &NVRTCDownstreamCompilerUtil::locateCompilers; - outFuncs[int(SLANG_PASS_THROUGH_DXC)] = &_locateDXCCompilers; - outFuncs[int(SLANG_PASS_THROUGH_FXC)] = &_locateFXCCompilers; - outFuncs[int(SLANG_PASS_THROUGH_GLSLANG)] = &_locateGlslangCompilers; -} - -/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DownstreamCompilerSet !!!!!!!!!!!!!!!!!!!!!!*/ - -void DownstreamCompilerSet::getCompilerDescs(List& outCompilerDescs) const -{ - outCompilerDescs.clear(); - for (DownstreamCompiler* compiler : m_compilers) - { - outCompilerDescs.add(compiler->getDesc()); - } -} - -Index DownstreamCompilerSet::_findIndex(const DownstreamCompiler::Desc& desc) const -{ - const Index count = m_compilers.getCount(); - for (Index i = 0; i < count; ++i) - { - if (m_compilers[i]->getDesc() == desc) - { - return i; - } - } - return -1; -} - -DownstreamCompiler* DownstreamCompilerSet::getCompiler(const DownstreamCompiler::Desc& compilerDesc) const -{ - const Index index = _findIndex(compilerDesc); - return index >= 0 ? m_compilers[index] : nullptr; -} - -void DownstreamCompilerSet::getCompilers(List& outCompilers) const -{ - outCompilers.clear(); - outCompilers.addRange((DownstreamCompiler*const*)m_compilers.begin(), m_compilers.getCount()); -} - -bool DownstreamCompilerSet::hasCompiler(SlangPassThrough compilerType) const -{ - for (DownstreamCompiler* compiler : m_compilers) - { - const auto& desc = compiler->getDesc(); - if (desc.type == compilerType) - { - return true; - } - } - return false; -} - - -void DownstreamCompilerSet::remove(SlangPassThrough compilerType) -{ - for (Index i = 0; i < m_compilers.getCount(); ++i) - { - DownstreamCompiler* compiler = m_compilers[i]; - if (compiler->getDesc().type == compilerType) - { - m_compilers.fastRemoveAt(i); - i--; - } - } -} - -void DownstreamCompilerSet::addCompiler(DownstreamCompiler* compiler) -{ - const Index index = _findIndex(compiler->getDesc()); - if (index >= 0) - { - m_compilers[index] = compiler; - } - else - { - m_compilers.add(compiler); - } -} - -} diff --git a/source/core/slang-downstream-compiler.h b/source/core/slang-downstream-compiler.h deleted file mode 100644 index 9c3b9055f..000000000 --- a/source/core/slang-downstream-compiler.h +++ /dev/null @@ -1,501 +0,0 @@ -#ifndef SLANG_DOWNSTREAM_COMPILER_H -#define SLANG_DOWNSTREAM_COMPILER_H - -#include "slang-common.h" -#include "slang-string.h" - -#include "slang-process-util.h" - -#include "slang-platform.h" -#include "slang-semantic-version.h" - -#include "slang-io.h" - -#include "../../slang-com-ptr.h" - -namespace Slang -{ - -struct DownstreamDiagnostic -{ - typedef DownstreamDiagnostic ThisType; - - enum class Severity - { - Unknown, - Info, - Warning, - Error, - CountOf, - }; - enum class Stage - { - Compile, - Link, - }; - - void reset() - { - severity = Severity::Unknown; - stage = Stage::Compile; - fileLine = 0; - } - - bool operator==(const ThisType& rhs) const - { - return severity == rhs.severity && - stage == rhs.stage && - text == rhs.text && - code == rhs.code && - filePath == rhs.filePath && - fileLine == rhs.fileLine; - } - bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } - - static UnownedStringSlice getSeverityText(Severity severity); - - Severity severity; ///< The severity of error - Stage stage; ///< The stage the error came from - String text; ///< The text of the error - String code; ///< The compiler specific error code - String filePath; ///< The path the error originated from - Int fileLine; ///< The line number the error came from -}; - -struct DownstreamDiagnostics -{ - typedef DownstreamDiagnostic Diagnostic; - - /// Reset to an initial empty state - void reset() { diagnostics.clear(); rawDiagnostics = String(); result = SLANG_OK; } - - /// Get the number of diagnostics by severity - Index getCountBySeverity(Diagnostic::Severity severity) const; - /// True if there are any diagnostics of severity - bool has(Diagnostic::Severity severity) const { return getCountBySeverity(severity) > 0; } - - /// Stores in outCounts, the amount of diagnostics for the stage of each severity - Int countByStage(Diagnostic::Stage stage, Index outCounts[Int(Diagnostic::Severity::CountOf)]) const; - - /// Append a summary to out - void appendSummary(StringBuilder& out) const; - /// Appends a summary that just identifies if there is an error of a type (not a count) - void appendSimplifiedSummary(StringBuilder& out) const; - - /// Remove all diagnostics of the type - void removeBySeverity(Diagnostic::Severity severity); - - String rawDiagnostics; - - SlangResult result; - List diagnostics; -}; - -class DownstreamCompileResult : public RefObject -{ -public: - - virtual SlangResult getHostCallableSharedLibrary(ComPtr& outLibrary) = 0; - virtual SlangResult getBinary(ComPtr& outBlob) = 0; - - const DownstreamDiagnostics& getDiagnostics() const { return m_diagnostics; } - - /// Ctor - DownstreamCompileResult(const DownstreamDiagnostics& diagnostics): - m_diagnostics(diagnostics) - {} - -protected: - DownstreamDiagnostics m_diagnostics; -}; - - -class BlobDownstreamCompileResult : public DownstreamCompileResult -{ -public: - typedef DownstreamCompileResult Super; - - virtual SlangResult getHostCallableSharedLibrary(ComPtr& outLibrary) SLANG_OVERRIDE { SLANG_UNUSED(outLibrary); return SLANG_FAIL; } - virtual SlangResult getBinary(ComPtr& outBlob) SLANG_OVERRIDE { outBlob = m_blob; return m_blob ? SLANG_OK : SLANG_FAIL; } - - BlobDownstreamCompileResult(const DownstreamDiagnostics& diags, ISlangBlob* blob): - Super(diags), - m_blob(blob) - { - - } -protected: - ComPtr m_blob; -}; - -class DownstreamCompiler: public RefObject -{ -public: - typedef RefObject Super; - - typedef DownstreamCompileResult CompileResult; - - typedef uint32_t SourceLanguageFlags; - struct SourceLanguageFlag - { - enum Enum : SourceLanguageFlags - { - Unknown = SourceLanguageFlags(1) << SLANG_SOURCE_LANGUAGE_UNKNOWN, - Slang = SourceLanguageFlags(1) << SLANG_SOURCE_LANGUAGE_SLANG, - HLSL = SourceLanguageFlags(1) << SLANG_SOURCE_LANGUAGE_HLSL, - GLSL = SourceLanguageFlags(1) << SLANG_SOURCE_LANGUAGE_GLSL, - C = SourceLanguageFlags(1) << SLANG_SOURCE_LANGUAGE_C, - CPP = SourceLanguageFlags(1) << SLANG_SOURCE_LANGUAGE_CPP, - CUDA = SourceLanguageFlags(1) << SLANG_SOURCE_LANGUAGE_CUDA, - }; - }; - - struct Info - { - Info():sourceLanguageFlags(0) {} - - Info(SourceLanguageFlags inSourceLanguageFlags): - sourceLanguageFlags(inSourceLanguageFlags) - {} - SourceLanguageFlags sourceLanguageFlags; - }; - struct Infos - { - Info infos[int(SLANG_PASS_THROUGH_COUNT_OF)]; - }; - - struct Desc - { - typedef Desc ThisType; - - HashCode getHashCode() const { return combineHash(HashCode(type), combineHash(HashCode(majorVersion), HashCode(minorVersion))); } - bool operator==(const ThisType& rhs) const { return type == rhs.type && majorVersion == rhs.majorVersion && minorVersion == rhs.minorVersion; } - bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } - - /// Get the version as a value - Int getVersionValue() const { return majorVersion * 100 + minorVersion; } - - void appendAsText(StringBuilder& out) const; - /// true if has a version set - bool hasVersion() const { return majorVersion || minorVersion; } - - /// Ctor - explicit Desc(SlangPassThrough inType = SLANG_PASS_THROUGH_NONE, Int inMajorVersion = 0, Int inMinorVersion = 0):type(inType), majorVersion(inMajorVersion), minorVersion(inMinorVersion) {} - - SlangPassThrough type; ///< The type of the compiler - Int majorVersion; ///< Major version (interpretation is type specific) - Int minorVersion; ///< Minor version - }; - - enum class OptimizationLevel - { - None, ///< Don't optimize at all. - Default, ///< Default optimization level: balance code quality and compilation time. - High, ///< Optimize aggressively. - Maximal, ///< Include optimizations that may take a very long time, or may involve severe space-vs-speed tradeoffs - }; - - enum class DebugInfoType - { - None, ///< Don't emit debug information at all. - Minimal, ///< Emit as little debug information as possible, while still supporting stack traces. - Standard, ///< Emit whatever is the standard level of debug information for each target. - Maximal, ///< Emit as much debug information as possible for each target. - }; - enum class FloatingPointMode - { - Default, - Fast, - Precise, - }; - - enum TargetType - { - Executable, ///< Produce an executable - SharedLibrary, ///< Produce a shared library object/dll - Object, ///< Produce an object file - }; - - enum PipelineType - { - Unknown, - Compute, - Rasterization, - RayTracing, - }; - - struct Define - { - String nameWithSig; ///< If macro takes parameters include in brackets - String value; - }; - - struct CapabilityVersion - { - enum class Kind - { - CUDASM, ///< What the version is for - }; - Kind kind; - SemanticVersion version; - }; - - struct CompileOptions - { - typedef uint32_t Flags; - struct Flag - { - enum Enum : Flags - { - EnableExceptionHandling = 0x01, - Verbose = 0x02, - EnableSecurityChecks = 0x04, - }; - }; - - OptimizationLevel optimizationLevel = OptimizationLevel::Default; - DebugInfoType debugInfoType = DebugInfoType::Standard; - TargetType targetType = TargetType::Executable; - SlangSourceLanguage sourceLanguage = SLANG_SOURCE_LANGUAGE_CPP; - FloatingPointMode floatingPointMode = FloatingPointMode::Default; - PipelineType pipelineType = PipelineType::Unknown; - - Flags flags = Flag::EnableExceptionHandling; - - PlatformKind platform = PlatformKind::Unknown; - - /// The path/name of the output module. Should not have the extension, as that will be added for each of the target types. - /// If not set a module path will be internally generated internally on a command line based compiler - String modulePath; - - List defines; - - /// The contents of the source to compile. This can be empty is sourceFiles is set. - /// If the compiler is a commandLine file this source will be written to a temporary file. - String sourceContents; - /// 'Path' that the contents originated from. NOTE! This is for reporting only and doesn't have to exist on file system - String sourceContentsPath; - - /// The names/paths of source to compile. This can be empty if sourceContents is set. - List sourceFiles; - - List includePaths; - List libraryPaths; - - List requiredCapabilityVersions; - }; - - typedef uint32_t ProductFlags; - struct ProductFlag - { - enum Enum : ProductFlags - { - Debug = 0x1, ///< Used by debugger during execution - Execution = 0x2, ///< Required for execution - Compile = 0x4, ///< A product *required* for compilation - Miscellaneous = 0x8, ///< Anything else - }; - enum Mask : ProductFlags - { - All = 0xf, ///< All the flags - }; - }; - - enum class Product - { - DebugRun, - Run, - CompileTemporary, - All, - }; - - /// Get the desc of this compiler - const Desc& getDesc() const { return m_desc; } - /// Compile using the specified options. The result is in resOut - virtual SlangResult compile(const CompileOptions& options, RefPtr& outResult) = 0; - /// Some downstream compilers are backed by a shared library. This allows access to the shared library to access internal functions. - virtual ISlangSharedLibrary* getSharedLibrary() { return nullptr; } - - /// Get info for a compiler type - static const Info& getInfo(SlangPassThrough compiler) { return s_infos.infos[int(compiler)]; } - /// True if this compiler can compile the specified language - static bool canCompile(SlangPassThrough compiler, SlangSourceLanguage sourceLanguage); - - /// Given a source language return as the equivalent compile target - static SlangCompileTarget getCompileTarget(SlangSourceLanguage sourceLanguage); - -protected: - static Infos s_infos; - - DownstreamCompiler(const Desc& desc) : - m_desc(desc) - {} - DownstreamCompiler() {} - - Desc m_desc; -}; - -class CommandLineDownstreamCompileResult : public DownstreamCompileResult -{ -public: - typedef DownstreamCompileResult Super; - - virtual SlangResult getHostCallableSharedLibrary(ComPtr& outLibrary) SLANG_OVERRIDE; - virtual SlangResult getBinary(ComPtr& outBlob) SLANG_OVERRIDE; - - CommandLineDownstreamCompileResult(const DownstreamDiagnostics& diagnostics, const String& moduleFilePath, TemporaryFileSet* temporaryFileSet) : - Super(diagnostics), - m_moduleFilePath(moduleFilePath), - m_temporaryFiles(temporaryFileSet) - { - } - - RefPtr m_temporaryFiles; - -protected: - - String m_moduleFilePath; - DownstreamCompiler::CompileOptions m_options; - ComPtr m_binaryBlob; - /// Cache of the shared library if appropriate - ComPtr m_hostCallableSharedLibrary; -}; - -class CommandLineDownstreamCompiler : public DownstreamCompiler -{ -public: - typedef DownstreamCompiler Super; - - // DownstreamCompiler - virtual SlangResult compile(const CompileOptions& options, RefPtr& outResult) SLANG_OVERRIDE; - - // Functions to be implemented for a specific CommandLine - - /// Given the compilation options and the module name, determines the actual file name used for output - virtual SlangResult calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath) = 0; - /// Given options determines the paths to products produced (including the 'moduleFilePath'). - /// Note that does *not* guarentee all products were or should be produced. Just aims to include all that could - /// be produced, such that can be removed on completion. - virtual SlangResult calcCompileProducts(const CompileOptions& options, ProductFlags flags, List& outPaths) = 0; - - virtual SlangResult calcArgs(const CompileOptions& options, CommandLine& cmdLine) = 0; - virtual SlangResult parseOutput(const ExecuteResult& exeResult, DownstreamDiagnostics& output) = 0; - - CommandLineDownstreamCompiler(const Desc& desc, const String& exeName) : - Super(desc) - { - m_cmdLine.setExecutableFilename(exeName); - } - - CommandLineDownstreamCompiler(const Desc& desc, const CommandLine& cmdLine) : - Super(desc), - m_cmdLine(cmdLine) - {} - - CommandLineDownstreamCompiler(const Desc& desc):Super(desc) {} - - CommandLine m_cmdLine; -}; - -class SharedLibraryDownstreamCompiler: public DownstreamCompiler -{ -public: - typedef DownstreamCompiler Super; - - // DownstreamCompiler - virtual SlangResult compile(const CompileOptions& options, RefPtr& outResult) SLANG_OVERRIDE { SLANG_UNUSED(options); SLANG_UNUSED(outResult); return SLANG_E_NOT_IMPLEMENTED; } - virtual ISlangSharedLibrary* getSharedLibrary() SLANG_OVERRIDE { return m_library; } - - SharedLibraryDownstreamCompiler(const Desc& desc, ISlangSharedLibrary* library): - Super(desc), - m_library(library) - { - } -protected: - ComPtr m_library; -}; - -class DownstreamCompilerSet : public RefObject -{ -public: - typedef RefObject Super; - - /// Find all the available compilers - void getCompilerDescs(List& outCompilerDescs) const; - /// Returns list of all compilers - void getCompilers(List& outCompilers) const; - - /// Get a compiler - DownstreamCompiler* getCompiler(const DownstreamCompiler::Desc& compilerDesc) const; - - /// Will replace if there is one with same desc - void addCompiler(DownstreamCompiler* compiler); - - /// Get a default compiler - DownstreamCompiler* getDefaultCompiler(SlangSourceLanguage sourceLanguage) const { return m_defaultCompilers[int(sourceLanguage)]; } - /// Set the default compiler - void setDefaultCompiler(SlangSourceLanguage sourceLanguage, DownstreamCompiler* compiler) { m_defaultCompilers[int(sourceLanguage)] = compiler; } - - /// True if has a compiler of the specified type - bool hasCompiler(SlangPassThrough compilerType) const; - - void remove(SlangPassThrough compilerType); - - void clear() { m_compilers.clear(); } - -protected: - - Index _findIndex(const DownstreamCompiler::Desc& desc) const; - - RefPtr m_defaultCompilers[int(SLANG_SOURCE_LANGUAGE_COUNT_OF)]; - // This could be a dictionary/map - but doing a linear search is going to be fine and it makes - // somethings easier. - List> m_compilers; -}; - -typedef SlangResult (*DownstreamCompilerLocatorFunc)(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set); - -/* Only purpose of having base-class here is to make all the DownstreamCompiler types available directly in derived Utils */ -struct DownstreamCompilerBaseUtil -{ - typedef DownstreamCompiler::CompileOptions CompileOptions; - typedef DownstreamCompiler::OptimizationLevel OptimizationLevel; - typedef DownstreamCompiler::TargetType TargetType; - typedef DownstreamCompiler::DebugInfoType DebugInfoType; - - typedef DownstreamDiagnostics::Diagnostic Diagnostic; - - typedef DownstreamCompiler::FloatingPointMode FloatingPointMode; - typedef DownstreamCompiler::ProductFlag ProductFlag; - typedef DownstreamCompiler::ProductFlags ProductFlags; -}; - -struct DownstreamCompilerUtil: public DownstreamCompilerBaseUtil -{ - enum class MatchType - { - MinGreaterEqual, - MinAbsolute, - Newest, - }; - - /// Find a compiler - static DownstreamCompiler* findCompiler(const DownstreamCompilerSet* set, MatchType matchType, const DownstreamCompiler::Desc& desc); - static DownstreamCompiler* findCompiler(const List& compilers, MatchType matchType, const DownstreamCompiler::Desc& desc); - - /// Find the compiler closest to the desc - static DownstreamCompiler* findClosestCompiler(const List& compilers, const DownstreamCompiler::Desc& desc); - static DownstreamCompiler* findClosestCompiler(const DownstreamCompilerSet* set, const DownstreamCompiler::Desc& desc); - - /// Get the information on the compiler used to compile this source - static const DownstreamCompiler::Desc& getCompiledWithDesc(); - - static void updateDefault(DownstreamCompilerSet* set, SlangSourceLanguage sourceLanguage); - static void updateDefaults(DownstreamCompilerSet* set); - - static void setDefaultLocators(DownstreamCompilerLocatorFunc outFuncs[int(SLANG_PASS_THROUGH_COUNT_OF)]); -}; - -} - -#endif diff --git a/source/core/slang-file-system.cpp b/source/core/slang-file-system.cpp new file mode 100644 index 000000000..992ae4155 --- /dev/null +++ b/source/core/slang-file-system.cpp @@ -0,0 +1,888 @@ +#include "slang-file-system.h" + +#include "../../slang-com-ptr.h" +#include "../core/slang-io.h" +#include "../core/slang-string-util.h" + +namespace Slang +{ + +// Allocate static const storage for the various interface IDs that the Slang API needs to expose +static const Guid IID_ISlangUnknown = SLANG_UUID_ISlangUnknown; +static const Guid IID_ISlangFileSystem = SLANG_UUID_ISlangFileSystem; +static const Guid IID_ISlangFileSystemExt = SLANG_UUID_ISlangFileSystemExt; +static const Guid IID_ISlangMutableFileSystem = SLANG_UUID_ISlangMutableFileSystem; + +static const Guid IID_SlangCacheFileSystem = SLANG_UUID_CacheFileSystem; + +SLANG_FORCE_INLINE static SlangResult _checkExt(FileSystemStyle style) { return Index(style) >= Index(FileSystemStyle::Ext) ? SLANG_OK : SLANG_E_NOT_IMPLEMENTED; } +SLANG_FORCE_INLINE static SlangResult _checkMutable(FileSystemStyle style) { return Index(style) >= Index(FileSystemStyle::Mutable) ? SLANG_OK : SLANG_E_NOT_IMPLEMENTED; } + +SLANG_FORCE_INLINE static bool _canCast(FileSystemStyle style, const Guid& guid) +{ + if (guid == IID_ISlangUnknown || guid == IID_ISlangFileSystem) + { + return true; + } + else if (guid == IID_ISlangFileSystemExt) + { + return Index(style) >= Index(FileSystemStyle::Ext); + } + else if (guid == IID_ISlangMutableFileSystem) + { + return Index(style) >= Index(FileSystemStyle::Mutable); + } + return false; +} + +static FileSystemStyle _getFileSystemStyle(ISlangFileSystem* system, ComPtr& out) +{ + SLANG_ASSERT(system); + + FileSystemStyle style = FileSystemStyle::Load; + + if (SLANG_SUCCEEDED(system->queryInterface(IID_ISlangMutableFileSystem, (void**)out.writeRef()))) + { + style = FileSystemStyle::Mutable; + } + else if (SLANG_SUCCEEDED(system->queryInterface(IID_ISlangFileSystemExt, (void**)out.writeRef()))) + { + style = FileSystemStyle::Ext; + } + else + { + style = FileSystemStyle::Load; + out = system; + } + + SLANG_ASSERT(out); + return style; +} + +// Cacluate a combined path, just using Path:: string processing +static SlangResult _calcCombinedPath(SlangPathType fromPathType, const char* fromPath, const char* path, ISlangBlob** pathOut) +{ + String relPath; + switch (fromPathType) + { + case SLANG_PATH_TYPE_FILE: + { + relPath = Path::combine(Path::getParentDirectory(fromPath), path); + break; + } + case SLANG_PATH_TYPE_DIRECTORY: + { + relPath = Path::combine(fromPath, path); + break; + } + } + + *pathOut = StringUtil::createStringBlob(relPath).detach(); + return SLANG_OK; +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!! OSFileSystem !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!*/ + +/* static */OSFileSystem OSFileSystem::g_load(FileSystemStyle::Load); +/* static */OSFileSystem OSFileSystem::g_ext(FileSystemStyle::Ext); +/* static */OSFileSystem OSFileSystem::g_mutable(FileSystemStyle::Mutable); + +ISlangUnknown* OSFileSystem::getInterface(const Guid& guid) +{ + return _canCast(m_style, guid) ? static_cast(this) : nullptr; +} + +static String _fixPathDelimiters(const char* pathIn) +{ +#if SLANG_WINDOWS_FAMILY + return pathIn; +#else + // To allow windows style \ delimiters on other platforms, we convert to our standard delimiter + String path(pathIn); + return StringUtil::calcCharReplaced(pathIn, '\\', Path::kPathDelimiter); +#endif +} + +SlangResult OSFileSystem::getFileUniqueIdentity(const char* pathIn, ISlangBlob** outUniqueIdentity) +{ + SLANG_RETURN_ON_FAIL(_checkExt(m_style)); + + // By default we use the canonical path to uniquely identify a file + return getCanonicalPath(pathIn, outUniqueIdentity); +} + +SlangResult OSFileSystem::getCanonicalPath(const char* path, ISlangBlob** outCanonicalPath) +{ + SLANG_RETURN_ON_FAIL(_checkExt(m_style)); + + String canonicalPath; + SLANG_RETURN_ON_FAIL(Path::getCanonical(_fixPathDelimiters(path), canonicalPath)); + *outCanonicalPath = StringUtil::createStringBlob(canonicalPath).detach(); + return SLANG_OK; +} + +SlangResult OSFileSystem::getSimplifiedPath(const char* pathIn, ISlangBlob** outSimplifiedPath) +{ + SLANG_RETURN_ON_FAIL(_checkExt(m_style)); + + String simplifiedPath = Path::simplify(pathIn); + *outSimplifiedPath = StringUtil::createStringBlob(simplifiedPath).detach(); + return SLANG_OK; +} + +SlangResult OSFileSystem::calcCombinedPath(SlangPathType fromPathType, const char* fromPath, const char* path, ISlangBlob** pathOut) +{ + SLANG_RETURN_ON_FAIL(_checkExt(m_style)); + + // Don't need to fix delimiters - because combine path handles both path delimiter types + return _calcCombinedPath(fromPathType, fromPath, path, pathOut); +} + +SlangResult SLANG_MCALL OSFileSystem::getPathType(const char* pathIn, SlangPathType* pathTypeOut) +{ + SLANG_RETURN_ON_FAIL(_checkExt(m_style)); + + return Path::getPathType(_fixPathDelimiters(pathIn), pathTypeOut); +} + + +SlangResult OSFileSystem::loadFile(char const* pathIn, ISlangBlob** outBlob) +{ + // Default implementation that uses the `core` libraries facilities for talking to the OS filesystem. + // + // TODO: we might want to conditionally compile these in, so that + // a user could create a build of Slang that doesn't include any OS + // filesystem calls. + + const String path = _fixPathDelimiters(pathIn); + if (!File::exists(path)) + { + return SLANG_E_NOT_FOUND; + } + + ScopedAllocation alloc; + SLANG_RETURN_ON_FAIL(File::readAllBytes(path, alloc)); + *outBlob = RawBlob::moveCreate(alloc).detach(); + return SLANG_OK; +} + +SlangResult OSFileSystem::enumeratePathContents(const char* path, FileSystemContentsCallBack callback, void* userData) +{ + SLANG_RETURN_ON_FAIL(_checkExt(m_style)); + + struct Visitor : Path::Visitor + { + void accept(Path::Type type, const UnownedStringSlice& filename) SLANG_OVERRIDE + { + m_buffer.Clear(); + m_buffer.append(filename); + + SlangPathType pathType; + switch (type) + { + case Path::Type::File: pathType = SLANG_PATH_TYPE_FILE; break; + case Path::Type::Directory: pathType = SLANG_PATH_TYPE_DIRECTORY; break; + default: return; + } + + m_callback(pathType, m_buffer.getBuffer(), m_userData); + } + + Visitor(FileSystemContentsCallBack callback, void* userData) : + m_callback(callback), + m_userData(userData) + { + } + StringBuilder m_buffer; + FileSystemContentsCallBack m_callback; + void* m_userData; + }; + + Visitor visitor(callback, userData); + Path::find(path, nullptr, &visitor); + + return SLANG_OK; +} + +SlangResult OSFileSystem::saveFile(const char* pathIn, const void* data, size_t size) +{ + SLANG_RETURN_ON_FAIL(_checkMutable(m_style)); + + const String path = _fixPathDelimiters(pathIn); + + try + { + FileStream stream(pathIn, FileMode::Create, FileAccess::Write, FileShare::ReadWrite); + + int64_t numWritten = stream.write(data, size); + + if (numWritten != int64_t(size)) + { + return SLANG_FAIL; + } + + } + catch (const IOException&) + { + return SLANG_E_CANNOT_OPEN; + } + + return SLANG_OK; +} + +SlangResult OSFileSystem::remove(const char* path) +{ + SLANG_RETURN_ON_FAIL(_checkMutable(m_style)); + return Path::remove(path); +} + +SlangResult OSFileSystem::createDirectory(const char* path) +{ + SLANG_RETURN_ON_FAIL(_checkMutable(m_style)); + return Path::createDirectory(path); +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! CacheFileSystem !!!!!!!!!!!!!!!!!!!!!!!!!!! + +/* static */ const Result CacheFileSystem::s_compressedResultToResult[] = +{ + SLANG_E_UNINITIALIZED, + SLANG_OK, ///< Ok + SLANG_E_NOT_FOUND, ///< File not found + SLANG_E_CANNOT_OPEN, ///< CannotOpen, + SLANG_FAIL, ///< Fail +}; + +/* static */CacheFileSystem::CompressedResult CacheFileSystem::toCompressedResult(Result res) +{ + if (SLANG_SUCCEEDED(res)) + { + return CompressedResult::Ok; + } + switch (res) + { + case SLANG_E_CANNOT_OPEN: return CompressedResult::CannotOpen; + case SLANG_E_NOT_FOUND: return CompressedResult::NotFound; + default: return CompressedResult::Fail; + } +} + +SLANG_NO_THROW SlangResult SLANG_MCALL CacheFileSystem::queryInterface(SlangUUID const& uuid, void** outObject) +{ + if (uuid == IID_SlangCacheFileSystem) + { + *outObject = this; + return SLANG_OK; + } + + if (_canCast(FileSystemStyle::Ext, uuid)) + { + addReference(); + *outObject = static_cast(this); + return SLANG_OK; + } + return SLANG_E_NO_INTERFACE; +} + +CacheFileSystem::CacheFileSystem(ISlangFileSystem* fileSystem, UniqueIdentityMode uniqueIdentityMode, PathStyle pathStyle) +{ + setInnerFileSystem(fileSystem, uniqueIdentityMode, pathStyle); +} + +CacheFileSystem::~CacheFileSystem() +{ + for (const auto& pair : m_uniqueIdentityMap) + { + PathInfo* pathInfo = pair.Value; + delete pathInfo; + } +} + +void CacheFileSystem::setInnerFileSystem(ISlangFileSystem* fileSystem, UniqueIdentityMode uniqueIdentityMode, PathStyle pathStyle) +{ + m_fileSystem = fileSystem; + + m_uniqueIdentityMode = uniqueIdentityMode; + m_pathStyle = pathStyle; + + m_fileSystemExt.setNull(); + + if (fileSystem) + { + // Try to get the more sophisticated interface + fileSystem->queryInterface(IID_ISlangFileSystemExt, (void**)m_fileSystemExt.writeRef()); + } + + switch (m_uniqueIdentityMode) + { + case UniqueIdentityMode::Default: + case UniqueIdentityMode::FileSystemExt: + { + // If it's not a complete file system, we will default to SimplifyAndHash style by default + m_uniqueIdentityMode = m_fileSystemExt ? UniqueIdentityMode::FileSystemExt : UniqueIdentityMode::SimplifyPathAndHash; + break; + } + default: break; + } + + if (pathStyle == PathStyle::Default) + { + // We'll assume it's simplify-able + m_pathStyle = PathStyle::Simplifiable; + // If we have fileSystemExt, we defer to that + if (m_fileSystemExt) + { + // We just defer to the m_fileSystem + m_pathStyle = PathStyle::FileSystemExt; + } + } + + // It can't be default + SLANG_ASSERT(m_uniqueIdentityMode != UniqueIdentityMode::Default); +} + +void CacheFileSystem::clearCache() +{ + for (const auto& pair : m_uniqueIdentityMap) + { + PathInfo* pathInfo = pair.Value; + delete pathInfo; + } + + m_uniqueIdentityMap.Clear(); + m_pathMap.Clear(); + + if (m_fileSystemExt) + { + m_fileSystemExt->clearCache(); + } +} + + +// Determines if we can simplify a path for a given mode +static bool _canSimplifyPath(CacheFileSystem::UniqueIdentityMode mode) +{ + typedef CacheFileSystem::UniqueIdentityMode UniqueIdentityMode; + switch (mode) + { + case UniqueIdentityMode::SimplifyPath: + case UniqueIdentityMode::SimplifyPathAndHash: + { + return true; + } + default: + { + return false; + } + } +} + +SlangResult CacheFileSystem::enumeratePathContents(const char* path, FileSystemContentsCallBack callback, void* userData) +{ + if (m_fileSystemExt) + { + return m_fileSystemExt->enumeratePathContents(path, callback, userData); + } + + // Okay.. the contents of the 'cache' *is* the filesystem. So lets iterate over that + // This will win no prizes for efficiency, but that is unlikely to matter for typical usage + + if (!_canSimplifyPath(m_uniqueIdentityMode)) + { + // As it stands if we can't simplify paths, it's kind of hard to make this + // all work. As we use the simplified path cache + return SLANG_E_NOT_IMPLEMENTED; + } + + // Simplify the path + String simplifiedPath = Path::simplify(path); + + // If the simplified path is just a . then we don't have any prefix + if (simplifiedPath == ".") + { + simplifiedPath = ""; + } + + for (auto& pair : m_pathMap) + { + // NOTE! The currentPath can be a *non* simplified path (the m_pathMap is the cache of paths simplified and other to a file/directory) + // Also note that there will always be the simplified version of the path in cache. + const String& currentPath = pair.Key; + + // If it doesn't start with simplified path, then it can't be a hit + if (!currentPath.startsWith(simplifiedPath)) + { + continue; + } + + UnownedStringSlice remaining(currentPath.getBuffer() + simplifiedPath.getLength(), currentPath.end()); + + // If it starts with a / delimiter strip it + if (remaining.getLength() > 0 && remaining[0] == '/') + { + remaining = UnownedStringSlice(remaining.begin() + 1, remaining.end()); + } + + // If it has a path separator then it's either not simplified - so we ignore (we only want to invoke on the simplified path version as there is only one + // of these for every PathInfo) + // or it is a child file/directory, and so we ignore that too. + if (remaining.indexOf('/') >= 0 || remaining.indexOf('\\') >= 0) + { + continue; + } + + // We *know* that remaining comes from the end of currentPath .We also know currentPath is zero terminated. + // So we can just use (normally this would be a problem because UnownedStringSlice is generally *not* followed by zero termination. + const char* foundPath = remaining.begin(); + // Let's check that fact... + SLANG_ASSERT(foundPath[remaining.getLength()] == 0); + + PathInfo* pathInfo = pair.Value; + + SlangPathType pathType; + if (SLANG_FAILED(_getPathType(pathInfo, currentPath.getBuffer(), &pathType))) + { + continue; + } + + callback(pathType, foundPath, userData); + } + + return SLANG_OK; +} + + +SlangResult CacheFileSystem::_calcUniqueIdentity(const String& path, String& outUniqueIdentity, ComPtr& outFileContents) +{ + switch (m_uniqueIdentityMode) + { + case UniqueIdentityMode::FileSystemExt: + { + // Try getting the uniqueIdentity by asking underlying file system + ComPtr uniqueIdentity; + SLANG_RETURN_ON_FAIL(m_fileSystemExt->getFileUniqueIdentity(path.getBuffer(), uniqueIdentity.writeRef())); + // Get the path as a string + outUniqueIdentity = StringUtil::getString(uniqueIdentity); + return SLANG_OK; + } + case UniqueIdentityMode::Path: + { + outUniqueIdentity = path; + return SLANG_OK; + } + case UniqueIdentityMode::SimplifyPath: + { + outUniqueIdentity = Path::simplify(path); + // If it still has relative elements can't uniquely identify, so give up + return Path::hasRelativeElement(outUniqueIdentity) ? SLANG_FAIL : SLANG_OK; + } + case UniqueIdentityMode::SimplifyPathAndHash: + case UniqueIdentityMode::Hash: + { + // If we don't have a file system -> assume cannot be found + if (m_fileSystem == nullptr) + { + return SLANG_E_NOT_FOUND; + } + + // I can only see if this is the same file as already loaded by loading the file and doing a hash + Result res = m_fileSystem->loadFile(path.getBuffer(), outFileContents.writeRef()); + if (SLANG_FAILED(res) || outFileContents == nullptr) + { + return SLANG_FAIL; + } + + // Calculate the hash on the contents + const uint64_t hash = getHashCode64((const char*)outFileContents->getBufferPointer(), outFileContents->getBufferSize()); + + String hashString = Path::getFileName(path); + hashString = hashString.toLower(); + + hashString.append(':'); + + // The uniqueIdentity is a combination of name and hash + hashString.append(hash, 16); + + outUniqueIdentity = hashString; + return SLANG_OK; + } + } + + return SLANG_FAIL; +} + +CacheFileSystem::PathInfo* CacheFileSystem::_resolveUniqueIdentityCacheInfo(const String& path) +{ + // Use the path to produce uniqueIdentity information + ComPtr fileContents; + String uniqueIdentity; + + SlangResult res = _calcUniqueIdentity(path, uniqueIdentity, fileContents); + if (SLANG_FAILED(res)) + { + // Was not able to create a uniqueIdentity - return failure as nullptr + return nullptr; + } + + // Now try looking up by uniqueIdentity path. If not found, add a new result + PathInfo* pathInfo = nullptr; + if (!m_uniqueIdentityMap.TryGetValue(uniqueIdentity, pathInfo)) + { + // Create with found uniqueIdentity + pathInfo = new PathInfo(uniqueIdentity); + m_uniqueIdentityMap.Add(uniqueIdentity, pathInfo); + } + + // At this point they must have same uniqueIdentity + SLANG_ASSERT(pathInfo->getUniqueIdentity() == uniqueIdentity); + + // If we have the file contents (because of calc-ing uniqueIdentity), and there isn't a read file blob already + // store the data as if read, so doesn't get read again + if (fileContents && !pathInfo->m_fileBlob) + { + pathInfo->m_fileBlob = fileContents; + pathInfo->m_loadFileResult = CompressedResult::Ok; + } + + return pathInfo; +} + +CacheFileSystem::PathInfo* CacheFileSystem::_resolveSimplifiedPathCacheInfo(const String& path) +{ + // If we can simplify the path, try looking up in path cache with simplified path (as long as it's different!) + if (_canSimplifyPath(m_uniqueIdentityMode)) + { + const String simplifiedPath = Path::simplify(path); + // Only lookup if the path is different - because otherwise will recurse forever... + if (simplifiedPath != path) + { + // This is a recursive call - and will ensure the simplified path is added to the cache + return _resolvePathCacheInfo(simplifiedPath); + } + } + + return _resolveUniqueIdentityCacheInfo(path); +} + +CacheFileSystem::PathInfo* CacheFileSystem::_resolvePathCacheInfo(const String& path) +{ + // Lookup in path cache + PathInfo* pathInfo; + if (m_pathMap.TryGetValue(path, pathInfo)) + { + // Found so done + return pathInfo; + } + + // Try getting or creating taking into account possible path simplification + pathInfo = _resolveSimplifiedPathCacheInfo(path); + // Always add the result to the path cache (even if null) + m_pathMap.Add(path, pathInfo); + return pathInfo; +} + +SlangResult CacheFileSystem::loadFile(char const* pathIn, ISlangBlob** blobOut) +{ + *blobOut = nullptr; + String path(pathIn); + PathInfo* info = _resolvePathCacheInfo(path); + if (!info) + { + return SLANG_FAIL; + } + + if (info->m_loadFileResult == CompressedResult::Uninitialized) + { + info->m_loadFileResult = toCompressedResult(m_fileSystem->loadFile(path.getBuffer(), info->m_fileBlob.writeRef())); + } + + *blobOut = info->m_fileBlob; + if (*blobOut) + { + (*blobOut)->addRef(); + } + return toResult(info->m_loadFileResult); +} + +SlangResult CacheFileSystem::getFileUniqueIdentity(const char* path, ISlangBlob** outUniqueIdentity) +{ + PathInfo* info = _resolvePathCacheInfo(path); + if (!info) + { + return SLANG_E_NOT_FOUND; + } + info->m_uniqueIdentity->addRef(); + *outUniqueIdentity = info->m_uniqueIdentity; + return SLANG_OK; +} + +SlangResult CacheFileSystem::calcCombinedPath(SlangPathType fromPathType, const char* fromPath, const char* path, ISlangBlob** pathOut) +{ + // Just defer to contained implementation + switch (m_pathStyle) + { + case PathStyle::FileSystemExt: + { + return m_fileSystemExt->calcCombinedPath(fromPathType, fromPath, path, pathOut); + } + default: + { + // Just use the default implementation + return _calcCombinedPath(fromPathType, fromPath, path, pathOut); + } + } +} + +SlangResult CacheFileSystem::_getPathType(PathInfo* info, const char* inPath, SlangPathType* outPathType) +{ + if (info->m_getPathTypeResult == CompressedResult::Uninitialized) + { + if (m_fileSystemExt) + { + info->m_getPathTypeResult = toCompressedResult(m_fileSystemExt->getPathType(inPath, &info->m_pathType)); + } + else + { + // Okay try to load the file + if (info->m_loadFileResult == CompressedResult::Uninitialized) + { + info->m_loadFileResult = toCompressedResult(m_fileSystem->loadFile(inPath, info->m_fileBlob.writeRef())); + } + + // Make the getPathResult the same as the load result + info->m_getPathTypeResult = info->m_loadFileResult; + // Just set to file... the result is what matters in this case + info->m_pathType = SLANG_PATH_TYPE_FILE; + } + } + + *outPathType = info->m_pathType; + return toResult(info->m_getPathTypeResult); +} + +SlangResult CacheFileSystem::getPathType(const char* inPath, SlangPathType* outPathType) +{ + PathInfo* info = _resolvePathCacheInfo(inPath); + if (!info) + { + return SLANG_E_NOT_FOUND; + } + + return _getPathType(info, inPath, outPathType); +} + +SlangResult CacheFileSystem::getSimplifiedPath(const char* path, ISlangBlob** outSimplifiedPath) +{ + // If we have a ISlangFileSystemExt we can just pass on the request to it + switch (m_pathStyle) + { + case PathStyle::FileSystemExt: + { + return m_fileSystemExt->getSimplifiedPath(path, outSimplifiedPath); + } + case PathStyle::Simplifiable: + { + String simplifiedPath = Path::simplify(path); + *outSimplifiedPath = StringUtil::createStringBlob(simplifiedPath).detach(); + return SLANG_OK; + } + default: return SLANG_E_NOT_IMPLEMENTED; + } +} + +SlangResult CacheFileSystem::getCanonicalPath(const char* path, ISlangBlob** outCanonicalPath) +{ + // A file must exist to get a canonical path... + PathInfo* info = _resolvePathCacheInfo(path); + if (!info) + { + return SLANG_E_NOT_FOUND; + } + + // We don't have this -> so read it ... + if (info->m_getCanonicalPathResult == CompressedResult::Uninitialized) + { + if (!m_fileSystemExt) + { + return SLANG_E_NOT_IMPLEMENTED; + } + + // Try getting the canonicalPath by asking underlying file system + ComPtr canonicalPathBlob; + SlangResult res = m_fileSystemExt->getCanonicalPath(path, canonicalPathBlob.writeRef()); + + if (SLANG_SUCCEEDED(res)) + { + // Get the path as a string + String canonicalPath = StringUtil::getString(canonicalPathBlob); + if (canonicalPath.getLength() > 0) + { + info->m_canonicalPath = new StringBlob(canonicalPath); + } + else + { + res = SLANG_FAIL; + } + } + + // Save the result + info->m_getCanonicalPathResult = toCompressedResult(res); + } + + if (info->m_canonicalPath) + { + info->m_canonicalPath->addRef(); + } + *outCanonicalPath = info->m_canonicalPath; + return SLANG_OK; +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! RelativeFileSystem !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +RelativeFileSystem::RelativeFileSystem(ISlangFileSystem* fileSystem, const String& relativePath, bool stripPath) : + m_relativePath(relativePath), + m_stripPath(stripPath) +{ + m_style = _getFileSystemStyle(fileSystem, m_fileSystem); +} + +ISlangUnknown* RelativeFileSystem::getInterface(const Guid& guid) +{ + return _canCast(m_style, guid) ? static_cast(this) : nullptr; +} + +SlangResult RelativeFileSystem::_calcCombinedPathInner(SlangPathType fromPathType, const char* fromPath, const char* path, ISlangBlob** outPath) +{ + ISlangFileSystemExt* fileSystem = _getExt(); + if (fileSystem) + { + return fileSystem->calcCombinedPath(fromPathType, fromPath, path, outPath); + } + else + { + return _calcCombinedPath(fromPathType, fromPath, path, outPath); + } +} + +SlangResult RelativeFileSystem::_getFixedPath(const char* path, String& outPath) +{ + ComPtr blob; + if (m_stripPath) + { + String strippedPath = Path::getFileName(path); + SLANG_RETURN_ON_FAIL(_calcCombinedPathInner(SLANG_PATH_TYPE_DIRECTORY, m_relativePath.getBuffer(), strippedPath.getBuffer(), blob.writeRef())); + } + else + { + SLANG_RETURN_ON_FAIL(_calcCombinedPathInner(SLANG_PATH_TYPE_DIRECTORY, m_relativePath.getBuffer(), path, blob.writeRef())); + } + + outPath = StringUtil::getString(blob); + return SLANG_OK; +} + +SlangResult RelativeFileSystem::loadFile(char const* path, ISlangBlob** outBlob) +{ + String fixedPath; + SLANG_RETURN_ON_FAIL(_getFixedPath(path, fixedPath)); + return m_fileSystem->loadFile(fixedPath.getBuffer(), outBlob); +} + +SlangResult RelativeFileSystem::getFileUniqueIdentity(const char* path, ISlangBlob** outUniqueIdentity) +{ + auto fileSystem = _getExt(); + if (!fileSystem) return SLANG_E_NOT_IMPLEMENTED; + + String fixedPath; + SLANG_RETURN_ON_FAIL(_getFixedPath(path, fixedPath)); + return fileSystem->getFileUniqueIdentity(fixedPath.getBuffer(), outUniqueIdentity); +} + +SlangResult RelativeFileSystem::calcCombinedPath(SlangPathType fromPathType, const char* fromPath, const char* path, ISlangBlob** outPath) +{ + auto fileSystem = _getExt(); + if (!fileSystem) return SLANG_E_NOT_IMPLEMENTED; + + String fixedFromPath; + SLANG_RETURN_ON_FAIL(_getFixedPath(fromPath, fixedFromPath)); + + return fileSystem->calcCombinedPath(fromPathType, fixedFromPath.getBuffer(), path, outPath); +} + +SlangResult RelativeFileSystem::getPathType(const char* path, SlangPathType* outPathType) +{ + auto fileSystem = _getExt(); + if (!fileSystem) return SLANG_E_NOT_IMPLEMENTED; + + String fixedPath; + SLANG_RETURN_ON_FAIL(_getFixedPath(path, fixedPath)); + return fileSystem->getPathType(fixedPath.getBuffer(), outPathType); +} + +SlangResult RelativeFileSystem::getSimplifiedPath(const char* path, ISlangBlob** outSimplifiedPath) +{ + auto fileSystem = _getExt(); + if (!fileSystem) return SLANG_E_NOT_IMPLEMENTED; + + return fileSystem->getSimplifiedPath(path, outSimplifiedPath); +} + +SlangResult RelativeFileSystem::getCanonicalPath(const char* path, ISlangBlob** outCanonicalPath) +{ + auto fileSystem = _getExt(); + if (!fileSystem) return SLANG_E_NOT_IMPLEMENTED; + + String fixedPath; + SLANG_RETURN_ON_FAIL(_getFixedPath(path, fixedPath)); + return fileSystem->getCanonicalPath(fixedPath.getBuffer(), outCanonicalPath); +} + +void RelativeFileSystem::clearCache() +{ + auto fileSystem = _getExt(); + if (!fileSystem) return; + + fileSystem->clearCache(); +} + +SlangResult RelativeFileSystem::enumeratePathContents(const char* path, FileSystemContentsCallBack callback, void* userData) +{ + auto fileSystem = _getExt(); + if (!fileSystem) return SLANG_E_NOT_IMPLEMENTED; + + String fixedPath; + SLANG_RETURN_ON_FAIL(_getFixedPath(path, fixedPath)); + return fileSystem->enumeratePathContents(fixedPath.getBuffer(), callback, userData); +} + +SlangResult RelativeFileSystem::saveFile(const char* path, const void* data, size_t size) +{ + auto fileSystem = _getMutable(); + if (!fileSystem) return SLANG_E_NOT_IMPLEMENTED; + + String fixedPath; + SLANG_RETURN_ON_FAIL(_getFixedPath(path, fixedPath)); + return fileSystem->saveFile(fixedPath.getBuffer(), data, size); +} + +SlangResult RelativeFileSystem::remove(const char* path) +{ + auto fileSystem = _getMutable(); + if (!fileSystem) return SLANG_E_NOT_IMPLEMENTED; + + String fixedPath; + SLANG_RETURN_ON_FAIL(_getFixedPath(path, fixedPath)); + return fileSystem->remove(fixedPath.getBuffer()); +} + +SlangResult RelativeFileSystem::createDirectory(const char* path) +{ + auto fileSystem = _getMutable(); + if (!fileSystem) return SLANG_E_NOT_IMPLEMENTED; + + String fixedPath; + SLANG_RETURN_ON_FAIL(_getFixedPath(path, fixedPath)); + return fileSystem->createDirectory(fixedPath.getBuffer()); +} + +} diff --git a/source/core/slang-file-system.h b/source/core/slang-file-system.h new file mode 100644 index 000000000..d5145404d --- /dev/null +++ b/source/core/slang-file-system.h @@ -0,0 +1,253 @@ +#ifndef SLANG_FILE_SYSTEM_H_INCLUDED +#define SLANG_FILE_SYSTEM_H_INCLUDED + +#include "../../slang.h" +#include "../../slang-com-helper.h" +#include "../../slang-com-ptr.h" + +#include "../core/slang-blob.h" + +#include "../core/slang-string-util.h" +#include "../core/slang-dictionary.h" + +namespace Slang +{ + +enum class FileSystemStyle +{ + Load, ///< Equivalent to ISlangFileSystem + Ext, ///< Equivalent to ISlangFileSystemExt + Mutable, ///< Equivalent to ISlangModifyableFileSystem +}; + +// Can be used for all styles of file system +class OSFileSystem : public ISlangMutableFileSystem +{ +public: + // ISlangUnknown + // override ref counting, as DefaultFileSystem is singleton + SLANG_IUNKNOWN_QUERY_INTERFACE + SLANG_NO_THROW uint32_t SLANG_MCALL addRef() SLANG_OVERRIDE { return 1; } + SLANG_NO_THROW uint32_t SLANG_MCALL release() SLANG_OVERRIDE { return 1; } + + // ISlangFileSystem + virtual SLANG_NO_THROW SlangResult SLANG_MCALL loadFile(char const* path, ISlangBlob** outBlob) SLANG_OVERRIDE; + + // ISlangFileSystemExt + virtual SLANG_NO_THROW SlangResult SLANG_MCALL getFileUniqueIdentity(const char* path, ISlangBlob** uniqueIdentityOut) SLANG_OVERRIDE; + virtual SLANG_NO_THROW SlangResult SLANG_MCALL calcCombinedPath(SlangPathType fromPathType, const char* fromPath, const char* path, ISlangBlob** pathOut) SLANG_OVERRIDE; + virtual SLANG_NO_THROW SlangResult SLANG_MCALL getPathType(const char* path, SlangPathType* pathTypeOut) SLANG_OVERRIDE; + virtual SLANG_NO_THROW SlangResult SLANG_MCALL getSimplifiedPath(const char* path, ISlangBlob** outSimplifiedPath) SLANG_OVERRIDE; + virtual SLANG_NO_THROW SlangResult SLANG_MCALL getCanonicalPath(const char* path, ISlangBlob** outCanonicalPath) SLANG_OVERRIDE; + virtual SLANG_NO_THROW void SLANG_MCALL clearCache() SLANG_OVERRIDE {} + virtual SLANG_NO_THROW SlangResult SLANG_MCALL enumeratePathContents(const char* path, FileSystemContentsCallBack callback, void* userData) SLANG_OVERRIDE; + + // ISlangModifyableFileSystem + virtual SLANG_NO_THROW SlangResult SLANG_MCALL saveFile(const char* path, const void* data, size_t size) SLANG_OVERRIDE; + virtual SLANG_NO_THROW SlangResult SLANG_MCALL remove(const char* path) SLANG_OVERRIDE; + virtual SLANG_NO_THROW SlangResult SLANG_MCALL createDirectory(const char* path) SLANG_OVERRIDE; + + /// Get a default instance + static ISlangFileSystem* getLoadSingleton() { return &g_load; } + static ISlangFileSystemExt* getExtSingleton() { return &g_ext; } + static ISlangMutableFileSystem* getMutableSingleton() { return &g_mutable; } + +private: + + /// Make so not constructible + OSFileSystem(FileSystemStyle style): + m_style(style) + {} + + virtual ~OSFileSystem() {} + + ISlangUnknown* getInterface(const Guid& guid); + + FileSystemStyle m_style; + + static OSFileSystem g_load; + static OSFileSystem g_ext; + static OSFileSystem g_mutable; +}; + + #define SLANG_UUID_CacheFileSystem { 0x2f4d1d03, 0xa0d1, 0x434b, { 0x87, 0x7a, 0x65, 0x5, 0xa4, 0xa0, 0x9a, 0x3b } }; + +/* Wraps an underlying ISlangFileSystem or ISlangFileSystemExt and provides caching, +as well as emulation of methods if only has ISlangFileSystem interface. Will query capabilities +of the interface on the constructor. + +NOTE! That this behavior is the same as previously in that.... +1) calcRelativePath, just returns the path as processed by the Path:: methods +2) getUniqueIdentity behavior depends on the UniqueIdentityMode. +*/ +class CacheFileSystem: public ISlangFileSystemExt, public RefObject +{ + public: + + enum class PathStyle + { + Default, ///< Pass to say use the default + Simplifiable, ///< It can be simplified by Path::Simplify + FileSystemExt, ///< Use file system + }; + + enum UniqueIdentityMode + { + Default, ///< If passed, will default to the others depending on what kind of ISlangFileSystem is passed in + Path, ///< Just use the path as is (old style slang behavior) + SimplifyPath, ///< Use the input path 'simplified' (ie removing . and .. aspects) + Hash, ///< Use hashing + SimplifyPathAndHash, ///< Tries simplifying path first, and if that doesn't work it hashes + FileSystemExt, ///< Use the file system extended interface. + }; + + /* Cannot change order/add members without changing s_compressedResultToResult */ + enum class CompressedResult: uint8_t + { + Uninitialized, ///< Holds no value + Ok, ///< Ok + NotFound, ///< File not found + CannotOpen, ///< Cannot open + Fail, ///< Generic failure + CountOf, + }; + + struct PathInfo + { + PathInfo(const String& uniqueIdentity) + { + m_uniqueIdentity = new StringBlob(uniqueIdentity); + m_uniqueIdentity->addRef(); + + m_loadFileResult = CompressedResult::Uninitialized; + m_getPathTypeResult = CompressedResult::Uninitialized; + m_getCanonicalPathResult = CompressedResult::Uninitialized; + + m_pathType = SLANG_PATH_TYPE_FILE; + } + + /// Get the unique identity path as a string + const String& getUniqueIdentity() const { SLANG_ASSERT(m_uniqueIdentity); return m_uniqueIdentity->getString(); } + + RefPtr m_uniqueIdentity; + CompressedResult m_loadFileResult; + CompressedResult m_getPathTypeResult; + CompressedResult m_getCanonicalPathResult; + + SlangPathType m_pathType; + ComPtr m_fileBlob; + RefPtr m_canonicalPath; + }; + + Dictionary& getPathMap() { return m_pathMap; } + Dictionary& getUniqueMap() { return m_uniqueIdentityMap; } + + // ISlangUnknown + SLANG_NO_THROW SlangResult SLANG_MCALL queryInterface(SlangUUID const& uuid, void** outObject) SLANG_OVERRIDE; + SLANG_REF_OBJECT_IUNKNOWN_ADD_REF + SLANG_REF_OBJECT_IUNKNOWN_RELEASE + + // ISlangFileSystem + virtual SLANG_NO_THROW SlangResult SLANG_MCALL loadFile(char const* path, ISlangBlob** outBlob) SLANG_OVERRIDE; + + // ISlangFileSystemExt + virtual SLANG_NO_THROW SlangResult SLANG_MCALL getFileUniqueIdentity(const char* path, ISlangBlob** outUniqueIdentity) SLANG_OVERRIDE; + virtual SLANG_NO_THROW SlangResult SLANG_MCALL calcCombinedPath(SlangPathType fromPathType, const char* fromPath, const char* path, ISlangBlob** pathOut) SLANG_OVERRIDE; + virtual SLANG_NO_THROW SlangResult SLANG_MCALL getPathType(const char* path, SlangPathType* outPathType) SLANG_OVERRIDE; + virtual SLANG_NO_THROW SlangResult SLANG_MCALL getSimplifiedPath(const char* path, ISlangBlob** outSimplifiedPath) SLANG_OVERRIDE; + virtual SLANG_NO_THROW SlangResult SLANG_MCALL getCanonicalPath(const char* path, ISlangBlob** outCanonicalPath) SLANG_OVERRIDE; + virtual SLANG_NO_THROW void SLANG_MCALL clearCache() SLANG_OVERRIDE; + virtual SLANG_NO_THROW SlangResult SLANG_MCALL enumeratePathContents(const char* path, FileSystemContentsCallBack callback, void* userData) SLANG_OVERRIDE; + + /// Get the unique identity mode + UniqueIdentityMode getUniqueIdentityMode() const { return m_uniqueIdentityMode; } + /// Get the path style + PathStyle getPathStyle() const { return m_pathStyle; } + + /// Set the inner file system + void setInnerFileSystem(ISlangFileSystem* fileSystem, UniqueIdentityMode uniqueIdentityMode = UniqueIdentityMode::Default, PathStyle pathStyle = PathStyle::Default); + + /// Ctor + CacheFileSystem(ISlangFileSystem* fileSystem, UniqueIdentityMode uniqueIdentityMode = UniqueIdentityMode::Default, PathStyle pathStyle = PathStyle::Default); + /// Dtor + virtual ~CacheFileSystem(); + + static CompressedResult toCompressedResult(Result res); + static Result toResult(CompressedResult compRes) { return s_compressedResultToResult[int(compRes)]; } + static const Result s_compressedResultToResult[int(CompressedResult::CountOf)]; + +protected: + + /// Given a path, works out a uniqueIdentity, based on the uniqueIdentityMode. outFileContents will be set if file had to be read to produce the uniqueIdentity (ie with Hash) + SlangResult _calcUniqueIdentity(const String& path, String& outUniqueIdentity, ComPtr& outFileContents); + + /// For a given path gets a PathInfo. Can return nullptr, if it is not possible to create the PathInfo for some reason + PathInfo* _resolvePathCacheInfo(const String& path); + /// Turns the path into a uniqueIdentity, and then tries to look up in the uniqueIdentityMap. + PathInfo* _resolveUniqueIdentityCacheInfo(const String& path); + /// Will simplify the path (if possible) to lookup on the pathCache else will create on uniqueIdentityMap + PathInfo* _resolveSimplifiedPathCacheInfo(const String& path); + + SlangResult _getPathType(PathInfo* pathInfo, const char* inPath, SlangPathType* pathTypeOut); + + /* TODO: This may be improved by mapping to a ISlangBlob. This makes output fast and easy, and if constructed + as a StringBlob, we can just static_cast to get as a string to use internally, instead of constantly converting. + It is probably the case we cannot do dynamic_cast on ISlangBlob if we don't know where constructed -> if outside of slang codebase + doing such a cast can cause an exception. So we *never* want to do dynamic cast from blobs which could be created by external code. */ + + Dictionary m_pathMap; ///< Maps a path to a PathInfo (and unique identity) + Dictionary m_uniqueIdentityMap; ///< Maps a unique identity for a file to its contents. This OWNs the PathInfo. + + UniqueIdentityMode m_uniqueIdentityMode; ///< Determines how the 'uniqueIdentity' is produced. Cannot be Default in usage. + PathStyle m_pathStyle; ///< Style of paths + + ComPtr m_fileSystem; ///< Must always be set + ComPtr m_fileSystemExt; ///< Optionally set -> if nullptr will fall back on the m_fileSystem and emulate all the other methods of ISlangFileSystemExt +}; + +class RelativeFileSystem : public ISlangMutableFileSystem, public RefObject +{ +public: + SLANG_REF_OBJECT_IUNKNOWN_ALL + + // ISlangFileSystem + virtual SLANG_NO_THROW SlangResult SLANG_MCALL loadFile(char const* path, ISlangBlob** outBlob) SLANG_OVERRIDE; + + // ISlangFileSystemExt + virtual SLANG_NO_THROW SlangResult SLANG_MCALL getFileUniqueIdentity(const char* path, ISlangBlob** outUniqueIdentity) SLANG_OVERRIDE; + virtual SLANG_NO_THROW SlangResult SLANG_MCALL calcCombinedPath(SlangPathType fromPathType, const char* fromPath, const char* path, ISlangBlob** pathOut) SLANG_OVERRIDE; + virtual SLANG_NO_THROW SlangResult SLANG_MCALL getPathType(const char* path, SlangPathType* outPathType) SLANG_OVERRIDE; + virtual SLANG_NO_THROW SlangResult SLANG_MCALL getSimplifiedPath(const char* path, ISlangBlob** outSimplifiedPath) SLANG_OVERRIDE; + virtual SLANG_NO_THROW SlangResult SLANG_MCALL getCanonicalPath(const char* path, ISlangBlob** outCanonicalPath) SLANG_OVERRIDE; + virtual SLANG_NO_THROW void SLANG_MCALL clearCache() SLANG_OVERRIDE; + virtual SLANG_NO_THROW SlangResult SLANG_MCALL enumeratePathContents(const char* path, FileSystemContentsCallBack callback, void* userData) SLANG_OVERRIDE; + + // ISlangModifyableFileSystem + virtual SLANG_NO_THROW SlangResult SLANG_MCALL saveFile(const char* path, const void* data, size_t size) SLANG_OVERRIDE; + virtual SLANG_NO_THROW SlangResult SLANG_MCALL remove(const char* path) SLANG_OVERRIDE; + virtual SLANG_NO_THROW SlangResult SLANG_MCALL createDirectory(const char* path) SLANG_OVERRIDE; + + RelativeFileSystem(ISlangFileSystem* fileSystem, const String& relativePath, bool stripPath = false); + +protected: + + ISlangFileSystemExt* _getExt() { return Index(m_style) >= Index(FileSystemStyle::Ext) ? reinterpret_cast(m_fileSystem.get()) : nullptr; } + ISlangMutableFileSystem* _getMutable() { return Index(m_style) >= Index(FileSystemStyle::Mutable) ? reinterpret_cast(m_fileSystem.get()) : nullptr; } + + SlangResult _calcCombinedPathInner(SlangPathType fromPathType, const char* fromPath, const char* path, ISlangBlob** pathOut); + + SlangResult _getFixedPath(const char* path, String& outPath); + + ISlangUnknown* getInterface(const Guid& guid); + + bool m_stripPath; + + FileSystemStyle m_style; + ComPtr m_fileSystem; ///< NOTE! Has to match what's in style, such style can be reached via reinterpret_cast + + String m_relativePath; +}; + +} + +#endif // SLANG_FILE_SYSTEM_H_INCLUDED diff --git a/source/core/slang-gcc-compiler-util.cpp b/source/core/slang-gcc-compiler-util.cpp deleted file mode 100644 index 56955f1da..000000000 --- a/source/core/slang-gcc-compiler-util.cpp +++ /dev/null @@ -1,645 +0,0 @@ -// slang-gcc-compiler-util.cpp -#include "slang-gcc-compiler-util.h" - -#include "slang-common.h" -#include "../../slang-com-helper.h" -#include "slang-string-util.h" - -#include "slang-io.h" -#include "slang-shared-library.h" - -namespace Slang -{ - -/* static */SlangResult GCCDownstreamCompilerUtil::parseVersion(const UnownedStringSlice& text, const UnownedStringSlice& prefix, DownstreamCompiler::Desc& outDesc) -{ - List lines; - StringUtil::calcLines(text, lines); - - for (auto line : lines) - { - if (line.startsWith(prefix)) - { - const UnownedStringSlice remainingSlice = UnownedStringSlice(line.begin() + prefix.getLength(), line.end()).trim(); - const Index versionEndIndex = remainingSlice.indexOf(' '); - if (versionEndIndex < 0) - { - return SLANG_FAIL; - } - - const UnownedStringSlice versionSlice(remainingSlice.begin(), remainingSlice.begin() + versionEndIndex); - - // Version is in format 0.0.0 - List split; - StringUtil::split(versionSlice, '.', split); - List digits; - - for (auto v : split) - { - Int version; - SLANG_RETURN_ON_FAIL(StringUtil::parseInt(v, version)); - digits.add(version); - } - - if (digits.getCount() < 2) - { - return SLANG_FAIL; - } - - outDesc.majorVersion = digits[0]; - outDesc.minorVersion = digits[1]; - return SLANG_OK; - } - } - - return SLANG_FAIL; -} - -SlangResult GCCDownstreamCompilerUtil::calcVersion(const String& exeName, DownstreamCompiler::Desc& outDesc) -{ - CommandLine cmdLine; - cmdLine.setExecutableFilename(exeName); - cmdLine.addArg("-v"); - - ExecuteResult exeRes; - SLANG_RETURN_ON_FAIL(ProcessUtil::execute(cmdLine, exeRes)); - - const UnownedStringSlice prefixes[] = - { - UnownedStringSlice::fromLiteral("clang version"), - UnownedStringSlice::fromLiteral("gcc version"), - UnownedStringSlice::fromLiteral("Apple LLVM version"), - }; - const SlangPassThrough types[] = - { - SLANG_PASS_THROUGH_CLANG, - SLANG_PASS_THROUGH_GCC, - SLANG_PASS_THROUGH_CLANG, - }; - - SLANG_COMPILE_TIME_ASSERT(SLANG_COUNT_OF(prefixes) == SLANG_COUNT_OF(types)); - - for (Index i = 0; i < SLANG_COUNT_OF(prefixes); ++i) - { - // Set the type - outDesc.type = types[i]; - // Extract the version - if (SLANG_SUCCEEDED(parseVersion(exeRes.standardError.getUnownedSlice(), prefixes[i], outDesc))) - { - return SLANG_OK; - } - } - return SLANG_FAIL; -} - -static SlangResult _parseSeverity(const UnownedStringSlice& in, DownstreamDiagnostic::Severity& outSeverity) -{ - typedef DownstreamDiagnostic::Severity Severity; - - if (in == "error" || in == "fatal error") - { - outSeverity = Severity::Error; - } - else if (in == "warning") - { - outSeverity = Severity::Warning; - } - else if (in == "info" || in == "note") - { - outSeverity = Severity::Info; - } - else - { - return SLANG_FAIL; - } - return SLANG_OK; -} - -namespace { // anonymous - -enum class LineParseResult -{ - Single, ///< It's a single line - Start, ///< Line was the start of a message - Continuation, ///< Not totally clear, add to previous line if nothing else hit - Ignore, ///< Ignore the line -}; - -} // anonymous - -static SlangResult _parseGCCFamilyLine(const UnownedStringSlice& line, LineParseResult& outLineParseResult, DownstreamDiagnostic& outDiagnostic) -{ - typedef DownstreamDiagnostic Diagnostic; - typedef Diagnostic::Severity Severity; - - // Set to default case - outLineParseResult = LineParseResult::Ignore; - - /* example error output from different scenarios */ - - /* - tests/cpp-compiler/c-compile-error.c: In function 'int main(int, char**)': - tests/cpp-compiler/c-compile-error.c:8:13: error: 'b' was not declared in this scope - int a = b + c; - ^ - tests/cpp-compiler/c-compile-error.c:8:17: error: 'c' was not declared in this scope - int a = b + c; - ^ - */ - - /* /tmp/ccS0JCWe.o:c-compile-link-error.c:(.rdata$.refptr.thing[.refptr.thing]+0x0): undefined reference to `thing' - collect2: error: ld returned 1 exit status*/ - - /* - clang: warning: treating 'c' input as 'c++' when in C++ mode, this behavior is deprecated [-Wdeprecated] - Undefined symbols for architecture x86_64: - "_thing", referenced from: - _main in c-compile-link-error-a83ace.o - ld: symbol(s) not found for architecture x86_64 - clang: error: linker command failed with exit code 1 (use -v to see invocation) */ - - /* /tmp/c-compile-link-error-ccf151.o: In function `main': - c-compile-link-error.c:(.text+0x19): undefined reference to `thing' - clang: error: linker command failed with exit code 1 (use -v to see invocation) - */ - - /* /tmp/c-compile-link-error-301c8c.o: In function `main': - /home/travis/build/shader-slang/slang/tests/cpp-compiler/c-compile-link-error.c:10: undefined reference to `thing' - clang-7: error: linker command failed with exit code 1 (use -v to see invocation)*/ - - /* /path/slang-cpp-prelude.h:4:10: fatal error: ../slang.h: No such file or directory - #include "../slang.h" - ^~~~~~~~~~~~ - compilation terminated.*/ - - outDiagnostic.stage = Diagnostic::Stage::Compile; - - List split; - StringUtil::split(line, ':', split); - - // On windows we can have paths that are a: etc... if we detect this we can combine 0 - 1 to be 1. - if (split.getCount() > 1 && split[0].getLength() == 1) - { - const char c = split[0][0]; - if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) - { - // We'll assume it's a path - UnownedStringSlice path(split[0].begin(), split[1].end()); - split.removeAt(0); - split[0] = path; - } - } - - if (split.getCount() == 2) - { - const auto split0 = split[0].trim(); - if (split0 == UnownedStringSlice::fromLiteral("ld")) - { - // We'll ignore for now - outDiagnostic.stage = Diagnostic::Stage::Link; - outDiagnostic.severity = Severity::Info; - outDiagnostic.text = split[1].trim(); - outLineParseResult = LineParseResult::Start; - return SLANG_OK; - } - - if (SLANG_SUCCEEDED(_parseSeverity(split0, outDiagnostic.severity))) - { - // Command line errors can be just contain 'error:' etc. Can be seen on apple/clang - outDiagnostic.stage = Diagnostic::Stage::Compile; - outDiagnostic.text = split[1].trim(); - outLineParseResult = LineParseResult::Single; - return SLANG_OK; - } - - outLineParseResult = LineParseResult::Ignore; - return SLANG_OK; - } - else if (split.getCount() == 3) - { - const auto split0 = split[0].trim(); - const auto split1 = split[1].trim(); - const auto text = split[2].trim(); - - // Check for special handling for clang (Can be Clang or clang apparently) - if (split0.startsWith(UnownedStringSlice::fromLiteral("clang")) || - split0.startsWith(UnownedStringSlice::fromLiteral("Clang")) ) - { - // Extract the type - SLANG_RETURN_ON_FAIL(_parseSeverity(split[1].trim(), outDiagnostic.severity)); - - if (text.startsWith("linker command failed")) - { - outDiagnostic.stage = Diagnostic::Stage::Link; - } - - outDiagnostic.text = text; - outLineParseResult = LineParseResult::Start; - return SLANG_OK; - } - else if (split1.startsWith("(.text")) - { - // This is a little weak... but looks like it's a link error - outDiagnostic.filePath = split[0]; - outDiagnostic.severity = Severity::Error; - outDiagnostic.stage = Diagnostic::Stage::Link; - outDiagnostic.text = text; - outLineParseResult = LineParseResult::Single; - return SLANG_OK; - } - else if (text.startsWith("ld returned")) - { - outDiagnostic.stage = DownstreamDiagnostic::Stage::Link; - SLANG_RETURN_ON_FAIL(_parseSeverity(split[1].trim(), outDiagnostic.severity)); - outDiagnostic.text = line; - outLineParseResult = LineParseResult::Single; - return SLANG_OK; - } - else if (text == "") - { - // This is probably a prelude line, we'll just ignore it - outLineParseResult = LineParseResult::Ignore; - return SLANG_OK; - } - } - else if (split.getCount() == 4) - { - // Probably a link error, give the source line - String ext = Path::getPathExt(split[0]); - - // Maybe a bit fragile -> but probably okay for now - if (ext != "o" && ext != "obj") - { - outLineParseResult = LineParseResult::Ignore; - return SLANG_OK; - } - else - { - outDiagnostic.filePath = split[1]; - outDiagnostic.fileLine = 0; - outDiagnostic.severity = Diagnostic::Severity::Error; - outDiagnostic.stage = Diagnostic::Stage::Link; - outDiagnostic.text = split[3]; - - outLineParseResult = LineParseResult::Start; - return SLANG_OK; - } - } - else if (split.getCount() >= 5) - { - // Probably a regular error line - SLANG_RETURN_ON_FAIL(_parseSeverity(split[3].trim(), outDiagnostic.severity)); - - outDiagnostic.filePath = split[0]; - SLANG_RETURN_ON_FAIL(StringUtil::parseInt(split[1], outDiagnostic.fileLine)); - - // Everything from 4 to the end is the error - outDiagnostic.text = UnownedStringSlice(split[4].begin(), split.getLast().end()); - - outLineParseResult = LineParseResult::Start; - return SLANG_OK; - } - - // Assume it's a continuation - outLineParseResult = LineParseResult::Continuation; - return SLANG_OK; -} - -/* static */SlangResult GCCDownstreamCompilerUtil::parseOutput(const ExecuteResult& exeRes, DownstreamDiagnostics& outOutput) -{ - LineParseResult prevLineResult = LineParseResult::Ignore; - - outOutput.reset(); - outOutput.rawDiagnostics = exeRes.standardError; - - for (auto line : LineParser(exeRes.standardError.getUnownedSlice())) - { - Diagnostic diagnostic; - diagnostic.reset(); - - LineParseResult lineRes; - - SLANG_RETURN_ON_FAIL(_parseGCCFamilyLine(line, lineRes, diagnostic)); - - switch (lineRes) - { - case LineParseResult::Start: - { - // It's start of a new message - outOutput.diagnostics.add(diagnostic); - prevLineResult = LineParseResult::Start; - break; - } - case LineParseResult::Single: - { - // It's a single message, without anything following - outOutput.diagnostics.add(diagnostic); - prevLineResult = LineParseResult::Ignore; - break; - } - case LineParseResult::Continuation: - { - if (prevLineResult == LineParseResult::Start || prevLineResult == LineParseResult::Continuation) - { - if (outOutput.diagnostics.getCount() > 0) - { - // We are now in a continuation, add to the last - auto& text = outOutput.diagnostics.getLast().text; - text.append("\n"); - text.append(line); - } - prevLineResult = LineParseResult::Continuation; - } - break; - } - case LineParseResult::Ignore: - { - prevLineResult = lineRes; - break; - } - default: return SLANG_FAIL; - } - } - - if (outOutput.has(Diagnostic::Severity::Error) || exeRes.resultCode != 0) - { - outOutput.result = SLANG_FAIL; - } - - return SLANG_OK; -} - -/* static */ SlangResult GCCDownstreamCompilerUtil::calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath) -{ - SLANG_ASSERT(options.modulePath.getLength()); - - outPath.Clear(); - - switch (options.targetType) - { - case TargetType::SharedLibrary: - { - outPath << SharedLibrary::calcPlatformPath(options.modulePath.getUnownedSlice()); - return SLANG_OK; - } - case TargetType::Executable: - { - outPath << options.modulePath; - outPath << ProcessUtil::getExecutableSuffix(); - return SLANG_OK; - } - case TargetType::Object: - { -#if __CYGWIN__ - outPath << options.modulePath << ".obj"; -#else - // Will be .o for typical gcc targets - outPath << options.modulePath << ".o"; -#endif - return SLANG_OK; - } - } - - return SLANG_FAIL; -} - -/* static */SlangResult GCCDownstreamCompilerUtil::calcCompileProducts(const CompileOptions& options, ProductFlags flags, List& outPaths) -{ - SLANG_ASSERT(options.modulePath.getLength()); - - outPaths.clear(); - - if (flags & ProductFlag::Execution) - { - StringBuilder builder; - SLANG_RETURN_ON_FAIL(calcModuleFilePath(options, builder)); - outPaths.add(builder); - } - - return SLANG_OK; -} - -/* static */SlangResult GCCDownstreamCompilerUtil::calcArgs(const CompileOptions& options, CommandLine& cmdLine) -{ - SLANG_ASSERT(options.sourceContents.getLength() == 0); - SLANG_ASSERT(options.modulePath.getLength()); - - PlatformKind platformKind = (options.platform == PlatformKind::Unknown) ? PlatformUtil::getPlatformKind() : options.platform; - - if (options.sourceLanguage == SLANG_SOURCE_LANGUAGE_CPP) - { - cmdLine.addArg("-fvisibility=hidden"); - - // Need C++14 for partial specialization - cmdLine.addArg("-std=c++14"); - } - - // TODO(JS): Here we always set -m32 on x86. It could be argued it is only necessary when creating a shared library - // but if we create an object file, we don't know what to choose because we don't know what final usage is. - // It could also be argued that the platformKind could define the actual desired target - but as it stands - // we only have a target of 'Linux' (as opposed to Win32/64). Really it implies we need an arch enumeration too. - // - // For now we just make X86 binaries try and produce x86 compatible binaries as fixes the immediate problems. -#if SLANG_PROCESSOR_X86 - /* Used to specify the processor more broadly. For a x86 binary we need to make sure we build x86 builds - even when on an x64 system. - -m32 - -m64*/ - cmdLine.addArg("-m32"); -#endif - - switch (options.optimizationLevel) - { - case OptimizationLevel::None: - { - // No optimization - cmdLine.addArg("-O0"); - break; - } - case OptimizationLevel::Default: - { - cmdLine.addArg("-Os"); - break; - } - case OptimizationLevel::High: - { - cmdLine.addArg("-O2"); - break; - } - case OptimizationLevel::Maximal: - { - cmdLine.addArg("-O4"); - break; - } - default: break; - } - - if (options.debugInfoType != DebugInfoType::None) - { - cmdLine.addArg("-g"); - } - - if (options.flags & CompileOptions::Flag::Verbose) - { - cmdLine.addArg("-v"); - } - - switch (options.floatingPointMode) - { - case FloatingPointMode::Default: break; - case FloatingPointMode::Precise: - { - //cmdLine.addArg("-fno-unsafe-math-optimizations"); - break; - } - case FloatingPointMode::Fast: - { - // We could enable SSE with -mfpmath=sse - // But that would only make sense on a x64/x86 type processor and only if that feature is present (it is on all x64) - cmdLine.addArg("-ffast-math"); - break; - } - } - - StringBuilder moduleFilePath; - calcModuleFilePath(options, moduleFilePath); - - cmdLine.addArg("-o"); - cmdLine.addArg(moduleFilePath); - - switch (options.targetType) - { - case TargetType::SharedLibrary: - { - // Shared library - cmdLine.addArg("-shared"); - - if (PlatformUtil::isFamily(PlatformFamily::Unix, platformKind)) - { - // Position independent - cmdLine.addArg("-fPIC"); - } - break; - } - case TargetType::Executable: - { - break; - } - case TargetType::Object: - { - // Don't link, just produce object file - cmdLine.addArg("-c"); - break; - } - default: break; - } - - // Add defines - for (const auto& define : options.defines) - { - StringBuilder builder; - - builder << "-D"; - builder << define.nameWithSig; - if (define.value.getLength()) - { - builder << "=" << define.value; - } - - cmdLine.addArg(builder); - } - - // Add includes - for (const auto& include : options.includePaths) - { - cmdLine.addArg("-I"); - cmdLine.addArg(include); - } - - // Link options - if (0) // && options.targetType != TargetType::Object) - { - //linkOptions << "-Wl,"; - //cmdLine.addArg(linkOptions); - } - - if (options.targetType == TargetType::SharedLibrary) - { - if (!PlatformUtil::isFamily(PlatformFamily::Apple, platformKind)) - { - // On MacOS, this linker option is not supported. That's ok though in - // so far as on MacOS it does report any unfound symbols without the option. - - // Linker flag to report any undefined symbols as a link error - cmdLine.addArg("-Wl,--no-undefined"); - } - } - - // Files to compile - for (const auto& sourceFile : options.sourceFiles) - { - cmdLine.addArg(sourceFile); - } - - for (const auto& libPath : options.libraryPaths) - { - // Note that any escaping of the path is handled in the ProcessUtil:: - cmdLine.addArg("-L"); - cmdLine.addArg(libPath); - cmdLine.addArg("-F"); - cmdLine.addArg(libPath); - } - - if (options.sourceLanguage == SLANG_SOURCE_LANGUAGE_CPP && !PlatformUtil::isFamily(PlatformFamily::Windows, platformKind)) - { - // Make STD libs available - cmdLine.addArg("-lstdc++"); - // Make maths lib available - cmdLine.addArg("-lm"); - } - - return SLANG_OK; -} - -/* static */SlangResult GCCDownstreamCompilerUtil::createCompiler(const String& path, const String& inExeName, RefPtr& outCompiler) -{ - String exeName(inExeName); - if (path.getLength() > 0) - { - exeName = Path::combine(path, inExeName); - } - - DownstreamCompiler::Desc desc; - SLANG_RETURN_ON_FAIL(GCCDownstreamCompilerUtil::calcVersion(exeName, desc)); - - RefPtr compiler(new GCCDownstreamCompiler(desc)); - compiler->m_cmdLine.setExecutableFilename(exeName); - - outCompiler = compiler; - return SLANG_OK; -} - -/* static */SlangResult GCCDownstreamCompilerUtil::locateGCCCompilers(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set) -{ - SLANG_UNUSED(loader); - RefPtr compiler; - if (SLANG_SUCCEEDED(createCompiler(path, "g++", compiler))) - { - set->addCompiler(compiler); - } - return SLANG_OK; -} - -/* static */SlangResult GCCDownstreamCompilerUtil::locateClangCompilers(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set) -{ - SLANG_UNUSED(loader); - - RefPtr compiler; - if (SLANG_SUCCEEDED(createCompiler(path, "clang", compiler))) - { - set->addCompiler(compiler); - } - return SLANG_OK; -} - -} diff --git a/source/core/slang-gcc-compiler-util.h b/source/core/slang-gcc-compiler-util.h deleted file mode 100644 index b97144e35..000000000 --- a/source/core/slang-gcc-compiler-util.h +++ /dev/null @@ -1,56 +0,0 @@ -#ifndef SLANG_GCC_COMPILER_UTIL_H -#define SLANG_GCC_COMPILER_UTIL_H - -#include "slang-downstream-compiler.h" - -namespace Slang -{ - -/* Utility for processing input and output of gcc-like compilers, including clang */ -struct GCCDownstreamCompilerUtil : public DownstreamCompilerBaseUtil -{ - /// Extracts version number into desc from text (assumes gcc/clang -v layout with a line with version) - static SlangResult parseVersion(const UnownedStringSlice& text, const UnownedStringSlice& prefix, DownstreamCompiler::Desc& outDesc); - - /// Runs the exeName, and extracts the version info into outDesc - static SlangResult calcVersion(const String& exeName, DownstreamCompiler::Desc& outDesc); - - /// Calculate gcc family compilers (including clang) cmdLine arguments from options - static SlangResult calcArgs(const CompileOptions& options, CommandLine& cmdLine); - - /// Parse ExecuteResult into Output - static SlangResult parseOutput(const ExecuteResult& exeRes, DownstreamDiagnostics& outOutput); - - /// Calculate the output module filename - static SlangResult calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath); - - /// Given options, calculate paths to products produced for a compilation - static SlangResult calcCompileProducts(const CompileOptions& options, ProductFlags flags, List& outPaths); - - /// Given a path and an exe name, detects if compiler is present, and if so adds to compiler set. - static SlangResult createCompiler(const String& path, const String& inExeName, RefPtr& outCompiler); - - static SlangResult locateGCCCompilers(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set); - - static SlangResult locateClangCompilers(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set); - -}; - -class GCCDownstreamCompiler : public CommandLineDownstreamCompiler -{ -public: - typedef CommandLineDownstreamCompiler Super; - typedef GCCDownstreamCompilerUtil Util; - - // CommandLineCPPCompiler impl - just forwards to the Util - virtual SlangResult calcArgs(const CompileOptions& options, CommandLine& cmdLine) SLANG_OVERRIDE { return Util::calcArgs(options, cmdLine); } - virtual SlangResult parseOutput(const ExecuteResult& exeResult, DownstreamDiagnostics& output) SLANG_OVERRIDE { return Util::parseOutput(exeResult, output); } - virtual SlangResult calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath) SLANG_OVERRIDE { return Util::calcModuleFilePath(options, outPath); } - virtual SlangResult calcCompileProducts(const CompileOptions& options, ProductFlags flags, List& outPaths) SLANG_OVERRIDE { return Util::calcCompileProducts(options, flags, outPaths); } - - GCCDownstreamCompiler(const Desc& desc):Super(desc) {} -}; - -} - -#endif diff --git a/source/core/slang-name-convention-util.cpp b/source/core/slang-name-convention-util.cpp deleted file mode 100644 index a5acc6370..000000000 --- a/source/core/slang-name-convention-util.cpp +++ /dev/null @@ -1,213 +0,0 @@ - -#include "slang-name-convention-util.h" - -#include "slang-char-util.h" -#include "slang-string-util.h" - -namespace Slang -{ - -/* static */NameConvention NameConventionUtil::getConvention(const UnownedStringSlice& slice) -{ - for (const char c : slice) - { - switch (c) - { - case '-': return NameConvention::Kabab; - case '_': return NameConvention::Snake; - default: break; - } - } - return NameConvention::Camel; -} - -/* static */void NameConventionUtil::split(NameConvention convention, const UnownedStringSlice& slice, List& out) -{ - switch (convention) - { - case NameConvention::Kabab: - { - StringUtil::split(slice, '-', out); - break; - } - case NameConvention::Snake: - { - StringUtil::split(slice, '_', out); - break; - } - case NameConvention::Camel: - { - typedef CharUtil::Flags CharFlags; - typedef CharUtil::Flag CharFlag; - - CharFlags prevFlags = 0; - const char*const end = slice.end(); - - const char* start = slice.begin(); - for (const char* cur = start; cur < end; ++cur) - { - const char c = *cur; - const CharUtil::Flags flags = CharUtil::getFlags(c); - - if (flags & CharFlag::Upper) - { - if (prevFlags & CharFlag::Lower) - { - // If we go from lower to upper, we have a transition - out.add(UnownedStringSlice(start, cur)); - start = cur; - } - else if ((prevFlags & CharFlag::Upper) && cur + 1 < end) - { - // This works with capital or uncapitalized acronyms, but if we have two capitalized acronyms following each other - it can't split. - // - // For example - // "IAABBSystem" -> "IAABB", "System" - // - // If it only accepted lower case acronyms the logic could be changed such that the following could be produced - // "IAabbSystem" -> "I", "Aabb", "System" - // - // Since Slang source largely goes with upper case acronyms, we work with the heuristic here.. - - if (CharUtil::isLower(cur[1])) - { - out.add(UnownedStringSlice(start, cur)); - start = cur; - } - } - } - - prevFlags = flags; - } - - // Add any end section - if (start < end) - { - out.add(UnownedStringSlice(start, end)); - } - break; - } - } -} - -void NameConventionUtil::split(const UnownedStringSlice& slice, List& out) -{ - split(getConvention(slice), slice, out); -} - -/* static */void NameConventionUtil::join(const UnownedStringSlice* slices, Index slicesCount, CharCase charCase, char joinChar, StringBuilder& out) -{ - if (slicesCount <= 0) - { - return; - } - - Index totalSize = slicesCount - 1; - for (Index i = 0; i < slicesCount; ++i) - { - totalSize += slices[i].getLength(); - } - - char*const dstStart = out.prepareForAppend(totalSize); - char* dst = dstStart; - - for (Index i = 0; i < slicesCount; ++i) - { - const UnownedStringSlice& slice = slices[i]; - const Index count = slice.getLength(); - const char*const src = slice.begin(); - - if (i > 0) - { - *dst++ = joinChar; - } - - switch (charCase) - { - case CharCase::Upper: - { - for (Index j = 0; j < count; ++j) - { - dst[j] = CharUtil::toUpper(src[j]); - } - break; - } - case CharCase::Lower: - { - for (Index j = 0; j < count; ++j) - { - dst[j] = CharUtil::toLower(src[j]); - } - break; - } - } - - dst += count; - } - - SLANG_ASSERT(dstStart + totalSize == dst); - out.appendInPlace(dstStart, totalSize); -} - -/* static */void NameConventionUtil::join(const UnownedStringSlice* slices, Index slicesCount, CharCase charCase, NameConvention convention, StringBuilder& out) -{ - switch (convention) - { - case NameConvention::Kabab: return join(slices, slicesCount, charCase, '-', out); - case NameConvention::Snake: return join(slices, slicesCount, charCase, '_', out); - case NameConvention::Camel: - { - Index totalSize = 0; - - for (Index i = 0; i < slicesCount; ++i) - { - totalSize += slices[i].getLength(); - } - - char*const dstStart = out.prepareForAppend(totalSize); - char* dst = dstStart; - - for (Index i = 0; i < slicesCount; ++i) - { - const UnownedStringSlice& slice = slices[i]; - Index count = slice.getLength(); - const char* src = slice.begin(); - - Int j = 0; - - if (count > 0 && !(i == 0 && charCase == CharCase::Lower)) - { - // Capitalize first letter of each word, unless on first word and 'lower' - dst[j] = CharUtil::toUpper(src[j]); - j++; - } - - for (; j < count; ++j) - { - dst[j] = CharUtil::toLower(src[j]); - } - - dst += count; - } - break; - } - } -} - -/* static */void NameConventionUtil::convert(NameConvention fromConvention, const UnownedStringSlice& slice, CharCase charCase, NameConvention toConvention, StringBuilder& out) -{ - // Split into slices - List slices; - split(fromConvention, slice, slices); - - // Join the slices in the toConvention - join(slices.getBuffer(), slices.getCount(), charCase, toConvention, out); -} - -/* static */void NameConventionUtil::convert(const UnownedStringSlice& slice, CharCase charCase, NameConvention toConvention, StringBuilder& out) -{ - convert(getConvention(slice), slice, charCase, toConvention, out); -} - -} - diff --git a/source/core/slang-name-convention-util.h b/source/core/slang-name-convention-util.h deleted file mode 100644 index d4a984ca0..000000000 --- a/source/core/slang-name-convention-util.h +++ /dev/null @@ -1,54 +0,0 @@ -#ifndef SLANG_CORE_NAME_CONVENTION_UTIL_H -#define SLANG_CORE_NAME_CONVENTION_UTIL_H - -#include "slang-string.h" -#include "slang-list.h" - -namespace Slang -{ - -enum class NameConvention -{ - Kabab, /// Words are separated with -. WORDS-ARE-SEPARATED - Snake, /// Words are separated with _. WORDS_ARE_SEPARATED - Camel, /// Words start with a capital. (Upper will make first words character capitalized, aka PascalCase) -}; - -enum class CharCase -{ - Upper, - Lower, -}; - -/* This utility is to enable easy conversion and interpretation of names that use standard conventions, typically in programming -languages. The conventions are largely how to represent multiple words together. - -Split is used to split up a name into it's constituent 'words' based on a convention. -Join is used to combine words based on a convention/character case - -Convert uses split and join to allow easy conversion between conventions. -*/ -struct NameConventionUtil -{ - /// Given a slice tries to determine the convention used. - /// If no separators are found, will assume Camel - static NameConvention getConvention(const UnownedStringSlice& slice); - - /// Given a slice and a naming convention, split into it's constituent parts. If convention isn't specified, will infer from slice using getConvention. - static void split(NameConvention convention, const UnownedStringSlice& slice, List& out); - static void split(const UnownedStringSlice& slice, List& out); - - /// Given slices, join together with the specified convention into out - static void join(const UnownedStringSlice* slices, Index slicesCount, CharCase charCase, NameConvention convention, StringBuilder& out); - - /// Join with a join char, and potentially changing case of input slices - static void join(const UnownedStringSlice* slices, Index slicesCount, CharCase charCase, char joinChar, StringBuilder& out); - - /// Convert from one convention to another. If fromConvention isn't specified, will infer from slice using getConvention. - static void convert(NameConvention fromConvention, const UnownedStringSlice& slice, CharCase charCase, NameConvention toConvention, StringBuilder& out); - static void convert(const UnownedStringSlice& slice, CharCase charCase, NameConvention toConvention, StringBuilder& out); -}; - -} - -#endif // SLANG_CORE_NAME_CONVENTION_UTIL_H diff --git a/source/core/slang-nvrtc-compiler.cpp b/source/core/slang-nvrtc-compiler.cpp deleted file mode 100644 index eb117379f..000000000 --- a/source/core/slang-nvrtc-compiler.cpp +++ /dev/null @@ -1,773 +0,0 @@ -// slang-nvrtc-compiler.cpp -#include "slang-nvrtc-compiler.h" - -#include "slang-common.h" -#include "../../slang-com-helper.h" - -#include "../core/slang-blob.h" - -#include "slang-string-util.h" -#include "slang-string-slice-pool.h" - -#include "slang-io.h" -#include "slang-shared-library.h" -#include "slang-semantic-version.h" - - -namespace nvrtc -{ - -typedef enum { - NVRTC_SUCCESS = 0, - NVRTC_ERROR_OUT_OF_MEMORY = 1, - NVRTC_ERROR_PROGRAM_CREATION_FAILURE = 2, - NVRTC_ERROR_INVALID_INPUT = 3, - NVRTC_ERROR_INVALID_PROGRAM = 4, - NVRTC_ERROR_INVALID_OPTION = 5, - NVRTC_ERROR_COMPILATION = 6, - NVRTC_ERROR_BUILTIN_OPERATION_FAILURE = 7, - NVRTC_ERROR_NO_NAME_EXPRESSIONS_AFTER_COMPILATION = 8, - NVRTC_ERROR_NO_LOWERED_NAMES_BEFORE_COMPILATION = 9, - NVRTC_ERROR_NAME_EXPRESSION_NOT_VALID = 10, - NVRTC_ERROR_INTERNAL_ERROR = 11 -} nvrtcResult; - -typedef struct _nvrtcProgram *nvrtcProgram; - -#define SLANG_NVRTC_FUNCS(x) \ - x(const char*, nvrtcGetErrorString, (nvrtcResult result)) \ - x(nvrtcResult, nvrtcVersion, (int *major, int *minor)) \ - x(nvrtcResult, nvrtcCreateProgram, (nvrtcProgram *prog, const char *src, const char *name, int numHeaders, const char * const *headers, const char * const *includeNames)) \ - x(nvrtcResult, nvrtcDestroyProgram, (nvrtcProgram *prog)) \ - x(nvrtcResult, nvrtcCompileProgram, (nvrtcProgram prog, int numOptions, const char * const *options)) \ - x(nvrtcResult, nvrtcGetPTXSize, (nvrtcProgram prog, size_t *ptxSizeRet)) \ - x(nvrtcResult, nvrtcGetPTX, (nvrtcProgram prog, char *ptx)) \ - x(nvrtcResult, nvrtcGetProgramLogSize, (nvrtcProgram prog, size_t *logSizeRet)) \ - x(nvrtcResult, nvrtcGetProgramLog, (nvrtcProgram prog, char *log))\ - x(nvrtcResult, nvrtcAddNameExpression, (nvrtcProgram prog, const char * const name_expression)) \ - x(nvrtcResult, nvrtcGetLoweredName, (nvrtcProgram prog, const char *const name_expression, const char** lowered_name)) - -} // namespace nvrtc - -namespace Slang -{ -using namespace nvrtc; - -static SlangResult _asResult(nvrtcResult res) -{ - switch (res) - { - case NVRTC_SUCCESS: - { - return SLANG_OK; - } - case NVRTC_ERROR_OUT_OF_MEMORY: - { - return SLANG_E_OUT_OF_MEMORY; - } - case NVRTC_ERROR_PROGRAM_CREATION_FAILURE: - case NVRTC_ERROR_INVALID_INPUT: - case NVRTC_ERROR_INVALID_PROGRAM: - { - return SLANG_FAIL; - } - case NVRTC_ERROR_INVALID_OPTION: - { - return SLANG_E_INVALID_ARG; - } - case NVRTC_ERROR_COMPILATION: - case NVRTC_ERROR_BUILTIN_OPERATION_FAILURE: - case NVRTC_ERROR_NO_NAME_EXPRESSIONS_AFTER_COMPILATION: - case NVRTC_ERROR_NO_LOWERED_NAMES_BEFORE_COMPILATION: - case NVRTC_ERROR_NAME_EXPRESSION_NOT_VALID: - { - return SLANG_FAIL; - } - case NVRTC_ERROR_INTERNAL_ERROR: - { - return SLANG_E_INTERNAL_FAIL; - } - default: return SLANG_FAIL; - } -} - -class NVRTCDownstreamCompiler : public DownstreamCompiler -{ -public: - typedef DownstreamCompiler Super; - - // DownstreamCompiler - virtual SlangResult compile(const CompileOptions& options, RefPtr& outResult) SLANG_OVERRIDE; - virtual ISlangSharedLibrary* getSharedLibrary() SLANG_OVERRIDE { return m_sharedLibrary; } - - /// Must be called before use - SlangResult init(ISlangSharedLibrary* library); - - NVRTCDownstreamCompiler() {} - -protected: - - struct ScopeProgram - { - ScopeProgram(NVRTCDownstreamCompiler* compiler, nvrtcProgram program): - m_compiler(compiler), - m_program(program) - { - } - ~ScopeProgram() - { - m_compiler->m_nvrtcDestroyProgram(&m_program); - } - NVRTCDownstreamCompiler* m_compiler; - nvrtcProgram m_program; - }; - - -#define SLANG_NVTRC_MEMBER_FUNCS(ret, name, params) \ - ret (*m_##name) params; - - SLANG_NVRTC_FUNCS(SLANG_NVTRC_MEMBER_FUNCS); - - ComPtr m_sharedLibrary; -}; - -#define SLANG_NVRTC_RETURN_ON_FAIL(x) { nvrtcResult _res = x; if (_res != NVRTC_SUCCESS) return _asResult(_res); } - -SlangResult NVRTCDownstreamCompiler::init(ISlangSharedLibrary* library) -{ -#define SLANG_NVTRC_GET_FUNC(ret, name, params) \ - m_##name = (ret (*) params)library->findFuncByName(#name); \ - if (m_##name == nullptr) return SLANG_FAIL; - - SLANG_NVRTC_FUNCS(SLANG_NVTRC_GET_FUNC) - - m_sharedLibrary = library; - - m_desc.type = SLANG_PASS_THROUGH_NVRTC; - - int major, minor; - m_nvrtcVersion(&major, &minor); - m_desc.majorVersion = major; - m_desc.minorVersion = minor; - - return SLANG_OK; -} - -static SlangResult _parseLocation(const UnownedStringSlice& in, DownstreamDiagnostic& outDiagnostic) -{ - const Index startIndex = in.indexOf('('); - - if (startIndex >= 0) - { - outDiagnostic.filePath = UnownedStringSlice(in.begin(), in.begin() + startIndex); - UnownedStringSlice remaining(in.begin() + startIndex + 1, in.end()); - const Int endIndex = remaining.indexOf(')'); - - UnownedStringSlice lineText = UnownedStringSlice(remaining.begin(), remaining.begin() + endIndex); - - Int line; - SLANG_RETURN_ON_FAIL(StringUtil::parseInt(lineText, line)); - outDiagnostic.fileLine = line; - } - else - { - outDiagnostic.fileLine = 0; - outDiagnostic.filePath = in; - } - return SLANG_OK; -} - -static bool _isDriveLetter(char c) -{ - return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); -} - -static bool _hasDriveLetter(const UnownedStringSlice& line) -{ - return line.getLength() > 2 && line[1] == ':' && _isDriveLetter(line[0]); -} - -static SlangResult _parseNVRTCLine(const UnownedStringSlice& line, DownstreamDiagnostic& outDiagnostic) -{ - typedef DownstreamDiagnostic Diagnostic; - typedef Diagnostic::Severity Severity; - - outDiagnostic.stage = Diagnostic::Stage::Compile; - - List split; - if (_hasDriveLetter(line)) - { - // The drive letter has :, which confuses things, so skip that and then fix up first entry - UnownedStringSlice lineWithoutDrive(line.begin() + 2, line.end()); - StringUtil::split(lineWithoutDrive, ':', split); - split[0] = UnownedStringSlice(line.begin(), split[0].end()); - } - else - { - StringUtil::split(line, ':', split); - } - - if (split.getCount() >= 3) - { - // tests/cuda/cuda-compile.cu(7): warning: variable "c" is used before its value is set - - const auto split1 = split[1].trim(); - - if (split1 == "error") - { - outDiagnostic.severity = Severity::Error; - } - else if (split1 == "warning") - { - outDiagnostic.severity = Severity::Warning; - } - outDiagnostic.text = split[2].trim(); - - SLANG_RETURN_ON_FAIL(_parseLocation(split[0], outDiagnostic)); - return SLANG_OK; - } - - return SLANG_E_NOT_FOUND; -} - -SlangResult NVRTCDownstreamCompiler::compile(const CompileOptions& options, RefPtr& outResult) -{ - // This compiler doesn't read files, they should be read externally and stored in sourceContents/sourceContentsPath - if (options.sourceFiles.getCount() > 0) - { - return SLANG_FAIL; - } - - CommandLine cmdLine; - - switch (options.debugInfoType) - { - case DebugInfoType::None: - { - break; - } - default: - { - cmdLine.addArg("--device-debug"); - break; - } - case DebugInfoType::Maximal: - { - cmdLine.addArg("--device-debug"); - cmdLine.addArg("--generate-line-info"); - break; - } - } - - // Don't seem to have such a control, so ignore for now - //switch (options.optimizationLevel) - //{ - // default: break; - //} - - switch (options.floatingPointMode) - { - case FloatingPointMode::Default: break; - case FloatingPointMode::Precise: - { - break; - } - case FloatingPointMode::Fast: - { - cmdLine.addArg("--use_fast_math"); - break; - } - } - - // Add defines - for (const auto& define : options.defines) - { - StringBuilder builder; - builder << "-D"; - builder << define.nameWithSig; - if (define.value.getLength()) - { - builder << "=" << define.value; - } - - cmdLine.addArg(builder); - } - - // Add includes - for (const auto& include : options.includePaths) - { - cmdLine.addArg("-I"); - cmdLine.addArg(include); - } - - // Neither of these options are strictly required, for general use of nvrtc, - // but are enabled to make use withing Slang work more smoothly - { - // Require c++14, as makes initialization construction with {} available and so simplifies code generation - cmdLine.addArg("-std=c++14"); - - // Disable all warnings - // This is arguably too much - but nvrtc does not appear to have a mechanism to switch off individual warnings. - // I tried the -Xcudafe mechanism but that does not appear to work for nvrtc - cmdLine.addArg("-w"); - } - - { - // The lowest supported CUDA architecture version supported - // by NVRTC is `compute_30`. - // - SemanticVersion version(3); - - // Newer releases of NVRTC only support `compute_35` and up - // (with everything before `compute_52` being deprecated). - // - if( m_desc.majorVersion >= 11 ) - { - version = SemanticVersion(3, 5); - } - - // If constructs used in the code to be compield require - // a higher architecture version than the minimum, then - // we will set the version to the highest version listed - // among the requirements. - // - for (const auto& capabilityVersion : options.requiredCapabilityVersions) - { - if (capabilityVersion.kind == DownstreamCompiler::CapabilityVersion::Kind::CUDASM) - { - if (capabilityVersion.version > version) - { - version = capabilityVersion.version; - } - } - } - - StringBuilder builder; - builder << "-arch=compute_"; - builder << version.m_major; - - SLANG_ASSERT(version.m_minor >= 0 && version.m_minor <= 9); - builder << char('0' + version.m_minor); - - cmdLine.addArg(builder); - } - - List headers; - List headerIncludeNames; - - // If compiling for OptiX, we need to add the appropriate search paths to the command line. - // - if(options.pipelineType == PipelineType::RayTracing) - { - // The device-side OptiX API is accessed through a constellation - // of headers provided by the OptiX SDK, so we need to set an - // include path for the compile that makes those visible. - // - // TODO: The OptiX SDK installer doesn't set any kind of environment - // variable to indicate where the SDK was installed, so we seemingly - // need to probe paths instead. The form of the path will differ - // betwene Windows and Unix-y platforms, and we will need some kind - // of approach to probe multiple versiosn and use the latest. - // - // HACK: For now I'm using the fixed path for the most recent SDK - // release on Windows. This means that OptiX cross-compilation will - // only "work" on a subset of platforms, but that doesn't matter - // for now since it doesn't really "work" at all. - // - cmdLine.addArg("-I"); - cmdLine.addArg("C:/ProgramData/NVIDIA Corporation/OptiX SDK 7.0.0/include/"); - - // The OptiX headers in turn `#include ` and expect that - // to work. We could try to also add in an include path from the CUDA - // SDK (which seems to provide a `stddef.h` in the most recent version), - // but using that version doesn't seem to work (and also bakes in a - // requirement that the user have the CUDA SDK installed in addition - // to the OptiX SDK). - // - // Instead, we will rely on the NVRTC feature that lets us set up - // memory buffers to be used as include files by the we compile. - // We will define a dummy `stddef.h` that includes the bare minimum - // lines required to get the OptiX headers to compile without complaint. - // - // TODO: Confirm that the `LP64` definition herei s actually needed. - // - headerIncludeNames.add("stddef.h"); - headers.add("#pragma once\n" "#define LP64\n"); - - // Finally, we want the CUDA prelude to be able to react to whether - // or not OptiX is required (most notably by `#include`ing the appropriate - // header(s)), so we will insert a preprocessor define to indicate - // the requirement. - // - cmdLine.addArg("-DSLANG_CUDA_ENABLE_OPTIX"); - } - - SLANG_ASSERT(headers.getCount() == headerIncludeNames.getCount()); - - nvrtcProgram program = nullptr; - nvrtcResult res = m_nvrtcCreateProgram(&program, options.sourceContents.getBuffer(), options.sourceContentsPath.getBuffer(), - (int) headers.getCount(), - headers.getBuffer(), - headerIncludeNames.getBuffer()); - if (res != NVRTC_SUCCESS) - { - return _asResult(res); - } - ScopeProgram scope(this, program); - - List dstOptions; - dstOptions.setCount(cmdLine.m_args.getCount()); - for (Index i = 0; i < cmdLine.m_args.getCount(); ++i) - { - dstOptions[i] = cmdLine.m_args[i].value.getBuffer(); - } - - res = m_nvrtcCompileProgram(program, int(dstOptions.getCount()), dstOptions.getBuffer()); - - RefPtr blob; - DownstreamDiagnostics diagnostics; - - diagnostics.result = _asResult(res); - - { - String rawDiagnostics; - - size_t logSize = 0; - SLANG_NVRTC_RETURN_ON_FAIL(m_nvrtcGetProgramLogSize(program, &logSize)); - - if (logSize) - { - char* dst = rawDiagnostics.prepareForAppend(Index(logSize)); - SLANG_NVRTC_RETURN_ON_FAIL(m_nvrtcGetProgramLog(program, dst)); - rawDiagnostics.appendInPlace(dst, Index(logSize)); - - diagnostics.rawDiagnostics = rawDiagnostics; - } - - // Parse the diagnostics here - for (auto line : LineParser(diagnostics.rawDiagnostics.getUnownedSlice())) - { - DownstreamDiagnostic diagnostic; - SlangResult lineRes = _parseNVRTCLine(line, diagnostic); - - if (SLANG_SUCCEEDED(lineRes)) - { - diagnostics.diagnostics.add(diagnostic); - } - else if (lineRes != SLANG_E_NOT_FOUND) - { - return lineRes; - } - } - - // if it has a compilation error.. set on output - if (diagnostics.has(DownstreamDiagnostic::Severity::Error)) - { - diagnostics.result = SLANG_FAIL; - } - } - - if (res == nvrtc::NVRTC_SUCCESS) - { - // We should parse the log to set up the diagnostics - size_t ptxSize; - SLANG_NVRTC_RETURN_ON_FAIL(m_nvrtcGetPTXSize(program, &ptxSize)); - - List ptx; - ptx.setCount(Index(ptxSize)); - - SLANG_NVRTC_RETURN_ON_FAIL(m_nvrtcGetPTX(program, (char*)ptx.getBuffer())); - - blob = ListBlob::moveCreate(ptx); - } - - outResult = new BlobDownstreamCompileResult(diagnostics, blob); - - return SLANG_OK; -} - -/* An implementation of Path::Visitor that can be used for finding NVRTC shared library installations. */ -struct NVRTCPathVisitor : Path::Visitor -{ - struct Candidate - { - typedef Candidate ThisType; - - bool operator==(const ThisType& rhs) const { return path == rhs.path && version == rhs.version; } - bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } - - static Candidate make(const String& path, const SemanticVersion& version) - { - Candidate can; - can.version = version; - can.path = path; - return can; - } - String path; - SemanticVersion version; - }; - - Index findVersion(const SemanticVersion& version) const - { - const Index count = m_candidates.getCount(); - for (Index i = 0; i < count; ++i) - { - if (m_candidates[i].version == version) - { - return i; - } - } - return -1; - } - - static bool _orderCandiate(const Candidate& a, const Candidate& b) { return a.version < b.version; } - void sortCandidates() { m_candidates.sort(_orderCandiate); } - - void accept(Path::Type type, const UnownedStringSlice& filename) SLANG_OVERRIDE - { - // Lets make sure it start's with nvrtc64, but not worry about case - if (type == Path::Type::File) - { - // If there is a defined extension, make sure it has it - if (m_postfix.getLength() && filename.getLength() >= m_postfix.getLength()) - { - // We test without case - really for windows - UnownedStringSlice filenamePostfix = filename.tail(filename.getLength() - m_postfix.getLength()); - if (!filenamePostfix.caseInsensitiveEquals(m_postfix.getUnownedSlice())) - { - return; - } - } - - if (filename.getLength() >= m_prefix.getLength() && - filename.subString(0, m_prefix.getLength()).caseInsensitiveEquals(m_prefix.getUnownedSlice())) - { - // Versions are typically (on windows) of the form - // nvrtc64_110_2.dll - // 11 - Major - // 0 Minor - // 2 Patch - Index endIndex = filename.indexOf('.'); - endIndex = (endIndex < 0) ? filename.getLength() : endIndex; - - UnownedStringSlice versionSlice = UnownedStringSlice(filename.begin() + m_prefix.getLength(), filename.begin() + endIndex); - - Int patch = 0; - UnownedStringSlice majorMinorSlice; - { - List slices; - StringUtil::split(versionSlice, '_', slices); - if (slices.getCount() >= 2) - { - // We don't bother checking for error here, if it's not parsable, it will be 0 - StringUtil::parseInt(slices[1], patch); - } - majorMinorSlice = slices[0]; - } - - if (majorMinorSlice.getLength() < 2) - { - // Must be a major and minor - return; - } - - UnownedStringSlice majorSlice = majorMinorSlice.head(majorMinorSlice.getLength() - 1); - UnownedStringSlice minorSlice = majorMinorSlice.subString(majorMinorSlice.getLength() - 1, 1); - - Int major; - Int minor; - - if (SLANG_FAILED(StringUtil::parseInt(majorSlice, major)) || - SLANG_FAILED(StringUtil::parseInt(minorSlice, minor))) - { - return; - } - - const SemanticVersion version = SemanticVersion(int(major), int(minor), int(patch)); - - // We may want to add multiple versions, if they are in different locations - as there may be multiple entries - // in the PATH, and only one works. We'll only know which works by loading -#if 0 - // We already found this version, so let's not add it again - if (findVersion(version) >= 0) - { - return; - } -#endif - - // Strip to make a shared library name - UnownedStringSlice sharedLibraryName = filename.tail(m_prefix.getLength() - m_sharedLibraryStem.getLength()); - sharedLibraryName = filename.head(filename.getLength() - m_postfix.getLength()); - - auto candidate = Candidate::make(Path::combine(m_basePath, sharedLibraryName), version); - - // If we already have this candidate, then skip - if (m_candidates.indexOf(candidate) >= 0) - { - return; - } - - // Add to the list of candidates - m_candidates.add(candidate); - } - } - } - - SlangResult findInDirectory(const String& path) - { - m_basePath = path; - return Path::find(path, nullptr, this); - } - - bool hasCandidates() const { return m_candidates.getCount() > 0; } - - NVRTCPathVisitor(const UnownedStringSlice& sharedLibraryStem): - m_sharedLibraryStem(sharedLibraryStem) - { - // Work out the prefix and postfix of the shader - StringBuilder buf; - SharedLibrary::appendPlatformFileName(sharedLibraryStem, buf); - const Index index = buf.indexOf(sharedLibraryStem); - SLANG_ASSERT(index >= 0); - - m_prefix = buf.getUnownedSlice().head(index + sharedLibraryStem.getLength()); - m_postfix = buf.getUnownedSlice().tail(index + sharedLibraryStem.getLength()); - } - - String m_prefix; - String m_postfix; - String m_basePath; - String m_sharedLibraryStem; - - List m_candidates; -}; - -static SlangResult _findAndLoadNVRTC(ISlangSharedLibraryLoader* loader, ComPtr& outLibrary) -{ -#if SLANG_WINDOWS_FAMILY - // We only need to search 64 bit versions on windows - NVRTCPathVisitor visitor(UnownedStringSlice::fromLiteral("nvrtc64_")); - - // First try the instance path (if supported on platform) - { - StringBuilder instancePath; - if (SLANG_SUCCEEDED(PlatformUtil::getInstancePath(instancePath))) - { - visitor.findInDirectory(instancePath); - } - } - - // If we don't have a candidate try CUDA_PATH - if (!visitor.hasCandidates()) - { - StringBuilder buf; - if (!SLANG_SUCCEEDED(PlatformUtil::getEnvironmentVariable(UnownedStringSlice::fromLiteral("CUDA_PATH"), buf))) - { - // Look for candidates in the directory - visitor.findInDirectory(Path::combine(buf, "bin")); - } - } - - // If we haven't we go searching through PATH - if (!visitor.hasCandidates()) - { - List splitPath; - - StringBuilder buf; - if (SLANG_SUCCEEDED(PlatformUtil::getEnvironmentVariable(UnownedStringSlice::fromLiteral("PATH"), buf))) - { - // Split so we get individual paths - List paths; - StringUtil::split(buf.getUnownedSlice(), ';', paths); - - // We use a pool to make sure we only check each path once - StringSlicePool pool(StringSlicePool::Style::Empty); - - // We are going to search the paths in order - for (const auto path : paths) - { - // PATH can have the same path multiple times. If we have already searched this path, we don't need to again - if (!pool.has(path)) - { - pool.add(path); - - Path::split(path, splitPath); - - // We could search every path, but here we restrict to paths that look like CUDA installations. - // It's a path that contains a CUDA directory and has bin - if (splitPath.indexOf("CUDA") >= 0 && splitPath[splitPath.getCount() - 1].caseInsensitiveEquals(UnownedStringSlice::fromLiteral("bin"))) - { - // Okay lets search it - visitor.findInDirectory(path); - } - } - } - } - } - - // Put into version order with oldest first. - visitor.sortCandidates(); - - // We want to start with the newest version... - for (Index i = visitor.m_candidates.getCount() - 1; i >= 0; --i) - { - const auto& candidate = visitor.m_candidates[i]; - if (SLANG_SUCCEEDED(loader->loadSharedLibrary(candidate.path.getBuffer(), outLibrary.writeRef()))) - { - return SLANG_OK; - } - } -#endif - // This is an official-ish list of versions is here: - // https://developer.nvidia.com/cuda-toolkit-archive - - // Filenames for NVRTC - // https://docs.nvidia.com/cuda/nvrtc/index.html - // - // From this it appears on platforms other than windows the SharedLibrary name - // should be nvrtc which is already tried, so we can give up now. - return SLANG_E_NOT_FOUND; -} - -/* static */SlangResult NVRTCDownstreamCompilerUtil::locateCompilers(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set) -{ - ComPtr library; - - // If the user supplies a path to their preferred version of NVRTC, - // we just use this. - if (path.getLength() != 0) - { - SLANG_RETURN_ON_FAIL(loader->loadSharedLibrary(path.getBuffer(), library.writeRef())); - } - else - { - // As a catch-all for non-Windows platforms, we search for - // a library simply named `nvrtc` (well, `libnvrtc`) which - // is expected to match whatever the user has installed. - // - // On Windows an installation could place the version of nvrtc it uses in the same directory - // as the slang binary, such that it's loaded. - // Using this name also allows a ISlangSharedLibraryLoader to easily identify what is required - // and perhaps load a specific version - if (SLANG_FAILED(loader->loadSharedLibrary("nvrtc", library.writeRef()))) - { - // Try something more sophisticated to locate NVRTC - SLANG_RETURN_ON_FAIL(_findAndLoadNVRTC(loader, library)); - } - } - - SLANG_ASSERT(library); - if (!library) - { - return SLANG_FAIL; - } - - RefPtr compiler(new NVRTCDownstreamCompiler); - SLANG_RETURN_ON_FAIL(compiler->init(library)); - - set->addCompiler(compiler); - return SLANG_OK; -} - - -} diff --git a/source/core/slang-nvrtc-compiler.h b/source/core/slang-nvrtc-compiler.h deleted file mode 100644 index 48c6d4da6..000000000 --- a/source/core/slang-nvrtc-compiler.h +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef SLANG_NVRTC_COMPILER_UTIL_H -#define SLANG_NVRTC_COMPILER_UTIL_H - -#include "slang-downstream-compiler.h" - -#include "../core/slang-platform.h" - -namespace Slang -{ - - -struct NVRTCDownstreamCompilerUtil -{ - static SlangResult locateCompilers(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set); -}; - -} - -#endif diff --git a/source/core/slang-string.cpp b/source/core/slang-string.cpp index 62bc19754..0a5b7d260 100644 --- a/source/core/slang-string.cpp +++ b/source/core/slang-string.cpp @@ -5,6 +5,12 @@ namespace Slang { + // HACK! + // JS: Many of the inlined functions of CharUtil just access a global map. That referencing this global is *NOT* enough to + // link correctly with CharUtil on linux for a shared library. The following call exists to try and force linkage of CharUtil + // for anything that uses core + static const auto s_charUtilLink = CharUtil::_ensureLink(); + // TODO: this belongs in a different file: SLANG_RETURN_NEVER void signalUnexpectedError(char const* message) diff --git a/source/core/slang-visual-studio-compiler-util.cpp b/source/core/slang-visual-studio-compiler-util.cpp deleted file mode 100644 index a3578483f..000000000 --- a/source/core/slang-visual-studio-compiler-util.cpp +++ /dev/null @@ -1,452 +0,0 @@ -// slang-visual-studio-compiler-util.cpp -#include "slang-visual-studio-compiler-util.h" - -#include "slang-common.h" -#include "../../slang-com-helper.h" -#include "slang-string-util.h" - -// if Visual Studio import the visual studio platform specific header -#if SLANG_VC -# include "windows/slang-win-visual-studio-util.h" -#endif - -#include "slang-io.h" - -namespace Slang -{ - -/* static */ SlangResult VisualStudioCompilerUtil::calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath) -{ - SLANG_ASSERT(options.modulePath.getLength()); - - outPath.Clear(); - - switch (options.targetType) - { - case TargetType::SharedLibrary: - { - outPath << options.modulePath << ".dll"; - return SLANG_OK; - } - case TargetType::Executable: - { - outPath << options.modulePath << ".exe"; - return SLANG_OK; - } - case TargetType::Object: - { - outPath << options.modulePath << ".obj"; - return SLANG_OK; - } - default: break; - } - - return SLANG_FAIL; -} - -/* static */SlangResult VisualStudioCompilerUtil::calcCompileProducts(const CompileOptions& options, ProductFlags flags, List& outPaths) -{ - SLANG_ASSERT(options.modulePath.getLength()); - - outPaths.clear(); - - if (flags & ProductFlag::Execution) - { - StringBuilder builder; - SLANG_RETURN_ON_FAIL(calcModuleFilePath(options, builder)); - outPaths.add(builder); - } - if (flags & ProductFlag::Miscellaneous) - { - outPaths.add(options.modulePath + ".ilk"); - - if (options.targetType == TargetType::SharedLibrary) - { - outPaths.add(options.modulePath + ".exp"); - outPaths.add(options.modulePath + ".lib"); - } - } - if (flags & ProductFlag::Compile) - { - outPaths.add(options.modulePath + ".obj"); - } - if (flags & ProductFlag::Debug) - { - // TODO(JS): Could try and determine based on debug information - outPaths.add(options.modulePath + ".pdb"); - } - - return SLANG_OK; -} - -/* static */SlangResult VisualStudioCompilerUtil::calcArgs(const CompileOptions& options, CommandLine& cmdLine) -{ - SLANG_ASSERT(options.sourceContents.getLength() == 0); - SLANG_ASSERT(options.modulePath.getLength()); - - // https://docs.microsoft.com/en-us/cpp/build/reference/compiler-options-listed-alphabetically?view=vs-2019 - - cmdLine.addArg("/nologo"); - // Generate complete debugging information - cmdLine.addArg("/Zi"); - // Display full path of source files in diagnostics - cmdLine.addArg("/FC"); - - if (options.flags & CompileOptions::Flag::EnableExceptionHandling) - { - if (options.sourceLanguage == SLANG_SOURCE_LANGUAGE_CPP) - { - // https://docs.microsoft.com/en-us/cpp/build/reference/eh-exception-handling-model?view=vs-2019 - // Assumes c functions cannot throw - cmdLine.addArg("/EHsc"); - } - } - - if (options.flags & CompileOptions::Flag::Verbose) - { - // Doesn't appear to be a VS equivalent - } - - if (options.flags & CompileOptions::Flag::EnableSecurityChecks) - { - cmdLine.addArg("/GS"); - } - else - { - cmdLine.addArg("/GS-"); - } - - switch (options.debugInfoType) - { - default: - { - // Multithreaded statically linked runtime library - cmdLine.addArg("/MD"); - break; - } - case DebugInfoType::None: - { - break; - } - case DebugInfoType::Maximal: - { - // Multithreaded statically linked *debug* runtime library - cmdLine.addArg("/MDd"); - break; - } - } - - // /Fd - followed by name of the pdb file - if (options.debugInfoType != DebugInfoType::None) - { - cmdLine.addPrefixPathArg("/Fd", options.modulePath, ".pdb"); - } - - switch (options.optimizationLevel) - { - case OptimizationLevel::None: - { - // No optimization - cmdLine.addArg("/Od"); - break; - } - case OptimizationLevel::Default: - { - break; - } - case OptimizationLevel::High: - { - cmdLine.addArg("/O2"); - break; - } - case OptimizationLevel::Maximal: - { - cmdLine.addArg("/Ox"); - break; - } - default: break; - } - - switch (options.floatingPointMode) - { - case FloatingPointMode::Default: break; - case FloatingPointMode::Precise: - { - // precise is default behavior, VS also has 'strict' - // - // ```/fp:strict has behavior similar to /fp:precise, that is, the compiler preserves the source ordering and rounding properties of floating-point code when - // it generates and optimizes object code for the target machine, and observes the standard when handling special values. In addition, the program may safely - // access or modify the floating-point environment at runtime.``` - - cmdLine.addArg("/fp:precise"); - break; - } - case FloatingPointMode::Fast: - { - cmdLine.addArg("/fp:fast"); - break; - } - } - - switch (options.targetType) - { - case TargetType::SharedLibrary: - { - // Create dynamic link library - if (options.debugInfoType == DebugInfoType::None) - { - cmdLine.addArg("/LDd"); - } - else - { - cmdLine.addArg("/LD"); - } - - cmdLine.addPrefixPathArg("/Fe", options.modulePath, ".dll"); - break; - } - case TargetType::Executable: - { - cmdLine.addPrefixPathArg("/Fe", options.modulePath, ".exe"); - break; - } - default: break; - } - - // Object file specify it's location - needed if we are out - cmdLine.addPrefixPathArg("/Fo", options.modulePath, ".obj"); - - // Add defines - for (const auto& define : options.defines) - { - StringBuilder builder; - builder << "/D"; - builder << define.nameWithSig; - if (define.value.getLength()) - { - builder << "=" << define.value; - } - - cmdLine.addArg(builder); - } - - // Add includes - for (const auto& include : options.includePaths) - { - cmdLine.addArg("/I"); - cmdLine.addArg(include); - } - - // https://docs.microsoft.com/en-us/cpp/build/reference/eh-exception-handling-model?view=vs-2019 - // /Eha - Specifies the model of exception handling. (a, s, c, r are options) - - // Files to compile - for (const auto& sourceFile : options.sourceFiles) - { - cmdLine.addArg(sourceFile); - } - - // Link options (parameters past /link go to linker) - cmdLine.addArg("/link"); - - for (const auto& libPath : options.libraryPaths) - { - // Note that any escaping of the path is handled in the ProcessUtil:: - cmdLine.addPrefixPathArg("/LIBPATH:", libPath); - } - - return SLANG_OK; -} - -static SlangResult _parseSeverity(const UnownedStringSlice& in, DownstreamDiagnostics::Diagnostic::Severity& outSeverity) -{ - typedef DownstreamDiagnostics::Diagnostic::Severity Type; - - if (in == "error" || in == "fatal error") - { - outSeverity = Type::Error; - } - else if (in == "warning") - { - outSeverity = Type::Warning; - } - else if (in == "info") - { - outSeverity = Type::Info; - } - else - { - return SLANG_FAIL; - } - return SLANG_OK; -} - -static SlangResult _parseVisualStudioLine(const UnownedStringSlice& line, DownstreamDiagnostics::Diagnostic& outDiagnostic) -{ - typedef DownstreamDiagnostics::Diagnostic Diagnostic; - - UnownedStringSlice linkPrefix = UnownedStringSlice::fromLiteral("LINK :"); - if (line.startsWith(linkPrefix)) - { - outDiagnostic.stage = Diagnostic::Stage::Link; - outDiagnostic.severity = Diagnostic::Severity::Info; - - outDiagnostic.text = UnownedStringSlice(line.begin() + linkPrefix.getLength(), line.end()); - - return SLANG_OK; - } - - outDiagnostic.stage = Diagnostic::Stage::Compile; - - const char*const start = line.begin(); - const char*const end = line.end(); - - UnownedStringSlice postPath; - // Handle the path and line no - { - const char* cur = start; - - // We have to assume it is a path up to the first : that isn't part of a drive specification - - if ((end - cur > 2) && Path::isDriveSpecification(UnownedStringSlice(start, start + 2))) - { - // Skip drive spec - cur += 2; - } - - // Find the first colon after this - Index colonIndex = UnownedStringSlice(cur, end).indexOf(':'); - if (colonIndex < 0) - { - return SLANG_FAIL; - } - - // Looks like we have a line number - if (cur[colonIndex - 1] == ')') - { - const char* lineNoEnd = cur + colonIndex - 1; - const char* lineNoStart = lineNoEnd; - while (lineNoStart > start && *lineNoStart != '(') - { - lineNoStart--; - } - // Check this appears plausible - if (*lineNoStart != '(' || *lineNoEnd != ')') - { - return SLANG_FAIL; - } - Int numDigits = 0; - Int lineNo = 0; - for (const char* digitCur = lineNoStart + 1; digitCur < lineNoEnd; ++digitCur) - { - char c = *digitCur; - if (c >= '0' && c <= '9') - { - lineNo = lineNo * 10 + (c - '0'); - numDigits++; - } - else - { - return SLANG_FAIL; - } - } - if (numDigits == 0) - { - return SLANG_FAIL; - } - - outDiagnostic.filePath = UnownedStringSlice(start, lineNoStart); - outDiagnostic.fileLine = lineNo; - } - else - { - outDiagnostic.filePath = UnownedStringSlice(start, cur + colonIndex); - outDiagnostic.fileLine = 0; - } - - // Save the remaining text in 'postPath' - postPath = UnownedStringSlice(cur + colonIndex + 1, end); - } - - // Split up the error section - UnownedStringSlice postError; - { - // tests/cpp-compiler/c-compile-link-error.exe : fatal error LNK1120: 1 unresolved externals - - const Index errorColonIndex = postPath.indexOf(':'); - if (errorColonIndex < 0) - { - return SLANG_FAIL; - } - - const UnownedStringSlice errorSection = UnownedStringSlice(postPath.begin(), postPath.begin() + errorColonIndex); - Index errorCodeIndex = errorSection.lastIndexOf(' '); - if (errorCodeIndex < 0) - { - return SLANG_FAIL; - } - - // Extract the code - outDiagnostic.code = UnownedStringSlice(errorSection.begin() + errorCodeIndex + 1, errorSection.end()); - if (outDiagnostic.code.startsWith(UnownedStringSlice::fromLiteral("LNK"))) - { - outDiagnostic.stage = Diagnostic::Stage::Link; - } - - // Extract the bit before the code - SLANG_RETURN_ON_FAIL(_parseSeverity(UnownedStringSlice(errorSection.begin(), errorSection.begin() + errorCodeIndex).trim(), outDiagnostic.severity)); - - // Link codes start with LNK prefix - postError = UnownedStringSlice(postPath.begin() + errorColonIndex + 1, end); - } - - outDiagnostic.text = postError; - - return SLANG_OK; -} - -/* static */SlangResult VisualStudioCompilerUtil::parseOutput(const ExecuteResult& exeRes, DownstreamDiagnostics& outDiagnostics) -{ - outDiagnostics.reset(); - - outDiagnostics.rawDiagnostics = exeRes.standardOutput; - - for (auto line : LineParser(exeRes.standardOutput.getUnownedSlice())) - { -#if 0 - fwrite(line.begin(), 1, line.size(), stdout); - fprintf(stdout, "\n"); -#endif - - Diagnostic diagnostic; - if (SLANG_SUCCEEDED(_parseVisualStudioLine(line, diagnostic))) - { - outDiagnostics.diagnostics.add(diagnostic); - } - } - - // if it has a compilation error.. set on output - if (outDiagnostics.has(Diagnostic::Severity::Error)) - { - outDiagnostics.result = SLANG_FAIL; - } - - return SLANG_OK; -} - -/* static */SlangResult VisualStudioCompilerUtil::locateCompilers(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set) -{ - SLANG_UNUSED(loader); - - // TODO(JS): We don't support fixed path for visual studio just yet - if (path.getLength() == 0) - { -#if SLANG_VC - return WinVisualStudioUtil::find(set); -#endif - } - - return SLANG_OK; -} - -} diff --git a/source/core/slang-visual-studio-compiler-util.h b/source/core/slang-visual-studio-compiler-util.h deleted file mode 100644 index 018dde212..000000000 --- a/source/core/slang-visual-studio-compiler-util.h +++ /dev/null @@ -1,42 +0,0 @@ -#ifndef SLANG_VISUAL_STUDIO_COMPILER_UTIL_H -#define SLANG_VISUAL_STUDIO_COMPILER_UTIL_H - -#include "slang-downstream-compiler.h" - -namespace Slang -{ - - -struct VisualStudioCompilerUtil : public DownstreamCompilerBaseUtil -{ - /// Calculate Visual Studio family compilers cmdLine arguments from options - static SlangResult calcArgs(const CompileOptions& options, CommandLine& cmdLine); - /// Parse Visual Studio exeRes into CPPCompiler::Output - static SlangResult parseOutput(const ExecuteResult& exeRes, DownstreamDiagnostics& outOutput); - - static SlangResult calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath); - - static SlangResult calcCompileProducts(const CompileOptions& options, ProductFlags flags, List& outPaths); - - static SlangResult locateCompilers(const String& path, ISlangSharedLibraryLoader* loader, DownstreamCompilerSet* set); -}; - -class VisualStudioDownstreamCompiler : public CommandLineDownstreamCompiler -{ -public: - typedef CommandLineDownstreamCompiler Super; - typedef VisualStudioCompilerUtil Util; - - // CommandLineDownstreamCompiler impl - just forwards to the Util - virtual SlangResult calcArgs(const CompileOptions& options, CommandLine& cmdLine) SLANG_OVERRIDE { return Util::calcArgs(options, cmdLine); } - virtual SlangResult parseOutput(const ExecuteResult& exeResult, DownstreamDiagnostics& output) SLANG_OVERRIDE { return Util::parseOutput(exeResult, output); } - virtual SlangResult calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath) SLANG_OVERRIDE { return Util::calcModuleFilePath(options, outPath); } - virtual SlangResult calcCompileProducts(const CompileOptions& options, ProductFlags productFlags, List& outPaths) SLANG_OVERRIDE { return Util::calcCompileProducts(options, productFlags, outPaths); } - - VisualStudioDownstreamCompiler(const Desc& desc):Super(desc) {} -}; - - -} - -#endif diff --git a/source/core/windows/slang-win-visual-studio-util.cpp b/source/core/windows/slang-win-visual-studio-util.cpp deleted file mode 100644 index 078406d99..000000000 --- a/source/core/windows/slang-win-visual-studio-util.cpp +++ /dev/null @@ -1,366 +0,0 @@ -#include "slang-win-visual-studio-util.h" - -#include "../slang-common.h" -#include "../slang-process-util.h" -#include "../slang-string-util.h" - -#include "../slang-visual-studio-compiler-util.h" - -#ifdef _WIN32 -# define WIN32_LEAN_AND_MEAN -# define NOMINMAX -# include -# undef WIN32_LEAN_AND_MEAN -# undef NOMINMAX - -# include - -#endif - -// The method used to invoke VS was originally inspired by some ideas in -// https://github.com/RuntimeCompiledCPlusPlus/RuntimeCompiledCPlusPlus/ - -namespace Slang { - -// Information on VS versioning can be found here -// https://en.wikipedia.org/wiki/Microsoft_Visual_C%2B%2B#Internal_version_numbering - - -namespace { // anonymous - -typedef WinVisualStudioUtil::Version Version; - -struct RegistryInfo -{ - const char* regName; ///< The name of the entry in the registry - const char* pathFix; ///< With the value from the registry how to fix the path -}; - -struct VersionInfo -{ - Version version; ///< The version - const char* name; ///< The name of the registry key -}; - -} // anonymous - -static SlangResult _readRegistryKey(const char* path, const char* keyName, String& outString) -{ - // https://docs.microsoft.com/en-us/windows/desktop/api/winreg/nf-winreg-regopenkeyexa - HKEY key; - LONG ret = RegOpenKeyExA(HKEY_LOCAL_MACHINE, path, 0, KEY_READ | KEY_WOW64_32KEY, &key); - if (ret != ERROR_SUCCESS) - { - return SLANG_FAIL; - } - - char value[MAX_PATH]; - DWORD size = MAX_PATH; - - // https://docs.microsoft.com/en-us/windows/desktop/api/winreg/nf-winreg-regqueryvalueexa - ret = RegQueryValueExA(key, keyName, nullptr, nullptr, (LPBYTE)value, &size); - RegCloseKey(key); - - if (ret != ERROR_SUCCESS) - { - return SLANG_FAIL; - } - - outString = value; - return SLANG_OK; -} - -// Make easier to set up the array -static Version _makeVersion(int main, int dot = 0) { return WinVisualStudioUtil::makeVersion(main, dot); } - -VersionInfo _makeVersionInfo(const char* name, int high, int dot = 0) -{ - VersionInfo info; - info.name = name; - info.version = WinVisualStudioUtil::makeVersion(high, dot); - return info; -} - -static const VersionInfo s_versionInfos[] = -{ - _makeVersionInfo("VS 2005", 8), - _makeVersionInfo("VS 2008", 9), - _makeVersionInfo("VS 2010", 10), - _makeVersionInfo("VS 2012", 11), - _makeVersionInfo("VS 2013", 12), - _makeVersionInfo("VS 2015", 14), - _makeVersionInfo("VS 2017", 15), - _makeVersionInfo("VS 2019", 16), -}; - -// When trying to figure out how this stuff works by running regedit - care is needed, -// because what regedit displays varies on which version of regedit is used. -// In order to use the registry paths used here it's necessary to use Start/Run with -// %systemroot%\syswow64\regedit to view 32 bit keys - -static const RegistryInfo s_regInfos[] = -{ - {"SOFTWARE\\Microsoft\\VisualStudio\\SxS\\VC7", "" }, - {"SOFTWARE\\Microsoft\\VisualStudio\\SxS\\VS7", "VC\\Auxiliary\\Build\\" }, -}; - -static bool _canUseVSWhere(Version version) -{ - // If greater than 15.0 we can use vswhere tool - return (int(version) >= int(_makeVersion(15))); -} - -static int _getRegistryKeyIndex(Version version) -{ - if (int(version) >= int(_makeVersion(15))) - { - return 1; - } - return 0; -} - -/* static */void WinVisualStudioUtil::getVersions(List& outVersions) -{ - const int count = SLANG_COUNT_OF(s_versionInfos); - outVersions.setCount(count); - - Version* dst = outVersions.begin(); - for (int i = 0; i < count; ++i) - { - dst[i] = s_versionInfos[i].version; - } -} - -/* static */WinVisualStudioUtil::Version WinVisualStudioUtil::getCompiledVersion() -{ - // Get the version of visual studio used to compile this source - const uint32_t version = _MSC_VER; - - switch (version) - { - case 1400: return _makeVersion(8); - case 1500: return _makeVersion(9); - case 1600: return _makeVersion(10); - case 1700: return _makeVersion(11); - case 1800: return _makeVersion(12); - case 1900: - { - return _makeVersion(14); - } - case 1911: - case 1912: - case 1913: - case 1914: - case 1915: - case 1916: - { - return _makeVersion(15); - } - case 1920: - { - return _makeVersion(16); - } - default: - { - int lastKnownVersion = 1920; - if (version > lastKnownVersion) - { - // Its an unknown newer version - return Version::Future; - } - break; - } - } - - // Unknown version - return Version::Unknown; -} - -static SlangResult _find(int versionIndex, WinVisualStudioUtil::VersionPath& outPath) -{ - const auto& versionInfo = s_versionInfos[versionIndex]; - - auto version = versionInfo.version; - - outPath.version = version; - outPath.vcvarsPath = String(); - - if (_canUseVSWhere(version)) - { - CommandLine cmd; - - // Lookup directly %ProgramFiles(x86)% path - // https://docs.microsoft.com/en-us/windows/desktop/api/shlobj_core/nf-shlobj_core-shgetfolderpatha - HWND hwnd = GetConsoleWindow(); - - char programFilesPath[_MAX_PATH]; - SHGetFolderPathA(hwnd, CSIDL_PROGRAM_FILESX86, NULL, 0, programFilesPath); - - String vswherePath = programFilesPath; - vswherePath.append("\\Microsoft Visual Studio\\Installer\\vswhere"); - - cmd.setExecutableFilename(vswherePath); - - StringBuilder versionName; - WinVisualStudioUtil::append(version, versionName); - - String args[] = { "-version", versionName, "-requires", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", "-property", "installationPath" }; - cmd.addArgs(args, SLANG_COUNT_OF(args)); - - ExecuteResult exeRes; - if (SLANG_SUCCEEDED(ProcessUtil::execute(cmd, exeRes))) - { - // We need to chopoff CR/LF if there is one - List lines; - StringUtil::calcLines(exeRes.standardOutput.getUnownedSlice(), lines); - - if (lines.getCount()) - { - outPath.vcvarsPath = lines[0]; - outPath.vcvarsPath.append("\\VC\\Auxiliary\\Build\\"); - return SLANG_OK; - } - } - } - - const Int keyIndex = _getRegistryKeyIndex(version); - if (keyIndex >= 0) - { - SLANG_ASSERT(keyIndex < SLANG_COUNT_OF(s_regInfos)); - - // Try reading the key - const auto& keyInfo = s_regInfos[keyIndex]; - - StringBuilder keyName; - WinVisualStudioUtil::append(versionInfo.version, keyName); - - String value; - if (SLANG_SUCCEEDED(_readRegistryKey(keyInfo.regName, keyName.getBuffer(), value))) - { - outPath.vcvarsPath = value; - return SLANG_OK; - } - } - - return SLANG_FAIL; -} - -/* static */SlangResult WinVisualStudioUtil::find(List& outVersionPaths) -{ - outVersionPaths.clear(); - - const int versionCount = SLANG_COUNT_OF(s_versionInfos); - - for (int i = versionCount - 1; i >= 0; --i) - { - VersionPath versionPath; - if (SLANG_SUCCEEDED(_find(i, versionPath))) - { - outVersionPaths.add(versionPath); - } - } - - return SLANG_OK; -} - -/* static */SlangResult WinVisualStudioUtil::find(Version version, VersionPath& outPath) -{ - const int versionCount = SLANG_COUNT_OF(s_versionInfos); - - for (int i = 0; i < versionCount; ++i) - { - const auto& versionInfo = s_versionInfos[i]; - if (versionInfo.version == version) - { - return _find(i, outPath); - } - } - return SLANG_FAIL; -} - -/* static */SlangResult WinVisualStudioUtil::find(DownstreamCompilerSet* set) -{ - const int versionCount = SLANG_COUNT_OF(s_versionInfos); - - for (int i = versionCount - 1; i >= 0; --i) - { - const auto& versionInfo = s_versionInfos[i]; - auto desc = getDesc(versionInfo.version); - - VersionPath versionPath; - if (!set->getCompiler(desc) && SLANG_SUCCEEDED(_find(i, versionPath))) - { - RefPtr compiler = new VisualStudioDownstreamCompiler(desc); - calcExecuteCompilerArgs(versionPath, compiler->m_cmdLine); - set->addCompiler(compiler); - } - } - - return SLANG_OK; -} - -/* static */void WinVisualStudioUtil::calcExecuteCompilerArgs(const VersionPath& versionPath, CommandLine& outCmdLine) -{ - // To invoke cl we need to run the suitable vcvars. In order to run this we have to have MS CommandLine. - // So here we build up a cl command line that is run by first running vcvars, and then executing cl with the parameters as passed to commandLine - - CommandLine cmdLine; - - cmdLine.setExecutableFilename("cmd.exe"); - { - String options[] = { "/q", "/c", "@prompt", "$" }; - cmdLine.addArgs(options, SLANG_COUNT_OF(options)); - } - - cmdLine.addArg("&&"); - - { - StringBuilder path; - path << versionPath.vcvarsPath; - path << "\\vcvarsall.bat"; - cmdLine.addArg(path); - } - -#if SLANG_PTR_IS_32 - cmdLine.addArg("x86"); -#else - cmdLine.addArg("x86_amd64"); -#endif - - cmdLine.addArg("&&"); - cmdLine.addArg("cl"); - - outCmdLine = cmdLine; -} - -/* static */SlangResult WinVisualStudioUtil::executeCompiler(const VersionPath& versionPath, const CommandLine& commandLine, ExecuteResult& outResult) -{ - CommandLine cmdLine; - calcExecuteCompilerArgs(versionPath, cmdLine); - // Append the command line options - cmdLine.addArgs(commandLine.m_args.getBuffer(), commandLine.m_args.getCount()); - return ProcessUtil::execute(cmdLine, outResult); -} - -/* static */void WinVisualStudioUtil::append(Version version, StringBuilder& outBuilder) -{ - switch (version) - { - case Version::Unknown: - { - outBuilder << "unknown"; - } - case Version::Future: - { - outBuilder << "future"; - break; - } - default: - { - outBuilder << (int(version) / 10) << "." << (int(version) % 10); - break; - } - } -} - -} // namespace Slang diff --git a/source/core/windows/slang-win-visual-studio-util.h b/source/core/windows/slang-win-visual-studio-util.h deleted file mode 100644 index 530d18582..000000000 --- a/source/core/windows/slang-win-visual-studio-util.h +++ /dev/null @@ -1,68 +0,0 @@ -#ifndef SLANG_WIN_VISUAL_STUDIO_UTIL_H -#define SLANG_WIN_VISUAL_STUDIO_UTIL_H - -#include "../slang-list.h" -#include "../slang-string.h" - -#include "../slang-process-util.h" - -#include "../slang-downstream-compiler.h" - -namespace Slang { - -struct WinVisualStudioUtil -{ - enum class Version: uint32_t - { - Unknown = 0, ///< This is an unknown (and not later) version - Future = 0xff * 10, ///< This is a version 'from the future' - that isn't specifically known. Will be treated as latest - }; - - struct VersionPath - { - Version version; ///< The visual studio version - String vcvarsPath; ///< The path to vcvars bat files, that need to be executed before executing the compiler - }; - - /// Find all the installations - static SlangResult find(List& outVersionPaths); - - /// Given a version find it's path - static SlangResult find(Version version, VersionPath& outPath); - - /// Find and add to the set (if not already there) - static SlangResult find(DownstreamCompilerSet* set); - - /// Create the cmdLine to start compiler for specified path - static void calcExecuteCompilerArgs(const VersionPath& versionPath, CommandLine& outCmdLine); - - /// Run visual studio on specified path with the parameters specified on the command line. Output placed in outResult. - static SlangResult executeCompiler(const VersionPath& versionPath, const CommandLine& commandLine, ExecuteResult& outResult); - - /// Get all the known version numbers - static void getVersions(List& outVersions); - - /// Gets the msc compiler used to compile this version. Returning Version(0) means unknown - static Version getCompiledVersion(); - - /// Create a version from a high and low indices - static Version makeVersion(int high, int low = 0) { SLANG_ASSERT(low >= 0 && low <= 9); return Version(high * 10 + low); } - - /// Convert a version number into a string - static void append(Version version, StringBuilder& outBuilder); - - /// Get version as desc - static DownstreamCompiler::Desc getDesc(Version version) - { - DownstreamCompiler::Desc desc; - desc.type = SLANG_PASS_THROUGH_VISUAL_STUDIO; - desc.majorVersion = Int(version) / 10; - desc.minorVersion = Int(version) % 10; - return desc; - } - -}; - -} // namespace Slang - -#endif diff --git a/source/slang/slang-ast-support-types.h b/source/slang/slang-ast-support-types.h index 1921a101f..168d358d6 100644 --- a/source/slang/slang-ast-support-types.h +++ b/source/slang/slang-ast-support-types.h @@ -2,7 +2,10 @@ #define SLANG_AST_SUPPORT_TYPES_H #include "../core/slang-basic.h" -#include "slang-lexer.h" + +#include "../compiler-core/slang-lexer.h" +#include "../compiler-core/slang-name.h" + #include "slang-profile.h" #include "slang-type-system-shared.h" #include "../../slang.h" @@ -16,7 +19,6 @@ #include "slang-ast-reflect.h" #include "slang-ref-object-reflect.h" -#include "slang-name.h" #include diff --git a/source/slang/slang-compiler.cpp b/source/slang/slang-compiler.cpp index 2d0287613..c034f5f7d 100755 --- a/source/slang/slang-compiler.cpp +++ b/source/slang/slang-compiler.cpp @@ -10,7 +10,9 @@ #include "slang-check.h" #include "slang-compiler.h" -#include "slang-lexer.h" + +#include "../compiler-core/slang-lexer.h" + #include "slang-lower-to-ir.h" #include "slang-mangle.h" #include "slang-parameter-binding.h" diff --git a/source/slang/slang-compiler.h b/source/slang/slang-compiler.h index 9ef75ab86..b1e3c3a70 100755 --- a/source/slang/slang-compiler.h +++ b/source/slang/slang-compiler.h @@ -3,9 +3,11 @@ #include "../core/slang-basic.h" #include "../core/slang-shared-library.h" - -#include "../core/slang-downstream-compiler.h" #include "../core/slang-archive-file-system.h" +#include "../core/slang-file-system.h" + +#include "../compiler-core/slang-downstream-compiler.h" +#include "../compiler-core/slang-name.h" #include "../core/slang-std-writers.h" @@ -13,12 +15,11 @@ #include "slang-capability.h" #include "slang-diagnostics.h" -#include "slang-name.h" + #include "slang-preprocessor.h" #include "slang-profile.h" #include "slang-syntax.h" -#include "slang-file-system.h" #include "slang-include-system.h" diff --git a/source/slang/slang-diagnostic-defs.h b/source/slang/slang-diagnostic-defs.h index 6387ff8b2..ef5a94501 100644 --- a/source/slang/slang-diagnostic-defs.h +++ b/source/slang/slang-diagnostic-defs.h @@ -43,7 +43,6 @@ DIAGNOSTIC(-1, Note, doYouForgetToMakeComponentAccessible, "do you forget to mak DIAGNOSTIC(-1, Note, seeDeclarationOf, "see declaration of '$0'") DIAGNOSTIC(-1, Note, seeOtherDeclarationOf, "see other declaration of '$0'") DIAGNOSTIC(-1, Note, seePreviousDeclarationOf, "see previous declaration of '$0'") -DIAGNOSTIC(-1, Note, seeTokenPasteLocation, "see token pasted location") DIAGNOSTIC(-1, Note, includeOutput, "include $0") // @@ -129,21 +128,6 @@ DIAGNOSTIC( 88, Error, unknownArchiveType, "archive type '%0' is unknown") DIAGNOSTIC( 100, Error, failedToLoadDownstreamCompiler, "failed to load downstream compiler '$0'") DIAGNOSTIC(99999, Note, noteFailedToLoadDynamicLibrary, "failed to load dynamic library '$0'") - -// -// 1xxxx - Lexical analysis -// - -DIAGNOSTIC(10000, Error, illegalCharacterPrint, "illegal character '$0'") -DIAGNOSTIC(10000, Error, illegalCharacterHex, "illegal character (0x$0)") -DIAGNOSTIC(10001, Error, illegalCharacterLiteral, "illegal character literal") - -DIAGNOSTIC(10002, Warning, octalLiteral, "'0' prefix indicates octal literal") -DIAGNOSTIC(10003, Error, invalidDigitForBase, "invalid digit for base-$1 literal: '$0'") - -DIAGNOSTIC(10004, Error, endOfFileInLiteral, "end of file in literal") -DIAGNOSTIC(10005, Error, newlineInLiteral, "newline in literal") - // // 15xxx - Preprocessing // diff --git a/source/slang/slang-diagnostics.cpp b/source/slang/slang-diagnostics.cpp index 4b450c9a9..4a831aacf 100644 --- a/source/slang/slang-diagnostics.cpp +++ b/source/slang/slang-diagnostics.cpp @@ -1,473 +1,15 @@ // slang-diagnostics.cpp #include "slang-diagnostics.h" -#include "slang-name.h" - #include "../core/slang-memory-arena.h" #include "../core/slang-dictionary.h" #include "../core/slang-string-util.h" -#include "../core/slang-char-util.h" -#include "../core/slang-name-convention-util.h" - -#include - -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#define NOMINMAX -#include -#undef WIN32_LEAN_AND_MEAN -#undef NOMINMAX -#include -#endif - -namespace Slang { - -void printDiagnosticArg(StringBuilder& sb, char const* str) -{ - sb << str; -} - -void printDiagnosticArg(StringBuilder& sb, int32_t val) -{ - sb << val; -} - -void printDiagnosticArg(StringBuilder& sb, uint32_t val) -{ - sb << val; -} - -void printDiagnosticArg(StringBuilder& sb, int64_t val) -{ - sb << val; -} - -void printDiagnosticArg(StringBuilder& sb, uint64_t val) -{ - sb << val; -} - -void printDiagnosticArg(StringBuilder& sb, double val) -{ - sb << val; -} - -void printDiagnosticArg(StringBuilder& sb, Slang::String const& str) -{ - sb << str; -} - -void printDiagnosticArg(StringBuilder& sb, Slang::UnownedStringSlice const& str) -{ - sb.append(str); -} - - -void printDiagnosticArg(StringBuilder& sb, Name* name) -{ - sb << getText(name); -} - - -void printDiagnosticArg(StringBuilder& sb, TokenType tokenType) -{ - sb << TokenTypeToString(tokenType); -} - -void printDiagnosticArg(StringBuilder& sb, Token const& token) -{ - sb << token.getContent(); -} - -SourceLoc const& getDiagnosticPos(Token const& token) -{ - return token.loc; -} - -// Take the format string for a diagnostic message, along with its arguments, and turn it into a -static void formatDiagnosticMessage(StringBuilder& sb, char const* format, int argCount, DiagnosticArg const* const* args) -{ - char const* spanBegin = format; - for(;;) - { - char const* spanEnd = spanBegin; - while (int c = *spanEnd) - { - if (c == '$') - break; - spanEnd++; - } - - sb.Append(spanBegin, int(spanEnd - spanBegin)); - if (!*spanEnd) - return; - - SLANG_ASSERT(*spanEnd == '$'); - spanEnd++; - int d = *spanEnd++; - switch (d) - { - // A double dollar sign `$$` is used to emit a single `$` - case '$': - sb.Append('$'); - break; - - // A single digit means to emit the corresponding argument. - // TODO: support more than 10 arguments, and add options - // to control formatting, etc. - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - { - int index = d - '0'; - if (index >= argCount) - { - // TODO(tfoley): figure out what a good policy will be for "panic" situations like this - throw InvalidOperationException("too few arguments for diagnostic message"); - } - else - { - DiagnosticArg const* arg = args[index]; - arg->printFunc(sb, arg->data); - } - } - break; - - default: - throw InvalidOperationException("invalid diagnostic message format"); - break; - } - - spanBegin = spanEnd; - } -} - -static void formatDiagnostic(const HumaneSourceLoc& humaneLoc, Diagnostic const& diagnostic, StringBuilder& outBuilder) -{ - outBuilder << humaneLoc.pathInfo.foundPath; - outBuilder << "("; - outBuilder << Int32(humaneLoc.line); - outBuilder << "): "; - - outBuilder << getSeverityName(diagnostic.severity); - - if (diagnostic.ErrorID >= 0) - { - outBuilder << " "; - outBuilder << diagnostic.ErrorID; - } - - outBuilder << ": "; - outBuilder << diagnostic.Message; - outBuilder << "\n"; -} - -static void _replaceTabWithSpaces(const UnownedStringSlice& slice, Int tabSize, StringBuilder& out) -{ - const char* start = slice.begin(); - const char*const end = slice.end(); - - const Index startLength = out.getLength(); - - for (const char* cur = start; cur < end; cur++) - { - if (*cur == '\t') - { - if (start < cur) - { - out.append(start, cur); - } - - // The amount of spaces we add depends on the current position. - const Index lastPosition = out.getLength() - startLength; - Index tabPosition = lastPosition; - - // Strip the tabPosition so it's back to the tab stop - // Special case if tabSize is a power of 2 - if ((tabSize & (tabSize - 1)) == 0) - { - tabPosition = tabPosition & ~Index(tabSize - 1); - } - else - { - tabPosition -= tabPosition % tabSize; - } - - // Move to next tab - tabPosition += tabSize; - - // The amount of spaces to simulate the tab - const Index spacesCount = tabPosition - lastPosition; - - // Add the spaces - out.appendRepeatedChar(' ', spacesCount); - - // Set the start at the first character past - start = cur + 1; - } - } - - if (start < end) - { - out.append(start, end); - } -} - -// Given multi-line text, and a position within the text (as a pointer into the memory of text) -// extract the line that contains pos -static UnownedStringSlice _extractLineContainingPosition(const UnownedStringSlice& text, const char* pos) -{ - SLANG_ASSERT(text.isMemoryContained(pos)); - - const char*const contentStart = text.begin(); - const char*const contentEnd = text.end(); - - // We want to determine the start of the line, and the end of the line - const char* start = pos; - for (; start > contentStart; --start) - { - const char c = *start; - if (c == '\n' || c == '\r') - { - // We want the character after, but we can only do this if not already at pos - start += int(start < pos); - break; - } - } - const char* end = pos; - for (; end < contentEnd; ++end) - { - const char c = *end; - if (c == '\n' || c == '\r') - { - break; - } - } - - return UnownedStringSlice(start, end); -} - -static void _sourceLocationNoteDiagnostic(SourceView* sourceView, SourceLoc sourceLoc, DiagnosticSink::SourceLocationLexer lexer, StringBuilder& sb) -{ - SourceFile* sourceFile = sourceView->getSourceFile(); - if (!sourceFile) - { - return; - } - - UnownedStringSlice content = sourceFile->getContent(); - - // Make sure the offset is within content. - // This is important because it's possible to have a 'SourceFile' that doesn't contain any content - // (for example when reconstructed via serialization with just line offsets, the actual source text 'content' isn't available). - const int offset = sourceView->getRange().getOffset(sourceLoc); - if (offset < 0 || offset >= content.getLength()) - { - return; - } - - // Work out the position of the SourceLoc in the source - const char*const pos = content.begin() + offset; - - UnownedStringSlice line = _extractLineContainingPosition(content, pos); - - // Trim any trailing white space - line = UnownedStringSlice(line.begin(), line.trim().end()); - - // TODO(JS): The tab size should ideally be configurable from command line. - // For now just go with 4. - const Index tabSize = 4; - - StringBuilder sourceLine; - StringBuilder caretLine; - - // First work out the sourceLine - _replaceTabWithSpaces(line, tabSize, sourceLine); - - // Now the caretLine which appears underneath the sourceLine - { - // Produce the text up to the caret position (at pos), taking into account tabs - _replaceTabWithSpaces(UnownedStringSlice(line.begin(), pos), tabSize, caretLine); - // Now make all spaces - const Index length = caretLine.getLength(); - caretLine.Clear(); - caretLine.appendRepeatedChar(' ', length); - - // Add caret - caretLine << "^"; +#include "../compiler-core/slang-name.h" +#include "../compiler-core/slang-core-diagnostics.h" - if (lexer) - { - UnownedStringSlice token = lexer(UnownedStringSlice(pos, line.end())); - - if (token.getLength() > 1) - { - caretLine.appendRepeatedChar('~', token.getLength() - 1); - } - } - } - - // We could have handling here for if the line is too long, that we surround the important section - // will ellipsis for example. - // For now we just output. - - sb << sourceLine << "\n"; - sb << caretLine << "\n"; -} - -static void formatDiagnostic( - DiagnosticSink* sink, - Diagnostic const& diagnostic, - StringBuilder& sb) +namespace Slang { - auto sourceManager = sink->getSourceManager(); - - SourceView* sourceView = nullptr; - HumaneSourceLoc humaneLoc; - const auto sourceLoc = diagnostic.loc; - { - sourceView = sourceManager->findSourceViewRecursively(sourceLoc); - if (sourceView) - { - humaneLoc = sourceView->getHumaneLoc(sourceLoc); - } - formatDiagnostic(humaneLoc, diagnostic, sb); - - { - SourceView* currentView = sourceView; - - while (currentView && currentView->getInitiatingSourceLoc().isValid() && currentView->getSourceFile()->getPathInfo().type == PathInfo::Type::TokenPaste) - { - SourceView* initiatingView = sourceManager->findSourceView(currentView->getInitiatingSourceLoc()); - if (initiatingView == nullptr) - { - break; - } - - const DiagnosticInfo& diagnosticInfo = Diagnostics::seeTokenPasteLocation; - - // Turn the message format into a message. For the moment it assumes no parameters. - StringBuilder msg; - formatDiagnosticMessage(msg, diagnosticInfo.messageFormat, 0, nullptr); - - // Set up the diagnostic. - Diagnostic initiationDiagnostic; - initiationDiagnostic.ErrorID = diagnosticInfo.id; - initiationDiagnostic.Message = msg.ProduceString(); - initiationDiagnostic.loc = sourceView->getInitiatingSourceLoc(); - initiationDiagnostic.severity = diagnosticInfo.severity; - - // TODO(JS): - // Not 100% clear what the best sourceLoc type is most useful here - we will go with default for now - HumaneSourceLoc pasteHumaneLoc = initiatingView->getHumaneLoc(sourceView->getInitiatingSourceLoc()); - - // Okay we should output where the token paste took place - formatDiagnostic(pasteHumaneLoc, initiationDiagnostic, sb); - - // Make the initiatingView the current view - currentView = initiatingView; - } - } - } - - // We don't don't output source line information if this is a 'note' as a note is extra information for one - // of the other main severity types, and so the information should already be output on the initial line - if (sourceView && sink->isFlagSet(DiagnosticSink::Flag::SourceLocationLine) && diagnostic.severity != Severity::Note) - { - _sourceLocationNoteDiagnostic(sourceView, sourceLoc, sink->getSourceLocationLexer(), sb); - } - - if (sourceView && sink->isFlagSet(DiagnosticSink::Flag::VerbosePath)) - { - auto actualHumaneLoc = sourceView->getHumaneLoc(diagnostic.loc, SourceLocType::Actual); - - // Look up the path verbosely (will get the canonical path if necessary) - actualHumaneLoc.pathInfo.foundPath = sourceView->getSourceFile()->calcVerbosePath(); - - // Only output if it's actually different - if (actualHumaneLoc.pathInfo.foundPath != humaneLoc.pathInfo.foundPath || - actualHumaneLoc.line != humaneLoc.line || - actualHumaneLoc.column != humaneLoc.column) - { - formatDiagnostic(actualHumaneLoc, diagnostic, sb); - } - } -} - -void DiagnosticSink::diagnoseImpl(SourceLoc const& pos, DiagnosticInfo const& info, int argCount, DiagnosticArg const* const* args) -{ - StringBuilder sb; - formatDiagnosticMessage(sb, info.messageFormat, argCount, args); - - Diagnostic diagnostic; - diagnostic.ErrorID = info.id; - diagnostic.Message = sb.ProduceString(); - diagnostic.loc = pos; - diagnostic.severity = info.severity; - - if (diagnostic.severity >= Severity::Error) - { - m_errorCount++; - } - - // Did the client supply a callback for us to use? - if( writer ) - { - // If so, pass the error string along to them - StringBuilder messageBuilder; - formatDiagnostic(this, diagnostic, messageBuilder); - - writer->write(messageBuilder.getBuffer(), messageBuilder.getLength()); - } - else - { - // If the user doesn't have a callback, then just - // collect our diagnostic messages into a buffer - formatDiagnostic(this, diagnostic, outputBuffer); - } - - if (diagnostic.severity >= Severity::Fatal) - { - // TODO: figure out a better policy for aborting compilation - throw AbortCompilationException(); - } -} - -void DiagnosticSink::diagnoseRaw( - Severity severity, - char const* message) -{ - return diagnoseRaw(severity, UnownedStringSlice(message)); -} - -void DiagnosticSink::diagnoseRaw( - Severity severity, - const UnownedStringSlice& message) -{ - if (severity >= Severity::Error) - { - m_errorCount++; - } - - // Did the client supply a callback for us to use? - if(writer) - { - // If so, pass the error string along to them - writer->write(message.begin(), message.getLength()); - } - else - { - // If the user doesn't have a callback, then just - // collect our diagnostic messages into a buffer - outputBuffer.append(message); - } - - if (severity >= Severity::Fatal) - { - // TODO: figure out a better policy for aborting compilation - throw InvalidOperationException(); - } -} namespace Diagnostics { @@ -476,89 +18,41 @@ namespace Diagnostics #undef DIAGNOSTIC } -static const DiagnosticInfo* const kAllDiagnostics[] = +static const DiagnosticInfo* const kCompilerDiagnostics[] = { #define DIAGNOSTIC(id, severity, name, messageFormat) &Diagnostics::name, #include "slang-diagnostic-defs.h" #undef DIAGNOSTIC }; -class DiagnosticsLookup : public RefObject +static DiagnosticsLookup* _newDiagnosticsLookup() { -public: - const DiagnosticInfo* findDiagostic(const UnownedStringSlice& slice) const - { - const Index* indexPtr = m_map.TryGetValue(slice); - return indexPtr ? kAllDiagnostics[*indexPtr] : nullptr; - } - Index _findDiagnosticIndex(const UnownedStringSlice& slice) const - { - const Index* indexPtr = m_map.TryGetValue(slice); - return indexPtr ? *indexPtr : 0; - } - static DiagnosticsLookup* getSingleton() - { - static RefPtr singleton = new DiagnosticsLookup; - return singleton; - } - -protected: - void _add(const char* name, Index index) - { - UnownedStringSlice nameSlice(name); - m_map.Add(nameSlice, index); - - // Add a dashed version (KababCase) - { - m_work.Clear(); + DiagnosticsLookup* lookup = new DiagnosticsLookup(kCompilerDiagnostics, SLANG_COUNT_OF(kCompilerDiagnostics)); - NameConventionUtil::convert(NameConvention::Camel, nameSlice, CharCase::Lower, NameConvention::Kabab, m_work); - - UnownedStringSlice dashSlice(m_arena.allocateString(m_work.getBuffer(), m_work.getLength()), m_work.getLength()); - m_map.AddIfNotExists(dashSlice, index); - } - } - void _addAlias(const char* name, const char* diagnosticName) + // Add all the diagnostics in 'core' + DiagnosticsLookup* coreLookup = getCoreDiagnosticsLookup(); + if (coreLookup) { - const Index index = _findDiagnosticIndex(UnownedStringSlice(diagnosticName)); - SLANG_ASSERT(index >= 0); - if (index >= 0) + for (auto diagnostic : coreLookup->getDiagnostics()) { - _add(name, index); + lookup->add(diagnostic); } } - DiagnosticsLookup(); - StringBuilder m_work; - Dictionary m_map; - MemoryArena m_arena; -}; + // Add the alias + lookup->addAlias("overlappingBindings", "parameterBindingsOverlap"); + return lookup; +} -DiagnosticsLookup::DiagnosticsLookup(): - m_arena(2048) +static DiagnosticsLookup* _getDiagnosticLookupSingleton() { - // TODO: We should eventually have a more formal system for associating individual - // diagnostics, or groups of diagnostics, with user-exposed names for use when - // enabling/disabling warnings (or turning warnings into errors, etc.). - // - // For now we build a map from diagnostic name to it's entry. Two entries are typically - // added - the 'original name' as associated with the diagnostic in lowerCamel, and - // a dashified version. - - for (Index i = 0; i < SLANG_COUNT_OF(kAllDiagnostics); ++i) - { - const DiagnosticInfo* diagnostic = kAllDiagnostics[i]; - _add(diagnostic->name, i); - } - - // Add any aliases - _addAlias("overlappingBindings", "parameterBindingsOverlap"); + static RefPtr s_lookup = _newDiagnosticsLookup(); + return s_lookup; } DiagnosticInfo const* findDiagnosticByName(UnownedStringSlice const& name) { - return DiagnosticsLookup::getSingleton()->findDiagostic(name); + return _getDiagnosticLookupSingleton()->findDiagostic(name); } - } // namespace Slang diff --git a/source/slang/slang-diagnostics.h b/source/slang/slang-diagnostics.h index 05080989a..05411a90a 100644 --- a/source/slang/slang-diagnostics.h +++ b/source/slang/slang-diagnostics.h @@ -1,280 +1,17 @@ -#ifndef RASTER_RENDERER_COMPILE_ERROR_H -#define RASTER_RENDERER_COMPILE_ERROR_H +#ifndef SLANG_DIAGNOSTICS_H +#define SLANG_DIAGNOSTICS_H #include "../core/slang-basic.h" #include "../core/slang-writer.h" -#include "slang-source-loc.h" -#include "slang-token.h" +#include "../compiler-core/slang-source-loc.h" +#include "../compiler-core/slang-diagnostic-sink.h" +#include "../compiler-core/slang-token.h" #include "../../slang.h" namespace Slang { - enum class Severity - { - Note, - Warning, - Error, - Fatal, - Internal, - }; - - // TODO(tfoley): move this into a source file... - inline const char* getSeverityName(Severity severity) - { - switch (severity) - { - case Severity::Note: return "note"; - case Severity::Warning: return "warning"; - case Severity::Error: return "error"; - case Severity::Fatal: return "fatal error"; - case Severity::Internal: return "internal error"; - default: return "unknown error"; - } - } - - // A structure to be used in static data describing different - // diagnostic messages. - struct DiagnosticInfo - { - int id; - Severity severity; - char const* name; ///< Unique name - char const* messageFormat; - }; - - class Diagnostic - { - public: - String Message; - SourceLoc loc; - int ErrorID; - Severity severity; - - Diagnostic() - { - ErrorID = -1; - } - Diagnostic( - const String & msg, - int id, - const SourceLoc & pos, - Severity severity) - : severity(severity) - { - Message = msg; - ErrorID = id; - loc = pos; - } - }; - - class Name; - - //enum class CodeGenTarget; - - //enum class Stage : SlangStage; - //enum class ProfileVersion; - - void printDiagnosticArg(StringBuilder& sb, char const* str); - - void printDiagnosticArg(StringBuilder& sb, int32_t val); - void printDiagnosticArg(StringBuilder& sb, uint32_t val); - - void printDiagnosticArg(StringBuilder& sb, int64_t val); - void printDiagnosticArg(StringBuilder& sb, uint64_t val); - - void printDiagnosticArg(StringBuilder& sb, double val); - - void printDiagnosticArg(StringBuilder& sb, Slang::String const& str); - void printDiagnosticArg(StringBuilder& sb, Slang::UnownedStringSlice const& str); - void printDiagnosticArg(StringBuilder& sb, Name* name); - - void printDiagnosticArg(StringBuilder& sb, TokenType tokenType); - void printDiagnosticArg(StringBuilder& sb, Token const& token); - - struct IRInst; - void printDiagnosticArg(StringBuilder& sb, IRInst* irObject); - - template - void printDiagnosticArg(StringBuilder& sb, RefPtr ptr) - { - printDiagnosticArg(sb, ptr.Ptr()); - } - - inline SourceLoc const& getDiagnosticPos(SourceLoc const& pos) { return pos; } - - SourceLoc const& getDiagnosticPos(Token const& token); - - - template - SourceLoc getDiagnosticPos(RefPtr const& ptr) - { - return getDiagnosticPos(ptr.Ptr()); - } - - struct DiagnosticArg - { - void* data; - void (*printFunc)(StringBuilder&, void*); - - template - struct Helper - { - static void printFunc(StringBuilder& sb, void* data) { printDiagnosticArg(sb, *(T*)data); } - }; - - template - DiagnosticArg(T const& arg) - : data((void*)&arg) - , printFunc(&Helper::printFunc) - {} - }; - - - class DiagnosticSink - { - public: - /// Flags to control some aspects of Diagnostic sink behavior - typedef uint32_t Flags; - struct Flag - { - enum Enum: Flags - { - VerbosePath = 0x1, ///< Will display a more verbose path (if available) - such as a canonical or absolute path - SourceLocationLine = 0x2, ///< If set will display the location line if source is available - }; - }; - - /// Used by diagnostic sink to be able to underline tokens. If not defined on the DiagnosticSink, - /// will only display a caret at the SourceLoc - typedef UnownedStringSlice(*SourceLocationLexer)(const UnownedStringSlice& text); - - /// Get the total amount of errors that have taken place on this DiagnosticSink - SLANG_FORCE_INLINE int getErrorCount() { return m_errorCount; } - - void diagnoseDispatch(SourceLoc const& pos, DiagnosticInfo const& info) - { - diagnoseImpl(pos, info, 0, nullptr); - } - - void diagnoseDispatch(SourceLoc const& pos, DiagnosticInfo const& info, DiagnosticArg const& arg0) - { - DiagnosticArg const* args[] = { &arg0 }; - diagnoseImpl(pos, info, 1, args); - } - - void diagnoseDispatch(SourceLoc const& pos, DiagnosticInfo const& info, DiagnosticArg const& arg0, DiagnosticArg const& arg1) - { - DiagnosticArg const* args[] = { &arg0, &arg1 }; - diagnoseImpl(pos, info, 2, args); - } - - void diagnoseDispatch(SourceLoc const& pos, DiagnosticInfo const& info, DiagnosticArg const& arg0, DiagnosticArg const& arg1, DiagnosticArg const& arg2) - { - DiagnosticArg const* args[] = { &arg0, &arg1, &arg2 }; - diagnoseImpl(pos, info, 3, args); - } - - void diagnoseDispatch(SourceLoc const& pos, DiagnosticInfo const& info, DiagnosticArg const& arg0, DiagnosticArg const& arg1, DiagnosticArg const& arg2, DiagnosticArg const& arg3) - { - DiagnosticArg const* args[] = { &arg0, &arg1, &arg2, &arg3 }; - diagnoseImpl(pos, info, 4, args); - } - - template - void diagnose(P const& pos, DiagnosticInfo const& info, Args const&... args ) - { - diagnoseDispatch(getDiagnosticPos(pos), info, args...); - } - - // Add a diagnostic with raw text - // (used when we get errors from a downstream compiler) - void diagnoseRaw(Severity severity, char const* message); - void diagnoseRaw(Severity severity, const UnownedStringSlice& message); - - /// During propagation of an exception for an internal - /// error, note that this source location was involved - void noteInternalErrorLoc(SourceLoc const& loc); - - /// Create a blob containing diagnostics if there were any errors. - /// *note* only works if writer is not set, the blob is created from outputBuffer - SlangResult getBlobIfNeeded(ISlangBlob** outBlob); - - /// Get the source manager used - SourceManager* getSourceManager() const { return m_sourceManager; } - /// Set the source manager used for lookup of source locs - void setSourceManager(SourceManager* inSourceManager) { m_sourceManager = inSourceManager; } - - /// Get the flags - Flags getFlags() const { return m_flags; } - /// Set a flag - void setFlag(Flag::Enum flag) { m_flags |= Flags(flag); } - /// Reset a flag - void resetFlag(Flag::Enum flag) { m_flags &= ~Flags(flag); } - /// Test if flag is set - bool isFlagSet(Flag::Enum flag) { return (m_flags & Flags(flag)) != 0; } - - /// Get the (optional) diagnostic sink lexer. This is used to - /// improve quality of highlighting a locations token. If not set, will just have a single - /// character caret at location - SourceLocationLexer getSourceLocationLexer() const { return m_sourceLocationLexer; } - - /// Ctor - DiagnosticSink(SourceManager* sourceManager, SourceLocationLexer sourceLocationLexer) - : m_sourceManager(sourceManager), - m_sourceLocationLexer(sourceLocationLexer) - { - // If we have a source location lexer, we'll by default enable source location output - if (sourceLocationLexer) - { - setFlag(Flag::SourceLocationLine); - } - } - - // Public members - - /// The outputBuffer will contain any diagnostics *iff* the writer is *not* set - StringBuilder outputBuffer; - /// If a writer is set output will *not* be written to the outputBuffer - ISlangWriter* writer = nullptr; - - protected: - void diagnoseImpl(SourceLoc const& pos, DiagnosticInfo const& info, int argCount, DiagnosticArg const* const* args); - - int m_errorCount = 0; - int m_internalErrorLocsNoted = 0; - - Flags m_flags = 0; - - // The source manager to use when mapping source locations to file+line info - SourceManager* m_sourceManager = nullptr; - - SourceLocationLexer m_sourceLocationLexer; - }; - - /// An `ISlangWriter` that writes directly to a diagnostic sink. - class DiagnosticSinkWriter : public AppendBufferWriter - { - public: - typedef AppendBufferWriter Super; - - DiagnosticSinkWriter(DiagnosticSink* sink) - : Super(WriterFlag::IsStatic) - , m_sink(sink) - {} - - // ISlangWriter - SLANG_NO_THROW virtual SlangResult SLANG_MCALL write(const char* chars, size_t numChars) SLANG_OVERRIDE - { - m_sink->diagnoseRaw(Severity::Note, UnownedStringSlice(chars, chars+numChars)); - return SLANG_OK; - } - - private: - DiagnosticSink* m_sink = nullptr; - }; - DiagnosticInfo const* findDiagnosticByName(UnownedStringSlice const& name); namespace Diagnostics diff --git a/source/slang/slang-emit-c-like.cpp b/source/slang/slang-emit-c-like.cpp index a0fa71fee..1b0fe7c44 100644 --- a/source/slang/slang-emit-c-like.cpp +++ b/source/slang/slang-emit-c-like.cpp @@ -2,6 +2,8 @@ #include "slang-emit-c-like.h" #include "../core/slang-writer.h" +#include "../compiler-core/slang-name.h" + #include "slang-ir-bind-existentials.h" #include "slang-ir-dce.h" #include "slang-ir-entry-point-uniforms.h" @@ -17,7 +19,7 @@ #include "slang-legalize-types.h" #include "slang-lower-to-ir.h" #include "slang-mangle.h" -#include "slang-name.h" + #include "slang-syntax.h" #include "slang-type-layout.h" #include "slang-visitor.h" diff --git a/source/slang/slang-emit.cpp b/source/slang/slang-emit.cpp index c97412cf0..01b682a39 100644 --- a/source/slang/slang-emit.cpp +++ b/source/slang/slang-emit.cpp @@ -4,6 +4,8 @@ #include "../core/slang-writer.h" #include "../core/slang-type-text-util.h" +#include "../compiler-core/slang-name.h" + #include "slang-ir-bind-existentials.h" #include "slang-ir-byte-address-legalize.h" #include "slang-ir-collect-global-uniforms.h" @@ -33,7 +35,7 @@ #include "slang-legalize-types.h" #include "slang-lower-to-ir.h" #include "slang-mangle.h" -#include "slang-name.h" + #include "slang-syntax.h" #include "slang-type-layout.h" #include "slang-visitor.h" diff --git a/source/slang/slang-file-system.cpp b/source/slang/slang-file-system.cpp deleted file mode 100644 index 992ae4155..000000000 --- a/source/slang/slang-file-system.cpp +++ /dev/null @@ -1,888 +0,0 @@ -#include "slang-file-system.h" - -#include "../../slang-com-ptr.h" -#include "../core/slang-io.h" -#include "../core/slang-string-util.h" - -namespace Slang -{ - -// Allocate static const storage for the various interface IDs that the Slang API needs to expose -static const Guid IID_ISlangUnknown = SLANG_UUID_ISlangUnknown; -static const Guid IID_ISlangFileSystem = SLANG_UUID_ISlangFileSystem; -static const Guid IID_ISlangFileSystemExt = SLANG_UUID_ISlangFileSystemExt; -static const Guid IID_ISlangMutableFileSystem = SLANG_UUID_ISlangMutableFileSystem; - -static const Guid IID_SlangCacheFileSystem = SLANG_UUID_CacheFileSystem; - -SLANG_FORCE_INLINE static SlangResult _checkExt(FileSystemStyle style) { return Index(style) >= Index(FileSystemStyle::Ext) ? SLANG_OK : SLANG_E_NOT_IMPLEMENTED; } -SLANG_FORCE_INLINE static SlangResult _checkMutable(FileSystemStyle style) { return Index(style) >= Index(FileSystemStyle::Mutable) ? SLANG_OK : SLANG_E_NOT_IMPLEMENTED; } - -SLANG_FORCE_INLINE static bool _canCast(FileSystemStyle style, const Guid& guid) -{ - if (guid == IID_ISlangUnknown || guid == IID_ISlangFileSystem) - { - return true; - } - else if (guid == IID_ISlangFileSystemExt) - { - return Index(style) >= Index(FileSystemStyle::Ext); - } - else if (guid == IID_ISlangMutableFileSystem) - { - return Index(style) >= Index(FileSystemStyle::Mutable); - } - return false; -} - -static FileSystemStyle _getFileSystemStyle(ISlangFileSystem* system, ComPtr& out) -{ - SLANG_ASSERT(system); - - FileSystemStyle style = FileSystemStyle::Load; - - if (SLANG_SUCCEEDED(system->queryInterface(IID_ISlangMutableFileSystem, (void**)out.writeRef()))) - { - style = FileSystemStyle::Mutable; - } - else if (SLANG_SUCCEEDED(system->queryInterface(IID_ISlangFileSystemExt, (void**)out.writeRef()))) - { - style = FileSystemStyle::Ext; - } - else - { - style = FileSystemStyle::Load; - out = system; - } - - SLANG_ASSERT(out); - return style; -} - -// Cacluate a combined path, just using Path:: string processing -static SlangResult _calcCombinedPath(SlangPathType fromPathType, const char* fromPath, const char* path, ISlangBlob** pathOut) -{ - String relPath; - switch (fromPathType) - { - case SLANG_PATH_TYPE_FILE: - { - relPath = Path::combine(Path::getParentDirectory(fromPath), path); - break; - } - case SLANG_PATH_TYPE_DIRECTORY: - { - relPath = Path::combine(fromPath, path); - break; - } - } - - *pathOut = StringUtil::createStringBlob(relPath).detach(); - return SLANG_OK; -} - -/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!! OSFileSystem !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!*/ - -/* static */OSFileSystem OSFileSystem::g_load(FileSystemStyle::Load); -/* static */OSFileSystem OSFileSystem::g_ext(FileSystemStyle::Ext); -/* static */OSFileSystem OSFileSystem::g_mutable(FileSystemStyle::Mutable); - -ISlangUnknown* OSFileSystem::getInterface(const Guid& guid) -{ - return _canCast(m_style, guid) ? static_cast(this) : nullptr; -} - -static String _fixPathDelimiters(const char* pathIn) -{ -#if SLANG_WINDOWS_FAMILY - return pathIn; -#else - // To allow windows style \ delimiters on other platforms, we convert to our standard delimiter - String path(pathIn); - return StringUtil::calcCharReplaced(pathIn, '\\', Path::kPathDelimiter); -#endif -} - -SlangResult OSFileSystem::getFileUniqueIdentity(const char* pathIn, ISlangBlob** outUniqueIdentity) -{ - SLANG_RETURN_ON_FAIL(_checkExt(m_style)); - - // By default we use the canonical path to uniquely identify a file - return getCanonicalPath(pathIn, outUniqueIdentity); -} - -SlangResult OSFileSystem::getCanonicalPath(const char* path, ISlangBlob** outCanonicalPath) -{ - SLANG_RETURN_ON_FAIL(_checkExt(m_style)); - - String canonicalPath; - SLANG_RETURN_ON_FAIL(Path::getCanonical(_fixPathDelimiters(path), canonicalPath)); - *outCanonicalPath = StringUtil::createStringBlob(canonicalPath).detach(); - return SLANG_OK; -} - -SlangResult OSFileSystem::getSimplifiedPath(const char* pathIn, ISlangBlob** outSimplifiedPath) -{ - SLANG_RETURN_ON_FAIL(_checkExt(m_style)); - - String simplifiedPath = Path::simplify(pathIn); - *outSimplifiedPath = StringUtil::createStringBlob(simplifiedPath).detach(); - return SLANG_OK; -} - -SlangResult OSFileSystem::calcCombinedPath(SlangPathType fromPathType, const char* fromPath, const char* path, ISlangBlob** pathOut) -{ - SLANG_RETURN_ON_FAIL(_checkExt(m_style)); - - // Don't need to fix delimiters - because combine path handles both path delimiter types - return _calcCombinedPath(fromPathType, fromPath, path, pathOut); -} - -SlangResult SLANG_MCALL OSFileSystem::getPathType(const char* pathIn, SlangPathType* pathTypeOut) -{ - SLANG_RETURN_ON_FAIL(_checkExt(m_style)); - - return Path::getPathType(_fixPathDelimiters(pathIn), pathTypeOut); -} - - -SlangResult OSFileSystem::loadFile(char const* pathIn, ISlangBlob** outBlob) -{ - // Default implementation that uses the `core` libraries facilities for talking to the OS filesystem. - // - // TODO: we might want to conditionally compile these in, so that - // a user could create a build of Slang that doesn't include any OS - // filesystem calls. - - const String path = _fixPathDelimiters(pathIn); - if (!File::exists(path)) - { - return SLANG_E_NOT_FOUND; - } - - ScopedAllocation alloc; - SLANG_RETURN_ON_FAIL(File::readAllBytes(path, alloc)); - *outBlob = RawBlob::moveCreate(alloc).detach(); - return SLANG_OK; -} - -SlangResult OSFileSystem::enumeratePathContents(const char* path, FileSystemContentsCallBack callback, void* userData) -{ - SLANG_RETURN_ON_FAIL(_checkExt(m_style)); - - struct Visitor : Path::Visitor - { - void accept(Path::Type type, const UnownedStringSlice& filename) SLANG_OVERRIDE - { - m_buffer.Clear(); - m_buffer.append(filename); - - SlangPathType pathType; - switch (type) - { - case Path::Type::File: pathType = SLANG_PATH_TYPE_FILE; break; - case Path::Type::Directory: pathType = SLANG_PATH_TYPE_DIRECTORY; break; - default: return; - } - - m_callback(pathType, m_buffer.getBuffer(), m_userData); - } - - Visitor(FileSystemContentsCallBack callback, void* userData) : - m_callback(callback), - m_userData(userData) - { - } - StringBuilder m_buffer; - FileSystemContentsCallBack m_callback; - void* m_userData; - }; - - Visitor visitor(callback, userData); - Path::find(path, nullptr, &visitor); - - return SLANG_OK; -} - -SlangResult OSFileSystem::saveFile(const char* pathIn, const void* data, size_t size) -{ - SLANG_RETURN_ON_FAIL(_checkMutable(m_style)); - - const String path = _fixPathDelimiters(pathIn); - - try - { - FileStream stream(pathIn, FileMode::Create, FileAccess::Write, FileShare::ReadWrite); - - int64_t numWritten = stream.write(data, size); - - if (numWritten != int64_t(size)) - { - return SLANG_FAIL; - } - - } - catch (const IOException&) - { - return SLANG_E_CANNOT_OPEN; - } - - return SLANG_OK; -} - -SlangResult OSFileSystem::remove(const char* path) -{ - SLANG_RETURN_ON_FAIL(_checkMutable(m_style)); - return Path::remove(path); -} - -SlangResult OSFileSystem::createDirectory(const char* path) -{ - SLANG_RETURN_ON_FAIL(_checkMutable(m_style)); - return Path::createDirectory(path); -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! CacheFileSystem !!!!!!!!!!!!!!!!!!!!!!!!!!! - -/* static */ const Result CacheFileSystem::s_compressedResultToResult[] = -{ - SLANG_E_UNINITIALIZED, - SLANG_OK, ///< Ok - SLANG_E_NOT_FOUND, ///< File not found - SLANG_E_CANNOT_OPEN, ///< CannotOpen, - SLANG_FAIL, ///< Fail -}; - -/* static */CacheFileSystem::CompressedResult CacheFileSystem::toCompressedResult(Result res) -{ - if (SLANG_SUCCEEDED(res)) - { - return CompressedResult::Ok; - } - switch (res) - { - case SLANG_E_CANNOT_OPEN: return CompressedResult::CannotOpen; - case SLANG_E_NOT_FOUND: return CompressedResult::NotFound; - default: return CompressedResult::Fail; - } -} - -SLANG_NO_THROW SlangResult SLANG_MCALL CacheFileSystem::queryInterface(SlangUUID const& uuid, void** outObject) -{ - if (uuid == IID_SlangCacheFileSystem) - { - *outObject = this; - return SLANG_OK; - } - - if (_canCast(FileSystemStyle::Ext, uuid)) - { - addReference(); - *outObject = static_cast(this); - return SLANG_OK; - } - return SLANG_E_NO_INTERFACE; -} - -CacheFileSystem::CacheFileSystem(ISlangFileSystem* fileSystem, UniqueIdentityMode uniqueIdentityMode, PathStyle pathStyle) -{ - setInnerFileSystem(fileSystem, uniqueIdentityMode, pathStyle); -} - -CacheFileSystem::~CacheFileSystem() -{ - for (const auto& pair : m_uniqueIdentityMap) - { - PathInfo* pathInfo = pair.Value; - delete pathInfo; - } -} - -void CacheFileSystem::setInnerFileSystem(ISlangFileSystem* fileSystem, UniqueIdentityMode uniqueIdentityMode, PathStyle pathStyle) -{ - m_fileSystem = fileSystem; - - m_uniqueIdentityMode = uniqueIdentityMode; - m_pathStyle = pathStyle; - - m_fileSystemExt.setNull(); - - if (fileSystem) - { - // Try to get the more sophisticated interface - fileSystem->queryInterface(IID_ISlangFileSystemExt, (void**)m_fileSystemExt.writeRef()); - } - - switch (m_uniqueIdentityMode) - { - case UniqueIdentityMode::Default: - case UniqueIdentityMode::FileSystemExt: - { - // If it's not a complete file system, we will default to SimplifyAndHash style by default - m_uniqueIdentityMode = m_fileSystemExt ? UniqueIdentityMode::FileSystemExt : UniqueIdentityMode::SimplifyPathAndHash; - break; - } - default: break; - } - - if (pathStyle == PathStyle::Default) - { - // We'll assume it's simplify-able - m_pathStyle = PathStyle::Simplifiable; - // If we have fileSystemExt, we defer to that - if (m_fileSystemExt) - { - // We just defer to the m_fileSystem - m_pathStyle = PathStyle::FileSystemExt; - } - } - - // It can't be default - SLANG_ASSERT(m_uniqueIdentityMode != UniqueIdentityMode::Default); -} - -void CacheFileSystem::clearCache() -{ - for (const auto& pair : m_uniqueIdentityMap) - { - PathInfo* pathInfo = pair.Value; - delete pathInfo; - } - - m_uniqueIdentityMap.Clear(); - m_pathMap.Clear(); - - if (m_fileSystemExt) - { - m_fileSystemExt->clearCache(); - } -} - - -// Determines if we can simplify a path for a given mode -static bool _canSimplifyPath(CacheFileSystem::UniqueIdentityMode mode) -{ - typedef CacheFileSystem::UniqueIdentityMode UniqueIdentityMode; - switch (mode) - { - case UniqueIdentityMode::SimplifyPath: - case UniqueIdentityMode::SimplifyPathAndHash: - { - return true; - } - default: - { - return false; - } - } -} - -SlangResult CacheFileSystem::enumeratePathContents(const char* path, FileSystemContentsCallBack callback, void* userData) -{ - if (m_fileSystemExt) - { - return m_fileSystemExt->enumeratePathContents(path, callback, userData); - } - - // Okay.. the contents of the 'cache' *is* the filesystem. So lets iterate over that - // This will win no prizes for efficiency, but that is unlikely to matter for typical usage - - if (!_canSimplifyPath(m_uniqueIdentityMode)) - { - // As it stands if we can't simplify paths, it's kind of hard to make this - // all work. As we use the simplified path cache - return SLANG_E_NOT_IMPLEMENTED; - } - - // Simplify the path - String simplifiedPath = Path::simplify(path); - - // If the simplified path is just a . then we don't have any prefix - if (simplifiedPath == ".") - { - simplifiedPath = ""; - } - - for (auto& pair : m_pathMap) - { - // NOTE! The currentPath can be a *non* simplified path (the m_pathMap is the cache of paths simplified and other to a file/directory) - // Also note that there will always be the simplified version of the path in cache. - const String& currentPath = pair.Key; - - // If it doesn't start with simplified path, then it can't be a hit - if (!currentPath.startsWith(simplifiedPath)) - { - continue; - } - - UnownedStringSlice remaining(currentPath.getBuffer() + simplifiedPath.getLength(), currentPath.end()); - - // If it starts with a / delimiter strip it - if (remaining.getLength() > 0 && remaining[0] == '/') - { - remaining = UnownedStringSlice(remaining.begin() + 1, remaining.end()); - } - - // If it has a path separator then it's either not simplified - so we ignore (we only want to invoke on the simplified path version as there is only one - // of these for every PathInfo) - // or it is a child file/directory, and so we ignore that too. - if (remaining.indexOf('/') >= 0 || remaining.indexOf('\\') >= 0) - { - continue; - } - - // We *know* that remaining comes from the end of currentPath .We also know currentPath is zero terminated. - // So we can just use (normally this would be a problem because UnownedStringSlice is generally *not* followed by zero termination. - const char* foundPath = remaining.begin(); - // Let's check that fact... - SLANG_ASSERT(foundPath[remaining.getLength()] == 0); - - PathInfo* pathInfo = pair.Value; - - SlangPathType pathType; - if (SLANG_FAILED(_getPathType(pathInfo, currentPath.getBuffer(), &pathType))) - { - continue; - } - - callback(pathType, foundPath, userData); - } - - return SLANG_OK; -} - - -SlangResult CacheFileSystem::_calcUniqueIdentity(const String& path, String& outUniqueIdentity, ComPtr& outFileContents) -{ - switch (m_uniqueIdentityMode) - { - case UniqueIdentityMode::FileSystemExt: - { - // Try getting the uniqueIdentity by asking underlying file system - ComPtr uniqueIdentity; - SLANG_RETURN_ON_FAIL(m_fileSystemExt->getFileUniqueIdentity(path.getBuffer(), uniqueIdentity.writeRef())); - // Get the path as a string - outUniqueIdentity = StringUtil::getString(uniqueIdentity); - return SLANG_OK; - } - case UniqueIdentityMode::Path: - { - outUniqueIdentity = path; - return SLANG_OK; - } - case UniqueIdentityMode::SimplifyPath: - { - outUniqueIdentity = Path::simplify(path); - // If it still has relative elements can't uniquely identify, so give up - return Path::hasRelativeElement(outUniqueIdentity) ? SLANG_FAIL : SLANG_OK; - } - case UniqueIdentityMode::SimplifyPathAndHash: - case UniqueIdentityMode::Hash: - { - // If we don't have a file system -> assume cannot be found - if (m_fileSystem == nullptr) - { - return SLANG_E_NOT_FOUND; - } - - // I can only see if this is the same file as already loaded by loading the file and doing a hash - Result res = m_fileSystem->loadFile(path.getBuffer(), outFileContents.writeRef()); - if (SLANG_FAILED(res) || outFileContents == nullptr) - { - return SLANG_FAIL; - } - - // Calculate the hash on the contents - const uint64_t hash = getHashCode64((const char*)outFileContents->getBufferPointer(), outFileContents->getBufferSize()); - - String hashString = Path::getFileName(path); - hashString = hashString.toLower(); - - hashString.append(':'); - - // The uniqueIdentity is a combination of name and hash - hashString.append(hash, 16); - - outUniqueIdentity = hashString; - return SLANG_OK; - } - } - - return SLANG_FAIL; -} - -CacheFileSystem::PathInfo* CacheFileSystem::_resolveUniqueIdentityCacheInfo(const String& path) -{ - // Use the path to produce uniqueIdentity information - ComPtr fileContents; - String uniqueIdentity; - - SlangResult res = _calcUniqueIdentity(path, uniqueIdentity, fileContents); - if (SLANG_FAILED(res)) - { - // Was not able to create a uniqueIdentity - return failure as nullptr - return nullptr; - } - - // Now try looking up by uniqueIdentity path. If not found, add a new result - PathInfo* pathInfo = nullptr; - if (!m_uniqueIdentityMap.TryGetValue(uniqueIdentity, pathInfo)) - { - // Create with found uniqueIdentity - pathInfo = new PathInfo(uniqueIdentity); - m_uniqueIdentityMap.Add(uniqueIdentity, pathInfo); - } - - // At this point they must have same uniqueIdentity - SLANG_ASSERT(pathInfo->getUniqueIdentity() == uniqueIdentity); - - // If we have the file contents (because of calc-ing uniqueIdentity), and there isn't a read file blob already - // store the data as if read, so doesn't get read again - if (fileContents && !pathInfo->m_fileBlob) - { - pathInfo->m_fileBlob = fileContents; - pathInfo->m_loadFileResult = CompressedResult::Ok; - } - - return pathInfo; -} - -CacheFileSystem::PathInfo* CacheFileSystem::_resolveSimplifiedPathCacheInfo(const String& path) -{ - // If we can simplify the path, try looking up in path cache with simplified path (as long as it's different!) - if (_canSimplifyPath(m_uniqueIdentityMode)) - { - const String simplifiedPath = Path::simplify(path); - // Only lookup if the path is different - because otherwise will recurse forever... - if (simplifiedPath != path) - { - // This is a recursive call - and will ensure the simplified path is added to the cache - return _resolvePathCacheInfo(simplifiedPath); - } - } - - return _resolveUniqueIdentityCacheInfo(path); -} - -CacheFileSystem::PathInfo* CacheFileSystem::_resolvePathCacheInfo(const String& path) -{ - // Lookup in path cache - PathInfo* pathInfo; - if (m_pathMap.TryGetValue(path, pathInfo)) - { - // Found so done - return pathInfo; - } - - // Try getting or creating taking into account possible path simplification - pathInfo = _resolveSimplifiedPathCacheInfo(path); - // Always add the result to the path cache (even if null) - m_pathMap.Add(path, pathInfo); - return pathInfo; -} - -SlangResult CacheFileSystem::loadFile(char const* pathIn, ISlangBlob** blobOut) -{ - *blobOut = nullptr; - String path(pathIn); - PathInfo* info = _resolvePathCacheInfo(path); - if (!info) - { - return SLANG_FAIL; - } - - if (info->m_loadFileResult == CompressedResult::Uninitialized) - { - info->m_loadFileResult = toCompressedResult(m_fileSystem->loadFile(path.getBuffer(), info->m_fileBlob.writeRef())); - } - - *blobOut = info->m_fileBlob; - if (*blobOut) - { - (*blobOut)->addRef(); - } - return toResult(info->m_loadFileResult); -} - -SlangResult CacheFileSystem::getFileUniqueIdentity(const char* path, ISlangBlob** outUniqueIdentity) -{ - PathInfo* info = _resolvePathCacheInfo(path); - if (!info) - { - return SLANG_E_NOT_FOUND; - } - info->m_uniqueIdentity->addRef(); - *outUniqueIdentity = info->m_uniqueIdentity; - return SLANG_OK; -} - -SlangResult CacheFileSystem::calcCombinedPath(SlangPathType fromPathType, const char* fromPath, const char* path, ISlangBlob** pathOut) -{ - // Just defer to contained implementation - switch (m_pathStyle) - { - case PathStyle::FileSystemExt: - { - return m_fileSystemExt->calcCombinedPath(fromPathType, fromPath, path, pathOut); - } - default: - { - // Just use the default implementation - return _calcCombinedPath(fromPathType, fromPath, path, pathOut); - } - } -} - -SlangResult CacheFileSystem::_getPathType(PathInfo* info, const char* inPath, SlangPathType* outPathType) -{ - if (info->m_getPathTypeResult == CompressedResult::Uninitialized) - { - if (m_fileSystemExt) - { - info->m_getPathTypeResult = toCompressedResult(m_fileSystemExt->getPathType(inPath, &info->m_pathType)); - } - else - { - // Okay try to load the file - if (info->m_loadFileResult == CompressedResult::Uninitialized) - { - info->m_loadFileResult = toCompressedResult(m_fileSystem->loadFile(inPath, info->m_fileBlob.writeRef())); - } - - // Make the getPathResult the same as the load result - info->m_getPathTypeResult = info->m_loadFileResult; - // Just set to file... the result is what matters in this case - info->m_pathType = SLANG_PATH_TYPE_FILE; - } - } - - *outPathType = info->m_pathType; - return toResult(info->m_getPathTypeResult); -} - -SlangResult CacheFileSystem::getPathType(const char* inPath, SlangPathType* outPathType) -{ - PathInfo* info = _resolvePathCacheInfo(inPath); - if (!info) - { - return SLANG_E_NOT_FOUND; - } - - return _getPathType(info, inPath, outPathType); -} - -SlangResult CacheFileSystem::getSimplifiedPath(const char* path, ISlangBlob** outSimplifiedPath) -{ - // If we have a ISlangFileSystemExt we can just pass on the request to it - switch (m_pathStyle) - { - case PathStyle::FileSystemExt: - { - return m_fileSystemExt->getSimplifiedPath(path, outSimplifiedPath); - } - case PathStyle::Simplifiable: - { - String simplifiedPath = Path::simplify(path); - *outSimplifiedPath = StringUtil::createStringBlob(simplifiedPath).detach(); - return SLANG_OK; - } - default: return SLANG_E_NOT_IMPLEMENTED; - } -} - -SlangResult CacheFileSystem::getCanonicalPath(const char* path, ISlangBlob** outCanonicalPath) -{ - // A file must exist to get a canonical path... - PathInfo* info = _resolvePathCacheInfo(path); - if (!info) - { - return SLANG_E_NOT_FOUND; - } - - // We don't have this -> so read it ... - if (info->m_getCanonicalPathResult == CompressedResult::Uninitialized) - { - if (!m_fileSystemExt) - { - return SLANG_E_NOT_IMPLEMENTED; - } - - // Try getting the canonicalPath by asking underlying file system - ComPtr canonicalPathBlob; - SlangResult res = m_fileSystemExt->getCanonicalPath(path, canonicalPathBlob.writeRef()); - - if (SLANG_SUCCEEDED(res)) - { - // Get the path as a string - String canonicalPath = StringUtil::getString(canonicalPathBlob); - if (canonicalPath.getLength() > 0) - { - info->m_canonicalPath = new StringBlob(canonicalPath); - } - else - { - res = SLANG_FAIL; - } - } - - // Save the result - info->m_getCanonicalPathResult = toCompressedResult(res); - } - - if (info->m_canonicalPath) - { - info->m_canonicalPath->addRef(); - } - *outCanonicalPath = info->m_canonicalPath; - return SLANG_OK; -} - -/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! RelativeFileSystem !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ - -RelativeFileSystem::RelativeFileSystem(ISlangFileSystem* fileSystem, const String& relativePath, bool stripPath) : - m_relativePath(relativePath), - m_stripPath(stripPath) -{ - m_style = _getFileSystemStyle(fileSystem, m_fileSystem); -} - -ISlangUnknown* RelativeFileSystem::getInterface(const Guid& guid) -{ - return _canCast(m_style, guid) ? static_cast(this) : nullptr; -} - -SlangResult RelativeFileSystem::_calcCombinedPathInner(SlangPathType fromPathType, const char* fromPath, const char* path, ISlangBlob** outPath) -{ - ISlangFileSystemExt* fileSystem = _getExt(); - if (fileSystem) - { - return fileSystem->calcCombinedPath(fromPathType, fromPath, path, outPath); - } - else - { - return _calcCombinedPath(fromPathType, fromPath, path, outPath); - } -} - -SlangResult RelativeFileSystem::_getFixedPath(const char* path, String& outPath) -{ - ComPtr blob; - if (m_stripPath) - { - String strippedPath = Path::getFileName(path); - SLANG_RETURN_ON_FAIL(_calcCombinedPathInner(SLANG_PATH_TYPE_DIRECTORY, m_relativePath.getBuffer(), strippedPath.getBuffer(), blob.writeRef())); - } - else - { - SLANG_RETURN_ON_FAIL(_calcCombinedPathInner(SLANG_PATH_TYPE_DIRECTORY, m_relativePath.getBuffer(), path, blob.writeRef())); - } - - outPath = StringUtil::getString(blob); - return SLANG_OK; -} - -SlangResult RelativeFileSystem::loadFile(char const* path, ISlangBlob** outBlob) -{ - String fixedPath; - SLANG_RETURN_ON_FAIL(_getFixedPath(path, fixedPath)); - return m_fileSystem->loadFile(fixedPath.getBuffer(), outBlob); -} - -SlangResult RelativeFileSystem::getFileUniqueIdentity(const char* path, ISlangBlob** outUniqueIdentity) -{ - auto fileSystem = _getExt(); - if (!fileSystem) return SLANG_E_NOT_IMPLEMENTED; - - String fixedPath; - SLANG_RETURN_ON_FAIL(_getFixedPath(path, fixedPath)); - return fileSystem->getFileUniqueIdentity(fixedPath.getBuffer(), outUniqueIdentity); -} - -SlangResult RelativeFileSystem::calcCombinedPath(SlangPathType fromPathType, const char* fromPath, const char* path, ISlangBlob** outPath) -{ - auto fileSystem = _getExt(); - if (!fileSystem) return SLANG_E_NOT_IMPLEMENTED; - - String fixedFromPath; - SLANG_RETURN_ON_FAIL(_getFixedPath(fromPath, fixedFromPath)); - - return fileSystem->calcCombinedPath(fromPathType, fixedFromPath.getBuffer(), path, outPath); -} - -SlangResult RelativeFileSystem::getPathType(const char* path, SlangPathType* outPathType) -{ - auto fileSystem = _getExt(); - if (!fileSystem) return SLANG_E_NOT_IMPLEMENTED; - - String fixedPath; - SLANG_RETURN_ON_FAIL(_getFixedPath(path, fixedPath)); - return fileSystem->getPathType(fixedPath.getBuffer(), outPathType); -} - -SlangResult RelativeFileSystem::getSimplifiedPath(const char* path, ISlangBlob** outSimplifiedPath) -{ - auto fileSystem = _getExt(); - if (!fileSystem) return SLANG_E_NOT_IMPLEMENTED; - - return fileSystem->getSimplifiedPath(path, outSimplifiedPath); -} - -SlangResult RelativeFileSystem::getCanonicalPath(const char* path, ISlangBlob** outCanonicalPath) -{ - auto fileSystem = _getExt(); - if (!fileSystem) return SLANG_E_NOT_IMPLEMENTED; - - String fixedPath; - SLANG_RETURN_ON_FAIL(_getFixedPath(path, fixedPath)); - return fileSystem->getCanonicalPath(fixedPath.getBuffer(), outCanonicalPath); -} - -void RelativeFileSystem::clearCache() -{ - auto fileSystem = _getExt(); - if (!fileSystem) return; - - fileSystem->clearCache(); -} - -SlangResult RelativeFileSystem::enumeratePathContents(const char* path, FileSystemContentsCallBack callback, void* userData) -{ - auto fileSystem = _getExt(); - if (!fileSystem) return SLANG_E_NOT_IMPLEMENTED; - - String fixedPath; - SLANG_RETURN_ON_FAIL(_getFixedPath(path, fixedPath)); - return fileSystem->enumeratePathContents(fixedPath.getBuffer(), callback, userData); -} - -SlangResult RelativeFileSystem::saveFile(const char* path, const void* data, size_t size) -{ - auto fileSystem = _getMutable(); - if (!fileSystem) return SLANG_E_NOT_IMPLEMENTED; - - String fixedPath; - SLANG_RETURN_ON_FAIL(_getFixedPath(path, fixedPath)); - return fileSystem->saveFile(fixedPath.getBuffer(), data, size); -} - -SlangResult RelativeFileSystem::remove(const char* path) -{ - auto fileSystem = _getMutable(); - if (!fileSystem) return SLANG_E_NOT_IMPLEMENTED; - - String fixedPath; - SLANG_RETURN_ON_FAIL(_getFixedPath(path, fixedPath)); - return fileSystem->remove(fixedPath.getBuffer()); -} - -SlangResult RelativeFileSystem::createDirectory(const char* path) -{ - auto fileSystem = _getMutable(); - if (!fileSystem) return SLANG_E_NOT_IMPLEMENTED; - - String fixedPath; - SLANG_RETURN_ON_FAIL(_getFixedPath(path, fixedPath)); - return fileSystem->createDirectory(fixedPath.getBuffer()); -} - -} diff --git a/source/slang/slang-file-system.h b/source/slang/slang-file-system.h deleted file mode 100644 index d5145404d..000000000 --- a/source/slang/slang-file-system.h +++ /dev/null @@ -1,253 +0,0 @@ -#ifndef SLANG_FILE_SYSTEM_H_INCLUDED -#define SLANG_FILE_SYSTEM_H_INCLUDED - -#include "../../slang.h" -#include "../../slang-com-helper.h" -#include "../../slang-com-ptr.h" - -#include "../core/slang-blob.h" - -#include "../core/slang-string-util.h" -#include "../core/slang-dictionary.h" - -namespace Slang -{ - -enum class FileSystemStyle -{ - Load, ///< Equivalent to ISlangFileSystem - Ext, ///< Equivalent to ISlangFileSystemExt - Mutable, ///< Equivalent to ISlangModifyableFileSystem -}; - -// Can be used for all styles of file system -class OSFileSystem : public ISlangMutableFileSystem -{ -public: - // ISlangUnknown - // override ref counting, as DefaultFileSystem is singleton - SLANG_IUNKNOWN_QUERY_INTERFACE - SLANG_NO_THROW uint32_t SLANG_MCALL addRef() SLANG_OVERRIDE { return 1; } - SLANG_NO_THROW uint32_t SLANG_MCALL release() SLANG_OVERRIDE { return 1; } - - // ISlangFileSystem - virtual SLANG_NO_THROW SlangResult SLANG_MCALL loadFile(char const* path, ISlangBlob** outBlob) SLANG_OVERRIDE; - - // ISlangFileSystemExt - virtual SLANG_NO_THROW SlangResult SLANG_MCALL getFileUniqueIdentity(const char* path, ISlangBlob** uniqueIdentityOut) SLANG_OVERRIDE; - virtual SLANG_NO_THROW SlangResult SLANG_MCALL calcCombinedPath(SlangPathType fromPathType, const char* fromPath, const char* path, ISlangBlob** pathOut) SLANG_OVERRIDE; - virtual SLANG_NO_THROW SlangResult SLANG_MCALL getPathType(const char* path, SlangPathType* pathTypeOut) SLANG_OVERRIDE; - virtual SLANG_NO_THROW SlangResult SLANG_MCALL getSimplifiedPath(const char* path, ISlangBlob** outSimplifiedPath) SLANG_OVERRIDE; - virtual SLANG_NO_THROW SlangResult SLANG_MCALL getCanonicalPath(const char* path, ISlangBlob** outCanonicalPath) SLANG_OVERRIDE; - virtual SLANG_NO_THROW void SLANG_MCALL clearCache() SLANG_OVERRIDE {} - virtual SLANG_NO_THROW SlangResult SLANG_MCALL enumeratePathContents(const char* path, FileSystemContentsCallBack callback, void* userData) SLANG_OVERRIDE; - - // ISlangModifyableFileSystem - virtual SLANG_NO_THROW SlangResult SLANG_MCALL saveFile(const char* path, const void* data, size_t size) SLANG_OVERRIDE; - virtual SLANG_NO_THROW SlangResult SLANG_MCALL remove(const char* path) SLANG_OVERRIDE; - virtual SLANG_NO_THROW SlangResult SLANG_MCALL createDirectory(const char* path) SLANG_OVERRIDE; - - /// Get a default instance - static ISlangFileSystem* getLoadSingleton() { return &g_load; } - static ISlangFileSystemExt* getExtSingleton() { return &g_ext; } - static ISlangMutableFileSystem* getMutableSingleton() { return &g_mutable; } - -private: - - /// Make so not constructible - OSFileSystem(FileSystemStyle style): - m_style(style) - {} - - virtual ~OSFileSystem() {} - - ISlangUnknown* getInterface(const Guid& guid); - - FileSystemStyle m_style; - - static OSFileSystem g_load; - static OSFileSystem g_ext; - static OSFileSystem g_mutable; -}; - - #define SLANG_UUID_CacheFileSystem { 0x2f4d1d03, 0xa0d1, 0x434b, { 0x87, 0x7a, 0x65, 0x5, 0xa4, 0xa0, 0x9a, 0x3b } }; - -/* Wraps an underlying ISlangFileSystem or ISlangFileSystemExt and provides caching, -as well as emulation of methods if only has ISlangFileSystem interface. Will query capabilities -of the interface on the constructor. - -NOTE! That this behavior is the same as previously in that.... -1) calcRelativePath, just returns the path as processed by the Path:: methods -2) getUniqueIdentity behavior depends on the UniqueIdentityMode. -*/ -class CacheFileSystem: public ISlangFileSystemExt, public RefObject -{ - public: - - enum class PathStyle - { - Default, ///< Pass to say use the default - Simplifiable, ///< It can be simplified by Path::Simplify - FileSystemExt, ///< Use file system - }; - - enum UniqueIdentityMode - { - Default, ///< If passed, will default to the others depending on what kind of ISlangFileSystem is passed in - Path, ///< Just use the path as is (old style slang behavior) - SimplifyPath, ///< Use the input path 'simplified' (ie removing . and .. aspects) - Hash, ///< Use hashing - SimplifyPathAndHash, ///< Tries simplifying path first, and if that doesn't work it hashes - FileSystemExt, ///< Use the file system extended interface. - }; - - /* Cannot change order/add members without changing s_compressedResultToResult */ - enum class CompressedResult: uint8_t - { - Uninitialized, ///< Holds no value - Ok, ///< Ok - NotFound, ///< File not found - CannotOpen, ///< Cannot open - Fail, ///< Generic failure - CountOf, - }; - - struct PathInfo - { - PathInfo(const String& uniqueIdentity) - { - m_uniqueIdentity = new StringBlob(uniqueIdentity); - m_uniqueIdentity->addRef(); - - m_loadFileResult = CompressedResult::Uninitialized; - m_getPathTypeResult = CompressedResult::Uninitialized; - m_getCanonicalPathResult = CompressedResult::Uninitialized; - - m_pathType = SLANG_PATH_TYPE_FILE; - } - - /// Get the unique identity path as a string - const String& getUniqueIdentity() const { SLANG_ASSERT(m_uniqueIdentity); return m_uniqueIdentity->getString(); } - - RefPtr m_uniqueIdentity; - CompressedResult m_loadFileResult; - CompressedResult m_getPathTypeResult; - CompressedResult m_getCanonicalPathResult; - - SlangPathType m_pathType; - ComPtr m_fileBlob; - RefPtr m_canonicalPath; - }; - - Dictionary& getPathMap() { return m_pathMap; } - Dictionary& getUniqueMap() { return m_uniqueIdentityMap; } - - // ISlangUnknown - SLANG_NO_THROW SlangResult SLANG_MCALL queryInterface(SlangUUID const& uuid, void** outObject) SLANG_OVERRIDE; - SLANG_REF_OBJECT_IUNKNOWN_ADD_REF - SLANG_REF_OBJECT_IUNKNOWN_RELEASE - - // ISlangFileSystem - virtual SLANG_NO_THROW SlangResult SLANG_MCALL loadFile(char const* path, ISlangBlob** outBlob) SLANG_OVERRIDE; - - // ISlangFileSystemExt - virtual SLANG_NO_THROW SlangResult SLANG_MCALL getFileUniqueIdentity(const char* path, ISlangBlob** outUniqueIdentity) SLANG_OVERRIDE; - virtual SLANG_NO_THROW SlangResult SLANG_MCALL calcCombinedPath(SlangPathType fromPathType, const char* fromPath, const char* path, ISlangBlob** pathOut) SLANG_OVERRIDE; - virtual SLANG_NO_THROW SlangResult SLANG_MCALL getPathType(const char* path, SlangPathType* outPathType) SLANG_OVERRIDE; - virtual SLANG_NO_THROW SlangResult SLANG_MCALL getSimplifiedPath(const char* path, ISlangBlob** outSimplifiedPath) SLANG_OVERRIDE; - virtual SLANG_NO_THROW SlangResult SLANG_MCALL getCanonicalPath(const char* path, ISlangBlob** outCanonicalPath) SLANG_OVERRIDE; - virtual SLANG_NO_THROW void SLANG_MCALL clearCache() SLANG_OVERRIDE; - virtual SLANG_NO_THROW SlangResult SLANG_MCALL enumeratePathContents(const char* path, FileSystemContentsCallBack callback, void* userData) SLANG_OVERRIDE; - - /// Get the unique identity mode - UniqueIdentityMode getUniqueIdentityMode() const { return m_uniqueIdentityMode; } - /// Get the path style - PathStyle getPathStyle() const { return m_pathStyle; } - - /// Set the inner file system - void setInnerFileSystem(ISlangFileSystem* fileSystem, UniqueIdentityMode uniqueIdentityMode = UniqueIdentityMode::Default, PathStyle pathStyle = PathStyle::Default); - - /// Ctor - CacheFileSystem(ISlangFileSystem* fileSystem, UniqueIdentityMode uniqueIdentityMode = UniqueIdentityMode::Default, PathStyle pathStyle = PathStyle::Default); - /// Dtor - virtual ~CacheFileSystem(); - - static CompressedResult toCompressedResult(Result res); - static Result toResult(CompressedResult compRes) { return s_compressedResultToResult[int(compRes)]; } - static const Result s_compressedResultToResult[int(CompressedResult::CountOf)]; - -protected: - - /// Given a path, works out a uniqueIdentity, based on the uniqueIdentityMode. outFileContents will be set if file had to be read to produce the uniqueIdentity (ie with Hash) - SlangResult _calcUniqueIdentity(const String& path, String& outUniqueIdentity, ComPtr& outFileContents); - - /// For a given path gets a PathInfo. Can return nullptr, if it is not possible to create the PathInfo for some reason - PathInfo* _resolvePathCacheInfo(const String& path); - /// Turns the path into a uniqueIdentity, and then tries to look up in the uniqueIdentityMap. - PathInfo* _resolveUniqueIdentityCacheInfo(const String& path); - /// Will simplify the path (if possible) to lookup on the pathCache else will create on uniqueIdentityMap - PathInfo* _resolveSimplifiedPathCacheInfo(const String& path); - - SlangResult _getPathType(PathInfo* pathInfo, const char* inPath, SlangPathType* pathTypeOut); - - /* TODO: This may be improved by mapping to a ISlangBlob. This makes output fast and easy, and if constructed - as a StringBlob, we can just static_cast to get as a string to use internally, instead of constantly converting. - It is probably the case we cannot do dynamic_cast on ISlangBlob if we don't know where constructed -> if outside of slang codebase - doing such a cast can cause an exception. So we *never* want to do dynamic cast from blobs which could be created by external code. */ - - Dictionary m_pathMap; ///< Maps a path to a PathInfo (and unique identity) - Dictionary m_uniqueIdentityMap; ///< Maps a unique identity for a file to its contents. This OWNs the PathInfo. - - UniqueIdentityMode m_uniqueIdentityMode; ///< Determines how the 'uniqueIdentity' is produced. Cannot be Default in usage. - PathStyle m_pathStyle; ///< Style of paths - - ComPtr m_fileSystem; ///< Must always be set - ComPtr m_fileSystemExt; ///< Optionally set -> if nullptr will fall back on the m_fileSystem and emulate all the other methods of ISlangFileSystemExt -}; - -class RelativeFileSystem : public ISlangMutableFileSystem, public RefObject -{ -public: - SLANG_REF_OBJECT_IUNKNOWN_ALL - - // ISlangFileSystem - virtual SLANG_NO_THROW SlangResult SLANG_MCALL loadFile(char const* path, ISlangBlob** outBlob) SLANG_OVERRIDE; - - // ISlangFileSystemExt - virtual SLANG_NO_THROW SlangResult SLANG_MCALL getFileUniqueIdentity(const char* path, ISlangBlob** outUniqueIdentity) SLANG_OVERRIDE; - virtual SLANG_NO_THROW SlangResult SLANG_MCALL calcCombinedPath(SlangPathType fromPathType, const char* fromPath, const char* path, ISlangBlob** pathOut) SLANG_OVERRIDE; - virtual SLANG_NO_THROW SlangResult SLANG_MCALL getPathType(const char* path, SlangPathType* outPathType) SLANG_OVERRIDE; - virtual SLANG_NO_THROW SlangResult SLANG_MCALL getSimplifiedPath(const char* path, ISlangBlob** outSimplifiedPath) SLANG_OVERRIDE; - virtual SLANG_NO_THROW SlangResult SLANG_MCALL getCanonicalPath(const char* path, ISlangBlob** outCanonicalPath) SLANG_OVERRIDE; - virtual SLANG_NO_THROW void SLANG_MCALL clearCache() SLANG_OVERRIDE; - virtual SLANG_NO_THROW SlangResult SLANG_MCALL enumeratePathContents(const char* path, FileSystemContentsCallBack callback, void* userData) SLANG_OVERRIDE; - - // ISlangModifyableFileSystem - virtual SLANG_NO_THROW SlangResult SLANG_MCALL saveFile(const char* path, const void* data, size_t size) SLANG_OVERRIDE; - virtual SLANG_NO_THROW SlangResult SLANG_MCALL remove(const char* path) SLANG_OVERRIDE; - virtual SLANG_NO_THROW SlangResult SLANG_MCALL createDirectory(const char* path) SLANG_OVERRIDE; - - RelativeFileSystem(ISlangFileSystem* fileSystem, const String& relativePath, bool stripPath = false); - -protected: - - ISlangFileSystemExt* _getExt() { return Index(m_style) >= Index(FileSystemStyle::Ext) ? reinterpret_cast(m_fileSystem.get()) : nullptr; } - ISlangMutableFileSystem* _getMutable() { return Index(m_style) >= Index(FileSystemStyle::Mutable) ? reinterpret_cast(m_fileSystem.get()) : nullptr; } - - SlangResult _calcCombinedPathInner(SlangPathType fromPathType, const char* fromPath, const char* path, ISlangBlob** pathOut); - - SlangResult _getFixedPath(const char* path, String& outPath); - - ISlangUnknown* getInterface(const Guid& guid); - - bool m_stripPath; - - FileSystemStyle m_style; - ComPtr m_fileSystem; ///< NOTE! Has to match what's in style, such style can be reached via reinterpret_cast - - String m_relativePath; -}; - -} - -#endif // SLANG_FILE_SYSTEM_H_INCLUDED diff --git a/source/slang/slang-include-system.h b/source/slang/slang-include-system.h index 3b368b70d..70f6dd81e 100644 --- a/source/slang/slang-include-system.h +++ b/source/slang/slang-include-system.h @@ -2,7 +2,7 @@ #define SLANG_INCLUDE_SYSTEM_H // slang-include-system.h -#include "slang-source-loc.h" +#include "../compiler-core/slang-source-loc.h" namespace Slang { diff --git a/source/slang/slang-ir-legalize-types.cpp b/source/slang/slang-ir-legalize-types.cpp index 5d26741c0..9b95e069d 100644 --- a/source/slang/slang-ir-legalize-types.cpp +++ b/source/slang/slang-ir-legalize-types.cpp @@ -10,12 +10,13 @@ // fully specialized (no more generics/interfaces), so // that the concrete type of everything is known. +#include "../compiler-core/slang-name.h" + #include "slang-ir.h" #include "slang-ir-clone.h" #include "slang-ir-insts.h" #include "slang-legalize-types.h" #include "slang-mangle.h" -#include "slang-name.h" namespace Slang { diff --git a/source/slang/slang-ir.h b/source/slang/slang-ir.h index ce3a5c29a..b27542424 100644 --- a/source/slang/slang-ir.h +++ b/source/slang/slang-ir.h @@ -8,11 +8,10 @@ // #include "../core/slang-basic.h" - -#include "slang-source-loc.h" - #include "../core/slang-memory-arena.h" +#include "../compiler-core/slang-source-loc.h" + #include "slang-type-system-shared.h" namespace Slang { diff --git a/source/slang/slang-legalize-types.h b/source/slang/slang-legalize-types.h index 1201f669d..8f2a7572f 100644 --- a/source/slang/slang-legalize-types.h +++ b/source/slang/slang-legalize-types.h @@ -27,7 +27,8 @@ #include "slang-ir-insts.h" #include "slang-syntax.h" #include "slang-type-layout.h" -#include "slang-name.h" + +#include "../compiler-core/slang-name.h" namespace Slang { diff --git a/source/slang/slang-lexer.cpp b/source/slang/slang-lexer.cpp deleted file mode 100644 index 6c8e9474a..000000000 --- a/source/slang/slang-lexer.cpp +++ /dev/null @@ -1,1400 +0,0 @@ -// slang-lexer.cpp -#include "slang-lexer.h" - -// This file implements the lexer/scanner, which is responsible for taking a raw stream of -// input bytes and turning it into semantically useful tokens. -// - -#include "slang-name.h" -#include "slang-source-loc.h" - -#include - -namespace Slang -{ - Token TokenReader::getEndOfFileToken() - { - return Token(TokenType::EndOfFile, UnownedStringSlice::fromLiteral(""), SourceLoc()); - } - - const Token* TokenList::begin() const - { - SLANG_ASSERT(m_tokens.getCount()); - return &m_tokens[0]; - } - - const Token* TokenList::end() const - { - SLANG_ASSERT(m_tokens.getCount()); - SLANG_ASSERT(m_tokens[m_tokens.getCount() - 1].type == TokenType::EndOfFile); - return &m_tokens[m_tokens.getCount() - 1]; - } - - TokenSpan::TokenSpan() - : m_begin(nullptr) - , m_end (nullptr) - {} - - TokenReader::TokenReader() - : m_cursor(nullptr) - , m_end (nullptr) - {} - - - Token& TokenReader::peekToken() - { - return m_nextToken; - } - - TokenType TokenReader::peekTokenType() const - { - return m_nextToken.type; - } - - SourceLoc TokenReader::peekLoc() const - { - return m_nextToken.loc; - } - - Token TokenReader::advanceToken() - { - if (!m_cursor) - return getEndOfFileToken(); - - Token token = m_nextToken; - if (m_cursor < m_end) - { - m_cursor++; - m_nextToken = *m_cursor; - } - else - m_nextToken.type = TokenType::EndOfFile; - return token; - } - - // Lexer - - void Lexer::initialize( - SourceView* sourceView, - DiagnosticSink* sink, - NamePool* namePool, - MemoryArena* memoryArena, - OptionFlags optionFlags) - { - m_sourceView = sourceView; - m_sink = sink; - m_namePool = namePool; - m_memoryArena = memoryArena; - - auto content = sourceView->getContent(); - - m_begin = content.begin(); - m_cursor = content.begin(); - m_end = content.end(); - - // Set the start location - m_startLoc = sourceView->getRange().begin; - - m_tokenFlags = TokenFlag::AtStartOfLine | TokenFlag::AfterWhitespace; - m_lexerFlags = 0; - m_optionFlags = optionFlags; - } - - Lexer::~Lexer() - { - } - - enum { kEOF = -1 }; - - // Get the next input byte, without any handling of - // escaped newlines, non-ASCII code points, source locations, etc. - static int _peekRaw(Lexer* lexer) - { - // If we are at the end of the input, return a designated end-of-file value - if(lexer->m_cursor == lexer->m_end) - return kEOF; - - // Otherwise, just look at the next byte - return *lexer->m_cursor; - } - - // Read one input byte without any special handling (similar to `peekRaw`) - static int _advanceRaw(Lexer* lexer) - { - // The logic here is basically the same as for `peekRaw()`, - // escape we advance `cursor` if we aren't at the end. - - if (lexer->m_cursor == lexer->m_end) - return kEOF; - - return *lexer->m_cursor++; - } - - // When the cursor is already at the first byte of an end-of-line sequence, - // consume one or two bytes that compose the sequence. - // - // Basically, a newline is one of: - // - // "\n" - // "\r" - // "\r\n" - // "\n\r" - // - // We always look for the longest match possible. - // - static void _handleNewLineInner(Lexer* lexer, int c) - { - SLANG_ASSERT(c == '\n' || c == '\r'); - - int d = _peekRaw(lexer); - if( (c ^ d) == ('\n' ^ '\r') ) - { - _advanceRaw(lexer); - } - } - - // Look ahead one code point, dealing with complications like - // escaped newlines. - static int _peek(Lexer* lexer) - { - // Look at the next raw byte, and decide what to do - int c = _peekRaw(lexer); - - if(c == '\\') - { - // We might have a backslash-escaped newline. - // Look at the next byte (if any) to see. - // - // Note(tfoley): We are assuming a null-terminated input here, - // so that we can safely look at the next byte without issue. - int d = lexer->m_cursor[1]; - switch (d) - { - case '\r': case '\n': - { - // The newline was escaped, so return the code point after *that* - - int e = lexer->m_cursor[2]; - if ((d ^ e) == ('\r' ^ '\n')) - return lexer->m_cursor[3]; - return e; - } - - default: - break; - } - } - // TODO: handle UTF-8 encoding for non-ASCII code points here - - // Default case is to just hand along the byte we read as an ASCII code point. - return c; - } - - // Get the next code point from the input, and advance the cursor. - static int _advance(Lexer* lexer) - { - // We are going to loop, but only as a way of handling - // escaped line endings. - for (;;) - { - // If we are at the end of the input, then the task is easy. - if (lexer->m_cursor == lexer->m_end) - return kEOF; - - // Look at the next raw byte, and decide what to do - int c = *lexer->m_cursor++; - - if (c == '\\') - { - // We might have a backslash-escaped newline. - // Look at the next byte (if any) to see. - // - // Note(tfoley): We are assuming a null-terminated input here, - // so that we can safely look at the next byte without issue. - int d = *lexer->m_cursor; - switch (d) - { - case '\r': case '\n': - // handle the end-of-line for our source location tracking - lexer->m_cursor++; - _handleNewLineInner(lexer, d); - - lexer->m_tokenFlags |= TokenFlag::ScrubbingNeeded; - - // Now try again, looking at the character after the - // escaped newline. - continue; - - default: - break; - } - } - - // TODO: Need to handle non-ASCII code points. - - // Default case is to return the raw byte we saw. - return c; - } - } - - static void _handleNewLine(Lexer* lexer) - { - int c = _advance(lexer); - _handleNewLineInner(lexer, c); - } - - static void _lexLineComment(Lexer* lexer) - { - for(;;) - { - switch(_peek(lexer)) - { - case '\n': case '\r': case kEOF: - return; - - default: - _advance(lexer); - continue; - } - } - } - - static void _lexBlockComment(Lexer* lexer) - { - for(;;) - { - switch(_peek(lexer)) - { - case kEOF: - // TODO(tfoley) diagnostic! - return; - - case '\n': case '\r': - _handleNewLine(lexer); - continue; - - case '*': - _advance(lexer); - switch( _peek(lexer) ) - { - case '/': - _advance(lexer); - return; - - default: - continue; - } - - default: - _advance(lexer); - continue; - } - } - } - - static void _lexHorizontalSpace(Lexer* lexer) - { - for(;;) - { - switch(_peek(lexer)) - { - case ' ': case '\t': - _advance(lexer); - continue; - - default: - return; - } - } - } - - static void _lexIdentifier(Lexer* lexer) - { - for(;;) - { - int c = _peek(lexer); - if(('a' <= c ) && (c <= 'z') - || ('A' <= c) && (c <= 'Z') - || ('0' <= c) && (c <= '9') - || (c == '_')) - { - _advance(lexer); - continue; - } - - return; - } - } - - static SourceLoc _getSourceLoc(Lexer* lexer) - { - return lexer->m_startLoc + (lexer->m_cursor - lexer->m_begin); - } - - static void _lexDigits(Lexer* lexer, int base) - { - for(;;) - { - int c = _peek(lexer); - - int digitVal = 0; - switch(c) - { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - digitVal = c - '0'; - break; - - case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': - if(base <= 10) return; - digitVal = 10 + c - 'a'; - break; - - case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': - if(base <= 10) return; - digitVal = 10 + c - 'A'; - break; - - default: - // Not more digits! - return; - } - - if(digitVal >= base) - { - char buffer[] = { (char) c, 0 }; - lexer->m_sink->diagnose(_getSourceLoc(lexer), Diagnostics::invalidDigitForBase, buffer, base); - } - - _advance(lexer); - } - } - - static TokenType _maybeLexNumberSuffix(Lexer* lexer, TokenType tokenType) - { - // Be liberal in what we accept here, so that figuring out - // the semantics of a numeric suffix is left up to the parser - // and semantic checking logic. - // - for( ;;) - { - int c = _peek(lexer); - - // Accept any alphanumeric character, plus underscores. - if(('a' <= c ) && (c <= 'z') - || ('A' <= c) && (c <= 'Z') - || ('0' <= c) && (c <= '9') - || (c == '_')) - { - _advance(lexer); - continue; - } - - // Stop at the first character that isn't - // alphanumeric. - return tokenType; - } - } - - static bool _isNumberExponent(int c, int base) - { - switch( c ) - { - default: - return false; - - case 'e': case 'E': - if(base != 10) return false; - break; - - case 'p': case 'P': - if(base != 16) return false; - break; - } - - return true; - } - - static bool _maybeLexNumberExponent(Lexer* lexer, int base) - { - if(!_isNumberExponent(_peek(lexer), base)) - return false; - - // we saw an exponent marker - _advance(lexer); - - // Now start to read the exponent - switch( _peek(lexer) ) - { - case '+': case '-': - _advance(lexer); - break; - } - - // TODO(tfoley): it would be an error to not see digits here... - - _lexDigits(lexer, 10); - - return true; - } - - static TokenType _lexNumberAfterDecimalPoint(Lexer* lexer, int base) - { - _lexDigits(lexer, base); - _maybeLexNumberExponent(lexer, base); - - return _maybeLexNumberSuffix(lexer, TokenType::FloatingPointLiteral); - } - - static TokenType _lexNumber(Lexer* lexer, int base) - { - // TODO(tfoley): Need to consider whehter to allow any kind of digit separator character. - - TokenType tokenType = TokenType::IntegerLiteral; - - // At the start of things, we just concern ourselves with digits - _lexDigits(lexer, base); - - if( _peek(lexer) == '.' ) - { - tokenType = TokenType::FloatingPointLiteral; - - _advance(lexer); - _lexDigits(lexer, base); - } - - if( _maybeLexNumberExponent(lexer, base)) - { - tokenType = TokenType::FloatingPointLiteral; - } - - _maybeLexNumberSuffix(lexer, tokenType); - return tokenType; - } - - static int _maybeReadDigit(char const** ioCursor, int base) - { - auto& cursor = *ioCursor; - - for(;;) - { - int c = *cursor; - switch(c) - { - default: - return -1; - - // TODO: need to decide on digit separator characters - case '_': - cursor++; - continue; - - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - cursor++; - return c - '0'; - - case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': - if(base > 10) - { - cursor++; - return 10 + c - 'a'; - } - return -1; - - case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': - if(base > 10) - { - cursor++; - return 10 + c - 'A'; - } - return -1; - } - } - } - - static int _readOptionalBase(char const** ioCursor) - { - auto& cursor = *ioCursor; - if( *cursor == '0' ) - { - cursor++; - switch(*cursor) - { - case 'x': case 'X': - cursor++; - return 16; - - case 'b': case 'B': - cursor++; - return 2; - - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - return 8; - - default: - return 10; - } - } - - return 10; - } - - - - IntegerLiteralValue getIntegerLiteralValue(Token const& token, UnownedStringSlice* outSuffix) - { - IntegerLiteralValue value = 0; - - const UnownedStringSlice content = token.getContent(); - - char const* cursor = content.begin(); - char const* end = content.end(); - - int base = _readOptionalBase(&cursor); - - for( ;;) - { - int digit = _maybeReadDigit(&cursor, base); - if(digit < 0) - break; - - value = value*base + digit; - } - - if(outSuffix) - { - *outSuffix = UnownedStringSlice(cursor, end); - } - - return value; - } - - FloatingPointLiteralValue getFloatingPointLiteralValue(Token const& token, UnownedStringSlice* outSuffix) - { - FloatingPointLiteralValue value = 0; - - const UnownedStringSlice content = token.getContent(); - - char const* cursor = content.begin(); - char const* end = content.end(); - - int radix = _readOptionalBase(&cursor); - - bool seenDot = false; - FloatingPointLiteralValue divisor = 1; - for( ;;) - { - if(*cursor == '.') - { - cursor++; - seenDot = true; - continue; - } - - int digit = _maybeReadDigit(&cursor, radix); - if(digit < 0) - break; - - value = value*radix + digit; - - if(seenDot) - { - divisor *= radix; - } - } - - // Now read optional exponent - if(_isNumberExponent(*cursor, radix)) - { - cursor++; - - bool exponentIsNegative = false; - switch(*cursor) - { - default: - break; - - case '-': - exponentIsNegative = true; - cursor++; - break; - - case '+': - cursor++; - break; - } - - int exponentRadix = 10; - int exponent = 0; - - for(;;) - { - int digit = _maybeReadDigit(&cursor, exponentRadix); - if(digit < 0) - break; - - exponent = exponent*exponentRadix + digit; - } - - FloatingPointLiteralValue exponentBase = 10; - if(radix == 16) - { - exponentBase = 2; - } - - FloatingPointLiteralValue exponentValue = pow(exponentBase, exponent); - - if( exponentIsNegative ) - { - divisor *= exponentValue; - } - else - { - value *= exponentValue; - } - } - - value /= divisor; - - if(outSuffix) - { - *outSuffix = UnownedStringSlice(cursor, end); - } - - return value; - } - - static void _lexStringLiteralBody(Lexer* lexer, char quote) - { - for(;;) - { - int c = _peek(lexer); - if(c == quote) - { - _advance(lexer); - return; - } - - switch(c) - { - case kEOF: - lexer->m_sink->diagnose(_getSourceLoc(lexer), Diagnostics::endOfFileInLiteral); - return; - - case '\n': case '\r': - lexer->m_sink->diagnose(_getSourceLoc(lexer), Diagnostics::newlineInLiteral); - return; - - case '\\': - // Need to handle various escape sequence cases - _advance(lexer); - switch(_peek(lexer)) - { - case '\'': - case '\"': - case '\\': - case '?': - case 'a': - case 'b': - case 'f': - case 'n': - case 'r': - case 't': - case 'v': - _advance(lexer); - break; - - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': - // octal escape: up to 3 characters - _advance(lexer); - for(int ii = 0; ii < 3; ++ii) - { - int d = _peek(lexer); - if(('0' <= d) && (d <= '7')) - { - _advance(lexer); - continue; - } - else - { - break; - } - } - break; - - case 'x': - // hexadecimal escape: any number of characters - _advance(lexer); - for(;;) - { - int d = _peek(lexer); - if(('0' <= d) && (d <= '9') - || ('a' <= d) && (d <= 'f') - || ('A' <= d) && (d <= 'F')) - { - _advance(lexer); - continue; - } - else - { - break; - } - } - break; - - // TODO: Unicode escape sequences - - } - break; - - default: - _advance(lexer); - continue; - } - } - } - - String getStringLiteralTokenValue(Token const& token) - { - SLANG_ASSERT(token.type == TokenType::StringLiteral - || token.type == TokenType::CharLiteral); - - const UnownedStringSlice content = token.getContent(); - - char const* cursor = content.begin(); - char const* end = content.end(); - SLANG_UNREFERENCED_VARIABLE(end); - - auto quote = *cursor++; - SLANG_ASSERT(quote == '\'' || quote == '"'); - - StringBuilder valueBuilder; - for(;;) - { - SLANG_ASSERT(cursor != end); - - auto c = *cursor++; - - // If we see a closing quote, then we are at the end of the string literal - if(c == quote) - { - SLANG_ASSERT(cursor == end); - return valueBuilder.ProduceString(); - } - - // Characters that don't being escape sequences are easy; - // just append them to the buffer and move on. - if(c != '\\') - { - valueBuilder.Append(c); - continue; - } - - // Now we look at another character to figure out the kind of - // escape sequence we are dealing with: - - char d = *cursor++; - - switch(d) - { - // Simple characters that just needed to be escaped - case '\'': - case '\"': - case '\\': - case '?': - valueBuilder.Append(d); - continue; - - // Traditional escape sequences for special characters - case 'a': valueBuilder.Append('\a'); continue; - case 'b': valueBuilder.Append('\b'); continue; - case 'f': valueBuilder.Append('\f'); continue; - case 'n': valueBuilder.Append('\n'); continue; - case 'r': valueBuilder.Append('\r'); continue; - case 't': valueBuilder.Append('\t'); continue; - case 'v': valueBuilder.Append('\v'); continue; - - // Octal escape: up to 3 characterws - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': - { - cursor--; - int value = 0; - for(int ii = 0; ii < 3; ++ii) - { - d = *cursor; - if(('0' <= d) && (d <= '7')) - { - value = value*8 + (d - '0'); - - cursor++; - continue; - } - else - { - break; - } - } - - // TODO: add support for appending an arbitrary code point? - valueBuilder.Append((char) value); - } - continue; - - // Hexadecimal escape: any number of characters - case 'x': - { - cursor--; - int value = 0; - for(;;) - { - d = *cursor++; - int digitValue = 0; - if(('0' <= d) && (d <= '9')) - { - digitValue = d - '0'; - } - else if( ('a' <= d) && (d <= 'f') ) - { - digitValue = d - 'a'; - } - else if( ('A' <= d) && (d <= 'F') ) - { - digitValue = d - 'A'; - } - else - { - cursor--; - break; - } - - value = value*16 + digitValue; - } - - // TODO: add support for appending an arbitrary code point? - valueBuilder.Append((char) value); - } - continue; - - // TODO: Unicode escape sequences - - } - } - } - - String getFileNameTokenValue(Token const& token) - { - const UnownedStringSlice content = token.getContent(); - - // A file name usually doesn't process escape sequences - // (this is import on Windows, where `\\` is a valid - // path separator character). - - // Just trim off the first and last characters to remove the quotes - // (whether they were `""` or `<>`. - return String(content.begin() + 1, content.end() - 1); - } - - - - static TokenType _lexTokenImpl(Lexer* lexer, LexerFlags effectiveFlags) - { - if(effectiveFlags & kLexerFlag_ExpectDirectiveMessage) - { - for(;;) - { - switch(_peek(lexer)) - { - default: - _advance(lexer); - continue; - - case kEOF: case '\r': case '\n': - break; - } - break; - } - return TokenType::DirectiveMessage; - } - - switch(_peek(lexer)) - { - default: - break; - - case kEOF: - if((effectiveFlags & kLexerFlag_InDirective) != 0) - return TokenType::EndOfDirective; - return TokenType::EndOfFile; - - case '\r': case '\n': - if((effectiveFlags & kLexerFlag_InDirective) != 0) - return TokenType::EndOfDirective; - _handleNewLine(lexer); - return TokenType::NewLine; - - case ' ': case '\t': - _lexHorizontalSpace(lexer); - return TokenType::WhiteSpace; - - case '.': - _advance(lexer); - switch(_peek(lexer)) - { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - return _lexNumberAfterDecimalPoint(lexer, 10); - - // TODO(tfoley): handle ellipsis (`...`) - - default: - return TokenType::Dot; - } - - case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - return _lexNumber(lexer, 10); - - case '0': - { - auto loc = _getSourceLoc(lexer); - _advance(lexer); - switch(_peek(lexer)) - { - default: - return _maybeLexNumberSuffix(lexer, TokenType::IntegerLiteral); - - case '.': - _advance(lexer); - return _lexNumberAfterDecimalPoint(lexer, 10); - - case 'x': case 'X': - _advance(lexer); - return _lexNumber(lexer, 16); - - case 'b': case 'B': - _advance(lexer); - return _lexNumber(lexer, 2); - - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - lexer->m_sink->diagnose(loc, Diagnostics::octalLiteral); - return _lexNumber(lexer, 8); - } - } - - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': case 'g': case 'h': case 'i': case 'j': - case 'k': case 'l': case 'm': case 'n': case 'o': - case 'p': case 'q': case 'r': case 's': case 't': - case 'u': case 'v': case 'w': case 'x': case 'y': - case 'z': - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': case 'G': case 'H': case 'I': case 'J': - case 'K': case 'L': case 'M': case 'N': case 'O': - case 'P': case 'Q': case 'R': case 'S': case 'T': - case 'U': case 'V': case 'W': case 'X': case 'Y': - case 'Z': - case '_': - _lexIdentifier(lexer); - return TokenType::Identifier; - - case '\"': - _advance(lexer); - _lexStringLiteralBody(lexer, '\"'); - return TokenType::StringLiteral; - - case '\'': - _advance(lexer); - _lexStringLiteralBody(lexer, '\''); - return TokenType::CharLiteral; - - case '+': - _advance(lexer); - switch(_peek(lexer)) - { - case '+': _advance(lexer); return TokenType::OpInc; - case '=': _advance(lexer); return TokenType::OpAddAssign; - default: - return TokenType::OpAdd; - } - - case '-': - _advance(lexer); - switch(_peek(lexer)) - { - case '-': _advance(lexer); return TokenType::OpDec; - case '=': _advance(lexer); return TokenType::OpSubAssign; - case '>': _advance(lexer); return TokenType::RightArrow; - default: - return TokenType::OpSub; - } - - case '*': - _advance(lexer); - switch(_peek(lexer)) - { - case '=': _advance(lexer); return TokenType::OpMulAssign; - default: - return TokenType::OpMul; - } - - case '/': - _advance(lexer); - switch(_peek(lexer)) - { - case '=': _advance(lexer); return TokenType::OpDivAssign; - case '/': _advance(lexer); _lexLineComment(lexer); return TokenType::LineComment; - case '*': _advance(lexer); _lexBlockComment(lexer); return TokenType::BlockComment; - default: - return TokenType::OpDiv; - } - - case '%': - _advance(lexer); - switch(_peek(lexer)) - { - case '=': _advance(lexer); return TokenType::OpModAssign; - default: - return TokenType::OpMod; - } - - case '|': - _advance(lexer); - switch(_peek(lexer)) - { - case '|': _advance(lexer); return TokenType::OpOr; - case '=': _advance(lexer); return TokenType::OpOrAssign; - default: - return TokenType::OpBitOr; - } - - case '&': - _advance(lexer); - switch(_peek(lexer)) - { - case '&': _advance(lexer); return TokenType::OpAnd; - case '=': _advance(lexer); return TokenType::OpAndAssign; - default: - return TokenType::OpBitAnd; - } - - case '^': - _advance(lexer); - switch(_peek(lexer)) - { - case '=': _advance(lexer); return TokenType::OpXorAssign; - default: - return TokenType::OpBitXor; - } - - case '>': - _advance(lexer); - switch(_peek(lexer)) - { - case '>': - _advance(lexer); - switch(_peek(lexer)) - { - case '=': _advance(lexer); return TokenType::OpShrAssign; - default: return TokenType::OpRsh; - } - case '=': _advance(lexer); return TokenType::OpGeq; - default: - return TokenType::OpGreater; - } - - case '<': - _advance(lexer); - switch(_peek(lexer)) - { - case '<': - _advance(lexer); - switch(_peek(lexer)) - { - case '=': _advance(lexer); return TokenType::OpShlAssign; - default: return TokenType::OpLsh; - } - case '=': _advance(lexer); return TokenType::OpLeq; - default: - return TokenType::OpLess; - } - - case '=': - _advance(lexer); - switch(_peek(lexer)) - { - case '=': _advance(lexer); return TokenType::OpEql; - default: - return TokenType::OpAssign; - } - - case '!': - _advance(lexer); - switch(_peek(lexer)) - { - case '=': _advance(lexer); return TokenType::OpNeq; - default: - return TokenType::OpNot; - } - - case '#': - _advance(lexer); - switch(_peek(lexer)) - { - case '#': _advance(lexer); return TokenType::PoundPound; - default: - return TokenType::Pound; - } - - case '~': _advance(lexer); return TokenType::OpBitNot; - - case ':': - { - _advance(lexer); - if (_peek(lexer) == ':') - { - _advance(lexer); - return TokenType::Scope; - } - return TokenType::Colon; - } - case ';': _advance(lexer); return TokenType::Semicolon; - case ',': _advance(lexer); return TokenType::Comma; - - case '{': _advance(lexer); return TokenType::LBrace; - case '}': _advance(lexer); return TokenType::RBrace; - case '[': _advance(lexer); return TokenType::LBracket; - case ']': _advance(lexer); return TokenType::RBracket; - case '(': _advance(lexer); return TokenType::LParent; - case ')': _advance(lexer); return TokenType::RParent; - - case '?': _advance(lexer); return TokenType::QuestionMark; - case '@': _advance(lexer); return TokenType::At; - case '$': _advance(lexer); return TokenType::Dollar; - - } - - // TODO(tfoley): If we ever wanted to support proper Unicode - // in identifiers, etc., then this would be the right place - // to perform a more expensive dispatch based on the actual - // code point (and not just the first byte). - - { - // If none of the above cases matched, then we have an - // unexpected/invalid character. - - auto loc = _getSourceLoc(lexer); - int c = _advance(lexer); - if(!(effectiveFlags & kLexerFlag_IgnoreInvalid)) - { - auto sink = lexer->m_sink; - if(c >= 0x20 && c <= 0x7E) - { - char buffer[] = { (char) c, 0 }; - sink->diagnose(loc, Diagnostics::illegalCharacterPrint, buffer); - } - else - { - // Fallback: print as hexadecimal - sink->diagnose(loc, Diagnostics::illegalCharacterHex, String((unsigned char)c, 16)); - } - } - - return TokenType::Invalid; - } - } - - Token Lexer::lexToken(LexerFlags extraFlags) - { - auto& flags = m_tokenFlags; - for(;;) - { - Token token; - token.loc = _getSourceLoc(this); - - char const* textBegin = m_cursor; - - auto tokenType = _lexTokenImpl(this, m_lexerFlags | extraFlags); - - // The low-level lexer produces tokens for things we want - // to ignore, such as white space, so we skip them here. - switch(tokenType) - { - case TokenType::Invalid: - flags = 0; - continue; - - case TokenType::NewLine: - flags = TokenFlag::AtStartOfLine | TokenFlag::AfterWhitespace; - continue; - - case TokenType::WhiteSpace: - { - flags |= TokenFlag::AfterWhitespace; - continue; - } - case TokenType::BlockComment: - case TokenType::LineComment: - { - flags |= TokenFlag::AfterWhitespace; - if (m_optionFlags & OptionFlag::TokenizeComments) - { - // We don't break here, and use the normal token adding logic - // because we want the behavior to be identical (in terms of flags etc) - // as if TokenizeComments is not enabled - char const* textEnd = m_cursor; - - token.type = tokenType; - token.flags = m_tokenFlags; - token.setContent(UnownedStringSlice(textBegin, textEnd)); - - return token; - } - - continue; - } - - // We don't want to skip the end-of-file token, but we *do* - // want to make sure it has appropriate flags to make our life easier - case TokenType::EndOfFile: - flags |= TokenFlag::AtStartOfLine | TokenFlag::AfterWhitespace; - break; - - // We will also do some book-keeping around preprocessor directives here: - // - // If we see a `#` at the start of a line, then we are entering a - // preprocessor directive. - case TokenType::Pound: - if((flags & TokenFlag::AtStartOfLine) != 0) - m_lexerFlags |= kLexerFlag_InDirective; - break; - // - // And if we saw an end-of-line during a directive, then we are - // now leaving that directive. - // - case TokenType::EndOfDirective: - m_lexerFlags &= ~kLexerFlag_InDirective; - break; - - default: - break; - } - - token.type = tokenType; - - char const* textEnd = m_cursor; - - // Note(tfoley): `StringBuilder::Append()` seems to crash when appending zero bytes - if(textEnd != textBegin) - { - // "scrubbing" token value here to remove escaped newlines... - // - // Only perform this work if we encountered an escaped newline - // while lexing this token (e.g., keep a flag on the lexer), or - // do it on-demand when the actual value of the token is needed. - if (m_tokenFlags & TokenFlag::ScrubbingNeeded) - { - // Allocate space that will always be more than enough for stripped contents - char* startDst = (char*)m_memoryArena->allocateUnaligned(textEnd - textBegin); - char* dst = startDst; - - auto tt = textBegin; - while (tt != textEnd) - { - char c = *tt++; - if (c == '\\') - { - char d = *tt; - switch (d) - { - case '\r': case '\n': - { - tt++; - char e = *tt; - if ((d ^ e) == ('\r' ^ '\n')) - { - tt++; - } - } - continue; - - default: - break; - } - } - *dst++ = c; - } - token.setContent(UnownedStringSlice(startDst, dst)); - } - else - { - token.setContent(UnownedStringSlice(textBegin, textEnd)); - } - } - - token.flags = flags; - - m_tokenFlags = 0; - - if (tokenType == TokenType::Identifier) - { - token.setName(m_namePool->getName(token.getContent())); - } - - return token; - } - } - - TokenList Lexer::lexAllTokens() - { - TokenList tokenList; - for(;;) - { - Token token = lexToken(); - tokenList.add(token); - - if(token.type == TokenType::EndOfFile) - return tokenList; - } - } - - /* static */UnownedStringSlice Lexer::sourceLocationLexer(const UnownedStringSlice& in) - { - Lexer lexer; - - SourceManager sourceManager; - sourceManager.initialize(nullptr, nullptr); - - auto sourceFile = sourceManager.createSourceFileWithString(PathInfo::makeUnknown(), in); - auto sourceView = sourceManager.createSourceView(sourceFile, nullptr, SourceLoc::fromRaw(0)); - - DiagnosticSink sink(&sourceManager, nullptr); - - MemoryArena arena; - - RootNamePool rootNamePool; - NamePool namePool; - namePool.setRootNamePool(&rootNamePool); - - lexer.initialize(sourceView, &sink, &namePool, &arena); - - Token tok = lexer.lexToken(); - - if (tok.type == TokenType::Invalid) - { - return UnownedStringSlice(); - } - - const int offset = sourceView->getRange().getOffset(tok.loc); - - SLANG_ASSERT(offset >= 0 && offset <= in.getLength()); - SLANG_ASSERT(Index(offset + tok.charsCount) <= in.getLength()); - - return UnownedStringSlice(in.begin() + offset, in.begin() + offset + tok.charsCount); - } - -} diff --git a/source/slang/slang-lexer.h b/source/slang/slang-lexer.h deleted file mode 100644 index f1fe89516..000000000 --- a/source/slang/slang-lexer.h +++ /dev/null @@ -1,164 +0,0 @@ -#ifndef SLANG_LEXER_H -#define SLANG_LEXER_H - -#include "../core/slang-basic.h" -#include "slang-diagnostics.h" - -namespace Slang -{ - struct NamePool; - - // - - struct TokenList - { - const Token* begin() const; - const Token* end() const; - - SLANG_FORCE_INLINE void add(const Token& token) { m_tokens.add(token); } - - List m_tokens; - }; - - struct TokenSpan - { - TokenSpan(); - TokenSpan( - TokenList const& tokenList) - : m_begin(tokenList.begin()) - , m_end (tokenList.end ()) - {} - - const Token* begin() const { return m_begin; } - const Token* end () const { return m_end ; } - - int getCount() { return (int)(m_end - m_begin); } - - const Token* m_begin; - const Token* m_end; - }; - - struct TokenReader - { - Token m_nextToken; - TokenReader(); - explicit TokenReader(TokenSpan const& tokens) - : m_cursor(tokens.begin()) - , m_end (tokens.end ()) - , m_nextToken(tokens.begin() ? *tokens.begin() : getEndOfFileToken()) - {} - explicit TokenReader(TokenList const& tokens) - : m_cursor(tokens.begin()) - , m_end (tokens.end ()) - , m_nextToken(tokens.begin() ? *tokens.begin() : getEndOfFileToken()) - {} - struct ParsingCursor - { - Token nextToken; - const Token* tokenReaderCursor = nullptr; - }; - ParsingCursor getCursor() - { - ParsingCursor rs; - rs.nextToken = m_nextToken; - rs.tokenReaderCursor = m_cursor; - return rs; - } - void setCursor(ParsingCursor cursor) - { - m_cursor = cursor.tokenReaderCursor; - m_nextToken = cursor.nextToken; - } - bool isAtCursor(const ParsingCursor& cursor) const - { - return cursor.tokenReaderCursor == m_cursor; - } - bool isAtEnd() const { return m_cursor == m_end; } - Token& peekToken(); - TokenType peekTokenType() const; - SourceLoc peekLoc() const; - - Token advanceToken(); - - int getCount() { return (int)(m_end - m_cursor); } - - const Token* m_cursor; - const Token* m_end; - static Token getEndOfFileToken(); - }; - - typedef unsigned int LexerFlags; - enum - { - kLexerFlag_InDirective = 1 << 0, ///< Turn end-of-line and end-of-file into end-of-directive - kLexerFlag_ExpectFileName = 1 << 1, ///< Support `<>` style strings for file paths - kLexerFlag_IgnoreInvalid = 1 << 2, ///< Suppress errors about invalid/unsupported characters - kLexerFlag_ExpectDirectiveMessage = 1 << 3, ///< Don't lexer ordinary tokens, and instead consume rest of line as a string - }; - - struct Lexer - { - typedef uint32_t OptionFlags; - struct OptionFlag - { - enum Enum : OptionFlags - { - TokenizeComments = 1 << 0, ///< If set comments will be output to the token stream - }; - }; - - void initialize( - SourceView* sourceView, - DiagnosticSink* sink, - NamePool* namePool, - MemoryArena* memoryArena, - OptionFlags optionFlags = 0); - - ~Lexer(); - - /// Runs the lexer to try and extract a single token, which is returned. - /// This can be used by the DiagnosticSink to be able to display more appropriate - /// information when displaying a source location - such as underscoring the - /// token at that location. - /// - /// NOTE! This function is relatively slow, and is designed for use around this specific - /// purpose. It does not return a token or a token type, because that information is - /// not needed by the DiagnosticSink. - static UnownedStringSlice sourceLocationLexer(const UnownedStringSlice& in); - - Token lexToken(LexerFlags extraFlags = 0); - - TokenList lexAllTokens(); - - SourceView* m_sourceView; - DiagnosticSink* m_sink; - NamePool* m_namePool; - - char const* m_cursor; - - char const* m_begin; - char const* m_end; - - /// The starting sourceLoc (same as first location of SourceView) - SourceLoc m_startLoc; - - TokenFlags m_tokenFlags; - LexerFlags m_lexerFlags; - OptionFlags m_optionFlags; - - MemoryArena* m_memoryArena; - }; - - - // Helper routines for extracting values from tokens - String getStringLiteralTokenValue(Token const& token); - String getFileNameTokenValue(Token const& token); - - typedef int64_t IntegerLiteralValue; - typedef double FloatingPointLiteralValue; - - IntegerLiteralValue getIntegerLiteralValue(Token const& token, UnownedStringSlice* outSuffix = 0); - FloatingPointLiteralValue getFloatingPointLiteralValue(Token const& token, UnownedStringSlice* outSuffix = 0); -} - -#endif diff --git a/source/slang/slang-lookup.cpp b/source/slang/slang-lookup.cpp index c50364201..f41d8ff3b 100644 --- a/source/slang/slang-lookup.cpp +++ b/source/slang/slang-lookup.cpp @@ -1,6 +1,7 @@ // slang-lookup.cpp #include "slang-lookup.h" -#include "slang-name.h" + +#include "../compiler-core/slang-name.h" namespace Slang { diff --git a/source/slang/slang-mangle.cpp b/source/slang/slang-mangle.cpp index a08b05a5d..ed38a1261 100644 --- a/source/slang/slang-mangle.cpp +++ b/source/slang/slang-mangle.cpp @@ -1,6 +1,6 @@ #include "slang-mangle.h" -#include "slang-name.h" +#include "../compiler-core/slang-name.h" #include "slang-syntax.h" namespace Slang diff --git a/source/slang/slang-name.cpp b/source/slang/slang-name.cpp deleted file mode 100644 index b6035982b..000000000 --- a/source/slang/slang-name.cpp +++ /dev/null @@ -1,42 +0,0 @@ -// slang-name.cpp -#include "slang-name.h" - -namespace Slang { - -String getText(Name* name) -{ - if (!name) return String(); - return name->text; -} - -UnownedStringSlice getUnownedStringSliceText(Name* name) -{ - return name ? name->text.getUnownedSlice() : UnownedStringSlice(); -} - -const char* getCstr(Name* name) -{ - return name ? name->text.getBuffer() : nullptr; -} - -Name* NamePool::getName(String const& text) -{ - RefPtr name; - if (rootPool->names.TryGetValue(text, name)) - return name; - - name = new Name(); - name->text = text; - rootPool->names.Add(text, name); - return name; -} - -Name* NamePool::tryGetName(String const& text) -{ - RefPtr name; - if (rootPool->names.TryGetValue(text, name)) - return name; - return nullptr; -} - -} // namespace Slang diff --git a/source/slang/slang-name.h b/source/slang/slang-name.h deleted file mode 100644 index cf702686b..000000000 --- a/source/slang/slang-name.h +++ /dev/null @@ -1,89 +0,0 @@ -// slang-name.h -#ifndef SLANG_NAME_H_INCLUDED -#define SLANG_NAME_H_INCLUDED - -// This file defines the `Name` type, used to represent -// the name of types, variables, etc. in the AST. - -#include "../core/slang-basic.h" - -namespace Slang { - -// The `Name` type is used to represent the name of a type, variable, etc. -// -// The key benefit of using `Name`s instead of raw strings is that `Name`s -// can be compared for equality just by testing pointer equality. Names -// also don't require any memory management; you can just retain an ordinary -// pointer to one and not deal with reference-counting overhead. -// -// In order to provide these benefits, a `Name` can only be created using -// a `NamePool` that owns the allocations for all the names (so they get -// cleaned up when the pool is deleted), and which is responsible for -// ensuring the uniqueness of name objects. -// -class Name : public RefObject -{ -public: - // The raw text of the name. - // - // Note that at some point in the future we might have other categories - // of name than "simple" names, and so this might change to a structured - // ADT instead of a simple string. - String text; -}; - -// Get the textual string representation of a name -// (e.g., so that it can be printed). -String getText(Name* name); - -/// Get the text as unowned string slice -UnownedStringSlice getUnownedStringSliceText(Name* name); - -// Get a name as a C style string, or nullptr if name is nullptr -const char* getCstr(Name* name); - -// A `RootNamePool` is used to store and look up names. -// If two systems need to work together with names, and be sure that they -// get equivalent names for a string like `"Foo"`, then they need to use -// the same root name pool (directly or indirectly). -// -struct RootNamePool -{ - // The mapping from text strings to the corresponding name. - Dictionary > names; -}; - -// A `NamePool` is effectively a way of storing a subset of the -// names that have been created through a `RootNamePool`. -// -// The intention is that eventually we will add the ability to clean -// up a `NamePool`, and remove the names it created from the corresponding -// `RootNamePool` *if* those names are no longer in use. -// -// The goal of such an approach would be to ensure that the memory -// usage of a `Session` can't bloat over time just because of multiple -// `CompileRequest`s being created, used, and then destroyed (each time -// adding just a few more strings to the name mapping). -// -struct NamePool -{ - // Find or create the `Name` that represents the given `text`. - Name* getName(String const& text); - // Try find the `Name` that represents the given `text`. - // If the name does not exist, return nullptr - Name* tryGetName(String const& text); - // Set the parent name pool to use for lookup - void setRootNamePool(RootNamePool* rootNamePool) - { - this->rootPool = rootNamePool; - } - - // - - // The root name pool to use for storage/lookup - RootNamePool* rootPool = nullptr; -}; - -} // namespace Slang - -#endif diff --git a/source/slang/slang-options.cpp b/source/slang/slang-options.cpp index fcc2e0e4c..b88fa9bca 100644 --- a/source/slang/slang-options.cpp +++ b/source/slang/slang-options.cpp @@ -10,11 +10,10 @@ #include "slang-compiler.h" #include "slang-profile.h" -#include "slang-file-system.h" - #include "slang-repro.h" #include "slang-serialize-ir.h" +#include "../core/slang-file-system.h" #include "../core/slang-type-text-util.h" #include "../core/slang-hex-dump-util.h" diff --git a/source/slang/slang-parser.h b/source/slang/slang-parser.h index a1077d4a7..0ec2dcb8a 100644 --- a/source/slang/slang-parser.h +++ b/source/slang/slang-parser.h @@ -1,7 +1,8 @@ #ifndef SLANG_PARSER_H #define SLANG_PARSER_H -#include "slang-lexer.h" +#include "../compiler-core/slang-lexer.h" + #include "slang-compiler.h" #include "slang-syntax.h" diff --git a/source/slang/slang-preprocessor.cpp b/source/slang/slang-preprocessor.cpp index f9d18332d..bc78e4d31 100644 --- a/source/slang/slang-preprocessor.cpp +++ b/source/slang/slang-preprocessor.cpp @@ -3,7 +3,7 @@ #include "slang-compiler.h" #include "slang-diagnostics.h" -#include "slang-lexer.h" +#include "../compiler-core/slang-lexer.h" // Needed so that we can construct modifier syntax to represent GLSL directives #include "slang-syntax.h" diff --git a/source/slang/slang-preprocessor.h b/source/slang/slang-preprocessor.h index efecb912b..9de82b9f2 100644 --- a/source/slang/slang-preprocessor.h +++ b/source/slang/slang-preprocessor.h @@ -4,7 +4,7 @@ #include "../core/slang-basic.h" -#include "slang-lexer.h" +#include "../compiler-core/slang-lexer.h" #include "slang-include-system.h" diff --git a/source/slang/slang-repro.cpp b/source/slang/slang-repro.cpp index 61ab3b75d..3618568c1 100644 --- a/source/slang/slang-repro.cpp +++ b/source/slang/slang-repro.cpp @@ -10,7 +10,7 @@ #include "slang-options.h" -#include "slang-source-loc.h" +#include "../compiler-core/slang-source-loc.h" namespace Slang { diff --git a/source/slang/slang-repro.h b/source/slang/slang-repro.h index dd9984395..38a76a50a 100644 --- a/source/slang/slang-repro.h +++ b/source/slang/slang-repro.h @@ -9,8 +9,7 @@ #include "slang-compiler.h" #include "../core/slang-offset-container.h" - -#include "slang-file-system.h" +#include "../core/slang-file-system.h" namespace Slang { diff --git a/source/slang/slang-serialize-ir-types.h b/source/slang/slang-serialize-ir-types.h index 326e41f6e..41759d378 100644 --- a/source/slang/slang-serialize-ir-types.h +++ b/source/slang/slang-serialize-ir-types.h @@ -9,8 +9,8 @@ #include "slang-serialize-types.h" #include "slang-serialize-source-loc.h" -#include "slang-name.h" -#include "slang-source-loc.h" +#include "../compiler-core/slang-name.h" +#include "../compiler-core/slang-source-loc.h" #include "slang-ir.h" diff --git a/source/slang/slang-serialize-misc-type-info.h b/source/slang/slang-serialize-misc-type-info.h index 08fd1269d..191514785 100644 --- a/source/slang/slang-serialize-misc-type-info.h +++ b/source/slang/slang-serialize-misc-type-info.h @@ -4,7 +4,7 @@ #include "slang-serialize-type-info.h" -#include "slang-source-loc.h" +#include "../compiler-core/slang-source-loc.h" #include "slang-compiler.h" namespace Slang { diff --git a/source/slang/slang-serialize-reflection.h b/source/slang/slang-serialize-reflection.h index 7eaf8543c..5ae87877e 100644 --- a/source/slang/slang-serialize-reflection.h +++ b/source/slang/slang-serialize-reflection.h @@ -2,7 +2,7 @@ #ifndef SLANG_SERIALIZE_REFLECTION_H #define SLANG_SERIALIZE_REFLECTION_H -#include "slang-name.h" +#include "../compiler-core/slang-name.h" namespace Slang { diff --git a/source/slang/slang-serialize-source-loc.h b/source/slang/slang-serialize-source-loc.h index 5ebd264cc..595a55ea6 100644 --- a/source/slang/slang-serialize-source-loc.h +++ b/source/slang/slang-serialize-source-loc.h @@ -8,8 +8,8 @@ #include "slang-serialize-types.h" -#include "slang-name.h" -#include "slang-source-loc.h" +#include "../compiler-core/slang-name.h" +#include "../compiler-core/slang-source-loc.h" namespace Slang { diff --git a/source/slang/slang-serialize.h b/source/slang/slang-serialize.h index ff402b35c..990a36adc 100644 --- a/source/slang/slang-serialize.h +++ b/source/slang/slang-serialize.h @@ -11,7 +11,7 @@ #include "slang-serialize-types.h" -#include "slang-name.h" +#include "../compiler-core/slang-name.h" namespace Slang { diff --git a/source/slang/slang-source-loc.cpp b/source/slang/slang-source-loc.cpp deleted file mode 100644 index 4b589fbf3..000000000 --- a/source/slang/slang-source-loc.cpp +++ /dev/null @@ -1,689 +0,0 @@ -// slang-source-loc.cpp -#include "slang-source-loc.h" - -#include "../core/slang-string-util.h" - -namespace Slang { - -/* !!!!!!!!!!!!!!!!!!!!!!!!! SourceView !!!!!!!!!!!!!!!!!!!!!!!!!!!! */ - -const String PathInfo::getMostUniqueIdentity() const -{ - switch (type) - { - case Type::Normal: return uniqueIdentity; - case Type::FoundPath: - case Type::FromString: - { - return foundPath; - } - default: return ""; - } -} - -bool PathInfo::operator==(const ThisType& rhs) const -{ - // They must be the same type - if (type != rhs.type) - { - return false; - } - - switch (type) - { - case Type::TokenPaste: - case Type::TypeParse: - case Type::Unknown: - case Type::CommandLine: - { - return true; - } - case Type::Normal: - { - return foundPath == rhs.foundPath && uniqueIdentity == rhs.uniqueIdentity; - } - case Type::FromString: - case Type::FoundPath: - { - // Only have a found path - return foundPath == rhs.foundPath; - } - default: break; - } - - return false; -} - -void PathInfo::appendDisplayName(StringBuilder& out) const -{ - switch (type) - { - case Type::TokenPaste: out << "[Token Paste]"; break; - case Type::TypeParse: out << "[Type Parse]"; break; - case Type::Unknown: out << "[Unknown]"; break; - case Type::CommandLine: out << "[Command Line]"; break; - case Type::Normal: - case Type::FromString: - case Type::FoundPath: - { - - out.appendChar('"'); - StringUtil::appendEscaped(foundPath.getUnownedSlice(), out); - out.appendChar('"'); - break; - } - default: break; - } -} - -/* !!!!!!!!!!!!!!!!!!!!!!!!! SourceView !!!!!!!!!!!!!!!!!!!!!!!!!!!! */ - -int SourceView::findEntryIndex(SourceLoc sourceLoc) const -{ - if (!m_range.contains(sourceLoc)) - { - return -1; - } - - const auto rawValue = sourceLoc.getRaw(); - - Index hi = m_entries.getCount(); - // If there are no entries, or it is in front of the first entry, then there is no associated entry - if (hi == 0 || - m_entries[0].m_startLoc.getRaw() > sourceLoc.getRaw()) - { - return -1; - } - - Index lo = 0; - while (lo + 1 < hi) - { - const Index mid = (hi + lo) >> 1; - const Entry& midEntry = m_entries[mid]; - SourceLoc::RawValue midValue = midEntry.m_startLoc.getRaw(); - if (midValue <= rawValue) - { - // The location we seek is at or after this entry - lo = mid; - } - else - { - // The location we seek is before this entry - hi = mid; - } - } - - return int(lo); -} - -void SourceView::addLineDirective(SourceLoc directiveLoc, StringSlicePool::Handle pathHandle, int line) -{ - SLANG_ASSERT(pathHandle != StringSlicePool::Handle(0)); - SLANG_ASSERT(m_range.contains(directiveLoc)); - - // Check that the directiveLoc values are always increasing - SLANG_ASSERT(m_entries.getCount() == 0 || (m_entries.getLast().m_startLoc.getRaw() < directiveLoc.getRaw())); - - // Calculate the offset - const int offset = m_range.getOffset(directiveLoc); - - // Get the line index in the original file - const int lineIndex = m_sourceFile->calcLineIndexFromOffset(offset); - - Entry entry; - entry.m_startLoc = directiveLoc; - entry.m_pathHandle = pathHandle; - - // We also need to make sure that any lookups for line numbers will - // get corrected based on this files location. - // We assume the line number coming from the directive is a line number, NOT an index, so the correction needs + 1 - // There is an additional + 1 because we want the NEXT line - ie the line after the #line directive, to the specified value - // Taking both into account means +2 is correct 'fix' - entry.m_lineAdjust = line - (lineIndex + 2); - - m_entries.add(entry); -} - -void SourceView::addLineDirective(SourceLoc directiveLoc, const String& path, int line) -{ - StringSlicePool::Handle pathHandle = getSourceManager()->getStringSlicePool().add(path.getUnownedSlice()); - return addLineDirective(directiveLoc, pathHandle, line); -} - -void SourceView::addDefaultLineDirective(SourceLoc directiveLoc) -{ - SLANG_ASSERT(m_range.contains(directiveLoc)); - // Check that the directiveLoc values are always increasing - SLANG_ASSERT(m_entries.getCount() == 0 || (m_entries.getLast().m_startLoc.getRaw() < directiveLoc.getRaw())); - - // Well if there are no entries, or the last one puts it in default case, then we don't need to add anything - if (m_entries.getCount() == 0 || (m_entries.getCount() && m_entries.getLast().isDefault())) - { - return; - } - - Entry entry; - entry.m_startLoc = directiveLoc; - entry.m_lineAdjust = 0; // No line adjustment... we are going back to default - entry.m_pathHandle = StringSlicePool::Handle(0); // Mark that there is no path, and that this is a 'default' - - SLANG_ASSERT(entry.isDefault()); - - m_entries.add(entry); -} - -HumaneSourceLoc SourceView::getHumaneLoc(SourceLoc loc, SourceLocType type) -{ - const int offset = m_range.getOffset(loc); - - // We need the line index from the original source file - const int lineIndex = m_sourceFile->calcLineIndexFromOffset(offset); - - // TODO: we should really translate the byte index in the line - // to deal with: - // - // - Non-ASCII characters, while might consume multiple bytes - // - // - Tab characters, which should really adjust how we report - // columns (although how are we supposed to know the setting - // that an IDE expects us to use when reporting locations?) - const int columnIndex = m_sourceFile->calcColumnIndex(lineIndex, offset); - - HumaneSourceLoc humaneLoc; - humaneLoc.column = columnIndex + 1; - humaneLoc.line = lineIndex + 1; - - // Make up a default entry - StringSlicePool::Handle pathHandle = StringSlicePool::Handle(0); - - // Only bother looking up the entry information if we want a 'Normal' lookup - const int entryIndex = (type == SourceLocType::Nominal) ? findEntryIndex(loc) : -1; - if (entryIndex >= 0) - { - const Entry& entry = m_entries[entryIndex]; - // Adjust the line - humaneLoc.line += entry.m_lineAdjust; - // Get the pathHandle.. - pathHandle = entry.m_pathHandle; - } - - humaneLoc.pathInfo = _getPathInfoFromHandle(pathHandle); - return humaneLoc; -} - -PathInfo SourceView::_getPathInfo() const -{ - if (m_viewPath.getLength()) - { - PathInfo pathInfo(m_sourceFile->getPathInfo()); - pathInfo.foundPath = m_viewPath; - return pathInfo; - } - else - { - return m_sourceFile->getPathInfo(); - } -} - -PathInfo SourceView::_getPathInfoFromHandle(StringSlicePool::Handle pathHandle) const -{ - // If there is no override path, then just the source files path - if (pathHandle == StringSlicePool::Handle(0)) - { - return _getPathInfo(); - } - else - { - return PathInfo::makePath(getSourceManager()->getStringSlicePool().getSlice(pathHandle)); - } -} - -PathInfo SourceView::getPathInfo(SourceLoc loc, SourceLocType type) -{ - if (type == SourceLocType::Actual) - { - return _getPathInfo(); - } - - const int entryIndex = findEntryIndex(loc); - return _getPathInfoFromHandle((entryIndex >= 0) ? m_entries[entryIndex].m_pathHandle : StringSlicePool::Handle(0)); -} - -/* !!!!!!!!!!!!!!!!!!!!!!! SourceFile !!!!!!!!!!!!!!!!!!!!!!!!!!!! */ - -void SourceFile::setLineBreakOffsets(const uint32_t* offsets, UInt numOffsets) -{ - m_lineBreakOffsets.clear(); - m_lineBreakOffsets.addRange(offsets, numOffsets); -} - -const List& SourceFile::getLineBreakOffsets() -{ - // We now have a raw input file that we can search for line breaks. - // We obviously don't want to do a linear scan over and over, so we will - // cache an array of line break locations in the file. - if (m_lineBreakOffsets.getCount() == 0) - { - UnownedStringSlice content(getContent()), line; - char const* contentBegin = content.begin(); - while (StringUtil::extractLine(content, line)) - { - m_lineBreakOffsets.add(uint32_t(line.begin() - contentBegin)); - } - // Note that we do *not* treat the end of the file as a line - // break, because otherwise we would report errors like - // "end of file inside string literal" with a line number - // that points at a line that doesn't exist. - } - - return m_lineBreakOffsets; -} - -SourceFile::OffsetRange SourceFile::getOffsetRangeAtLineIndex(Index lineIndex) -{ - const List& offsets = getLineBreakOffsets(); - const Index count = offsets.getCount(); - - if (lineIndex >= count - 1) - { - // Work out the line start - const uint32_t offsetEnd = uint32_t(getContentSize()); - const uint32_t offsetStart = (lineIndex >= count) ? offsetEnd : offsets[lineIndex]; - // The line is the span from start, to the end of the content - return OffsetRange{ offsetStart, offsetEnd }; - } - else - { - const uint32_t offsetStart = offsets[lineIndex]; - const uint32_t offsetEnd = offsets[lineIndex + 1]; - return OffsetRange { offsetStart, offsetEnd }; - } -} - -UnownedStringSlice SourceFile::getLineAtIndex(Index lineIndex) -{ - const OffsetRange range = getOffsetRangeAtLineIndex(lineIndex); - - if (range.isValid() && hasContent()) - { - const UnownedStringSlice content = getContent(); - SLANG_ASSERT(range.end <= uint32_t(content.getLength())); - - const char*const text = content.begin(); - return UnownedStringSlice(text + range.start, text + range.end); - } - - return UnownedStringSlice(); -} - -UnownedStringSlice SourceFile::getLineContainingOffset(uint32_t offset) -{ - const Index lineIndex = calcLineIndexFromOffset(offset); - return getLineAtIndex(lineIndex); -} - -bool SourceFile::isOffsetOnLine(uint32_t offset, Index lineIndex) -{ - const OffsetRange range = getOffsetRangeAtLineIndex(lineIndex); - return range.isValid() && range.containsInclusive(offset); -} - -int SourceFile::calcLineIndexFromOffset(int offset) -{ - SLANG_ASSERT(UInt(offset) <= getContentSize()); - - // Make sure we have the line break offsets - const auto& lineBreakOffsets = getLineBreakOffsets(); - - // At this point we can assume the `lineBreakOffsets` array has been filled in. - // We will use a binary search to find the line index that contains our - // chosen offset. - Index lo = 0; - Index hi = lineBreakOffsets.getCount(); - - while (lo + 1 < hi) - { - const Index mid = (hi + lo) >> 1; - const uint32_t midOffset = lineBreakOffsets[mid]; - if (midOffset <= uint32_t(offset)) - { - lo = mid; - } - else - { - hi = mid; - } - } - - return int(lo); -} - -int SourceFile::calcColumnIndex(int lineIndex, int offset) -{ - const auto& lineBreakOffsets = getLineBreakOffsets(); - return offset - lineBreakOffsets[lineIndex]; -} - -/* !!!!!!!!!!!!!!!!!!!!!!!!! SourceFile !!!!!!!!!!!!!!!!!!!!!!!!!!!! */ - -void SourceFile::setContents(ISlangBlob* blob) -{ - const UInt contentSize = blob->getBufferSize(); - - SLANG_ASSERT(contentSize == m_contentSize); - - char const* contentBegin = (char const*)blob->getBufferPointer(); - char const* contentEnd = contentBegin + contentSize; - - m_contentBlob = blob; - m_content = UnownedStringSlice(contentBegin, contentEnd); -} - -void SourceFile::setContents(const String& content) -{ - ComPtr contentBlob = StringUtil::createStringBlob(content); - setContents(contentBlob); -} - -SourceFile::SourceFile(SourceManager* sourceManager, const PathInfo& pathInfo, size_t contentSize) : - m_sourceManager(sourceManager), - m_pathInfo(pathInfo), - m_contentSize(contentSize) -{ -} - -SourceFile::~SourceFile() -{ -} - -String SourceFile::calcVerbosePath() const -{ - ISlangFileSystemExt* fileSystemExt = getSourceManager()->getFileSystemExt(); - - if (fileSystemExt) - { - String canonicalPath; - ComPtr canonicalPathBlob; - if (SLANG_SUCCEEDED(fileSystemExt->getCanonicalPath(m_pathInfo.foundPath.getBuffer(), canonicalPathBlob.writeRef()))) - { - canonicalPath = StringUtil::getString(canonicalPathBlob); - } - if (canonicalPath.getLength() > 0) - { - return canonicalPath; - } - } - - return m_pathInfo.foundPath; -} - -/* !!!!!!!!!!!!!!!!!!!!!!!!! SourceManager !!!!!!!!!!!!!!!!!!!!!!!!!!!! */ - -void SourceManager::initialize( - SourceManager* p, - ISlangFileSystemExt* fileSystemExt) -{ - m_fileSystemExt = fileSystemExt; - - m_parent = p; - - if( p ) - { - // If we have a parent source manager, then we assume that all code at that level - // has already been loaded, and it is safe to start our own source locations - // right after those from the parent. - // - // TODO: more clever allocation in cases where that might not be reasonable - m_startLoc = p->m_nextLoc; - } - else - { - // Location zero is reserved for an invalid location, - // so we need to start reserving locations starting at 1. - m_startLoc = SourceLoc::fromRaw(1); - } - - m_nextLoc = m_startLoc; -} - -SourceManager::~SourceManager() -{ - for (auto item : m_sourceViews) - { - delete item; - } - - for (auto item : m_sourceFiles) - { - delete item; - } -} - -UnownedStringSlice SourceManager::allocateStringSlice(const UnownedStringSlice& slice) -{ - const UInt numChars = slice.getLength(); - - char* dst = (char*)m_memoryArena.allocate(numChars); - ::memcpy(dst, slice.begin(), numChars); - - return UnownedStringSlice(dst, numChars); -} - -SourceRange SourceManager::allocateSourceRange(UInt size) -{ - // TODO: consider using atomics here - - - SourceLoc beginLoc = m_nextLoc; - SourceLoc endLoc = beginLoc + size; - - // We need to be able to represent the location that is *at* the end of - // the input source, so the next available location for a new file - // must be placed one after the end of this one. - - m_nextLoc = endLoc + 1; - - return SourceRange(beginLoc, endLoc); -} - -SourceFile* SourceManager::createSourceFileWithSize(const PathInfo& pathInfo, size_t contentSize) -{ - SourceFile* sourceFile = new SourceFile(this, pathInfo, contentSize); - m_sourceFiles.add(sourceFile); - return sourceFile; -} - -SourceFile* SourceManager::createSourceFileWithString(const PathInfo& pathInfo, const String& contents) -{ - SourceFile* sourceFile = new SourceFile(this, pathInfo, contents.getLength()); - m_sourceFiles.add(sourceFile); - sourceFile->setContents(contents); - return sourceFile; -} - -SourceFile* SourceManager::createSourceFileWithBlob(const PathInfo& pathInfo, ISlangBlob* blob) -{ - SourceFile* sourceFile = new SourceFile(this, pathInfo, blob->getBufferSize()); - m_sourceFiles.add(sourceFile); - sourceFile->setContents(blob); - return sourceFile; -} - -SourceView* SourceManager::createSourceView(SourceFile* sourceFile, const PathInfo* pathInfo, SourceLoc initiatingSourceLoc) -{ - SourceRange range = allocateSourceRange(sourceFile->getContentSize()); - - SourceView* sourceView = nullptr; - if (pathInfo && - (pathInfo->foundPath.getLength() && sourceFile->getPathInfo().foundPath != pathInfo->foundPath)) - { - sourceView = new SourceView(sourceFile, range, &pathInfo->foundPath, initiatingSourceLoc); - } - else - { - sourceView = new SourceView(sourceFile, range, nullptr, initiatingSourceLoc); - } - - m_sourceViews.add(sourceView); - - return sourceView; -} - -SourceView* SourceManager::findSourceView(SourceLoc loc) const -{ - Index hi = m_sourceViews.getCount(); - // It must be in the range of this manager and have associated views for it to possibly be a hit - if (!getSourceRange().contains(loc) || hi == 0) - { - return nullptr; - } - - // If we don't have very many, we may as well just linearly search - if (hi <= 8) - { - for (int i = 0; i < hi; ++i) - { - SourceView* view = m_sourceViews[i]; - if (view->getRange().contains(loc)) - { - return view; - } - } - return nullptr; - } - - const SourceLoc::RawValue rawLoc = loc.getRaw(); - - // Binary chop to see if we can find the associated SourceUnit - Index lo = 0; - while (lo + 1 < hi) - { - Index mid = (hi + lo) >> 1; - - SourceView* midView = m_sourceViews[mid]; - if (midView->getRange().contains(loc)) - { - return midView; - } - - const SourceLoc::RawValue midValue = midView->getRange().begin.getRaw(); - if (midValue <= rawLoc) - { - // The location we seek is at or after this entry - lo = mid; - } - else - { - // The location we seek is before this entry - hi = mid; - } - } - - // Check if low is actually a hit - SourceView* view = m_sourceViews[lo]; - return (view->getRange().contains(loc)) ? view : nullptr; -} - -SourceView* SourceManager::findSourceViewRecursively(SourceLoc loc) const -{ - // Start with this manager - const SourceManager* manager = this; - do - { - SourceView* sourceView = manager->findSourceView(loc); - // If we found a hit we are done - if (sourceView) - { - return sourceView; - } - // Try the parent - manager = manager->m_parent; - } - while (manager); - // Didn't find it - return nullptr; -} - -SourceFile* SourceManager::findSourceFile(const String& uniqueIdentity) const -{ - SourceFile*const* filePtr = m_sourceFileMap.TryGetValue(uniqueIdentity); - return (filePtr) ? *filePtr : nullptr; -} - -SourceFile* SourceManager::findSourceFileRecursively(const String& uniqueIdentity) const -{ - const SourceManager* manager = this; - do - { - SourceFile* sourceFile = manager->findSourceFile(uniqueIdentity); - if (sourceFile) - { - return sourceFile; - } - manager = manager->m_parent; - } while (manager); - return nullptr; -} - -SourceFile* SourceManager::findSourceFileByContentRecursively(const char* text) -{ - const SourceManager* manager = this; - do - { - SourceFile* sourceFile = manager->findSourceFileByContent(text); - if (sourceFile) - { - return sourceFile; - } - manager = manager->m_parent; - } while (manager); - return nullptr; -} - -SourceFile* SourceManager::findSourceFileByContent(const char* text) const -{ - for (SourceFile* sourceFile : getSourceFiles()) - { - auto content = sourceFile->getContent(); - - if (text >= content.begin() && text <= content.end()) - { - return sourceFile; - } - } - return nullptr; -} - -void SourceManager::addSourceFile(const String& uniqueIdentity, SourceFile* sourceFile) -{ - SLANG_ASSERT(!findSourceFileRecursively(uniqueIdentity)); - m_sourceFileMap.Add(uniqueIdentity, sourceFile); -} - -HumaneSourceLoc SourceManager::getHumaneLoc(SourceLoc loc, SourceLocType type) -{ - SourceView* sourceView = findSourceViewRecursively(loc); - if (sourceView) - { - return sourceView->getHumaneLoc(loc, type); - } - else - { - return HumaneSourceLoc(); - } -} - -PathInfo SourceManager::getPathInfo(SourceLoc loc, SourceLocType type) -{ - SourceView* sourceView = findSourceViewRecursively(loc); - if (sourceView) - { - return sourceView->getPathInfo(loc, type); - } - else - { - return PathInfo::makeUnknown(); - } -} - -} // namespace Slang diff --git a/source/slang/slang-source-loc.h b/source/slang/slang-source-loc.h deleted file mode 100644 index 54811918f..000000000 --- a/source/slang/slang-source-loc.h +++ /dev/null @@ -1,487 +0,0 @@ -// slang-source-loc.h -#ifndef SLANG_SOURCE_LOC_H_INCLUDED -#define SLANG_SOURCE_LOC_H_INCLUDED - -#include "../core/slang-basic.h" -#include "../core/slang-memory-arena.h" -#include "../core/slang-string-slice-pool.h" - -#include "../../slang-com-ptr.h" -#include "../../slang.h" - -namespace Slang { - -/** Overview: - -There needs to be a mechanism where we can easily and quickly track a specific locations in any source file used during a compilation. -This is important because that original location is meaningful to the user as it relates to their original source. Thus SourceLoc are -used so we can display meaningful and accurate errors/warnings as well as being able to always map generated code locations back to their origins. - -A 'SourceLoc' along with associated structures (SourceView, SourceFile, SourceMangager) this can pinpoint the location down to the byte across the -compilation. This could be achieved by storing for every token and instruction the file, line and column number came from. The SourceLoc is used in -lots of places - every AST node, every Token from the lexer, every IRInst - so we really want to make it small. So for this reason we actually -encode SourceLoc as a single integer and then use the associated structures when needed to determine what the location actually refers to - -the source file, line and column number, or in effect the byte in the original file. - -Unfortunately there is extra complications. When a source is parsed it's interpretation (in terms of how a piece of source maps to an 'original' file etc) -can be overridden - for example by using #line directives. Moreover a single source file can be parsed multiple times. When it's parsed multiple times the -interpretation of the mapping (#line directives for example) can change. This is the purpose of the SourceView - it holds the interpretation of a source file -for a specific Lex/Parse. - -Another complication is that not all 'source' comes from SourceFiles, a macro expansion, may generate new 'source' we need to handle this, but also be able -to have a SourceLoc map to the expansion unambiguously. This is handled by creating a SourceFile and SourceView that holds only the macro generated -specific information. - -SourceFile - Is the immutable text contents of a file (or perhaps some generated source - say from doing a macro substitution) -SourceView - Tracks a single parse of a SourceFile. Each SourceView defines a range of source locations used. If a SourceFile is parsed twice, two -SourceViews are created, with unique SourceRanges. This is so that it is possible to tell which specific parse a SourceLoc is from - and so know the right -interpretation for that lex/parse. -*/ - -struct PathInfo -{ - typedef PathInfo ThisType; - - /// To be more rigorous about where a path comes from, the type identifies what a paths origin is - enum class Type : uint8_t - { - Unknown, ///< The path is not known - Normal, ///< Normal has both path and uniqueIdentity - FoundPath, ///< Just has a found path (uniqueIdentity is unknown, or even 'unknowable') - FromString, ///< Created from a string (so found path might not be defined and should not be taken as to map to a loaded file) - TokenPaste, ///< No paths, just created to do a macro expansion - TypeParse, ///< No path, just created to do a type parse - CommandLine, ///< A macro constructed from the command line - }; - - /// True if has a canonical path - SLANG_FORCE_INLINE bool hasUniqueIdentity() const { return type == Type::Normal && uniqueIdentity.getLength() > 0; } - /// True if has a regular found path - SLANG_FORCE_INLINE bool hasFoundPath() const { return type == Type::Normal || type == Type::FoundPath || (type == Type::FromString && foundPath.getLength() > 0); } - /// True if has a found path that has originated from a file (as opposed to string or some other origin) - SLANG_FORCE_INLINE bool hasFileFoundPath() const { return (type == Type::Normal || type == Type::FoundPath) && foundPath.getLength() > 0; } - - bool operator==(const ThisType& rhs) const; - bool operator!=(const ThisType& rhs) const { return !(*this == rhs); } - - /// Returns the 'most unique' identity for the path. If has a 'uniqueIdentity' returns that, else the foundPath, else "". - const String getMostUniqueIdentity() const; - - /// Append to out, how to display the path - void appendDisplayName(StringBuilder& out) const; - - // So simplify construction. In normal usage it's safer to use make methods over constructing directly. - static PathInfo makeUnknown() { return PathInfo { Type::Unknown, String(), String() }; } - static PathInfo makeTokenPaste() { return PathInfo{ Type::TokenPaste, "token paste", String()}; } - static PathInfo makeNormal(const String& foundPathIn, const String& uniqueIdentity) { SLANG_ASSERT(uniqueIdentity.getLength() > 0 && foundPathIn.getLength() > 0); return PathInfo { Type::Normal, foundPathIn, uniqueIdentity }; } - static PathInfo makePath(const String& pathIn) { SLANG_ASSERT(pathIn.getLength() > 0); return PathInfo { Type::FoundPath, pathIn, String()}; } - static PathInfo makeTypeParse() { return PathInfo { Type::TypeParse, "type string", String() }; } - static PathInfo makeCommandLine() { return PathInfo { Type::CommandLine, "command line", String() }; } - static PathInfo makeFromString(const String& userPath) { return PathInfo{ Type::FromString, userPath, String() }; } - - Type type; ///< The type of path - String foundPath; ///< The path where the file was found (might contain relative elements) - String uniqueIdentity; ///< The unique identity of the file on the path found -}; - -class SourceLoc -{ -public: - typedef SourceLoc ThisType; - typedef uint32_t RawValue; - -private: - RawValue raw; - -public: - SourceLoc() - : raw(0) - {} - - SourceLoc( - SourceLoc const& loc) - : raw(loc.raw) - {} - - SLANG_FORCE_INLINE bool operator==(const ThisType& rhs) const { return raw == rhs.raw; } - SLANG_FORCE_INLINE bool operator!=(const ThisType& rhs) const { return !(raw == rhs.raw); } - - RawValue getRaw() const { return raw; } - void setRaw(RawValue value) { raw = value; } - - static SourceLoc fromRaw(RawValue value) - { - SourceLoc result; - result.setRaw(value); - return result; - } - - bool isValid() const - { - return raw != 0; - } - SourceLoc& operator=(const ThisType& rhs) = default; -}; - -inline SourceLoc operator+(SourceLoc loc, Int offset) -{ - return SourceLoc::fromRaw(SourceLoc::RawValue(Int(loc.getRaw()) + offset)); -} - -// A range of locations in the input source -struct SourceRange -{ - /// True if the loc is in the range. Range is inclusive on begin to end. - bool contains(SourceLoc loc) const { const auto rawLoc = loc.getRaw(); return rawLoc >= begin.getRaw() && rawLoc <= end.getRaw(); } - /// Get the total size - UInt getSize() const { return UInt(end.getRaw() - begin.getRaw()); } - - /// Get the offset of a loc in this range - int getOffset(SourceLoc loc) const { SLANG_ASSERT(contains(loc)); return int(loc.getRaw() - begin.getRaw()); } - - /// Convert an offset to a loc - SourceLoc getSourceLocFromOffset(uint32_t offset) const { SLANG_ASSERT(offset <= getSize()); return begin + Int(offset); } - - SourceRange() - {} - - SourceRange(SourceLoc loc) - : begin(loc) - , end(loc) - {} - - SourceRange(SourceLoc begin, SourceLoc end) - : begin(begin) - , end(end) - {} - - SourceLoc begin; - SourceLoc end; -}; - - -// Pre-declare -struct SourceManager; - -// A logical or physical storage object for a range of input code -// that has logically contiguous source locations. -class SourceFile -{ -public: - - struct OffsetRange - { - /// We need a value to indicate an invalid range. We can't use 0 as that is valid for an offset range - /// We can't use a negative number, and don't want to make signed so we get the full 32-bits. - /// So we just use the max value as invalid - static const uint32_t kInvalid = 0xffffffff; - - /// True if the range is valid - SLANG_FORCE_INLINE bool isValid() const { return end >= start && start != kInvalid; } - /// True if offset is within range (inclusively) - SLANG_FORCE_INLINE bool containsInclusive(uint32_t offset) const { return offset >= start && offset <= end; } - - /// Get the count - SLANG_FORCE_INLINE uint32_t getCount() const { return end - start; } - - /// Return an invalid range. - static OffsetRange makeInvalid() { return OffsetRange{ kInvalid, kInvalid }; } - - uint32_t start; - uint32_t end; - }; - - /// Returns the line break offsets (in bytes from start of content) - /// Note that this is lazily evaluated - the line breaks are only calculated on the first request - const List& getLineBreakOffsets(); - - /// Returns true if the offset is on the specified line - /// NOTE! If offsets are not fully setup (because we don't have source), will only be correct for lines that have offsets - bool isOffsetOnLine(uint32_t offset, Index lineIndex); - - /// Get the line containing the offset. Requires that content is available, else will return an empty slice. - UnownedStringSlice getLineContainingOffset(uint32_t offset); - - /// Get the line at the specified line index. Requires that content is available, else will return an empty slice. - UnownedStringSlice getLineAtIndex(Index lineIndex); - - /// Get the offset range at the specified line index. Works without content. - OffsetRange getOffsetRangeAtLineIndex(Index lineIndex); - - /// Set the line break offsets - void setLineBreakOffsets(const uint32_t* offsets, UInt numOffsets); - - /// Calculate the line based on the offset - int calcLineIndexFromOffset(int offset); - - /// Calculate the offset for a line - int calcColumnIndex(int line, int offset); - - /// Get the content holding blob - ISlangBlob* getContentBlob() const { return m_contentBlob; } - - /// True if has full set content - bool hasContent() const { return m_contentBlob != nullptr; } - - /// Get the content size - size_t getContentSize() const { return m_contentSize; } - - /// Get the content - const UnownedStringSlice& getContent() const { return m_content; } - - /// Get path info - const PathInfo& getPathInfo() const { return m_pathInfo; } - - /// Set the content as a blob - void setContents(ISlangBlob* blob); - /// Set the content as a string - void setContents(const String& content); - - /// Calculate a display path -> can canonicalize if necessary - String calcVerbosePath() const; - - /// Get the source manager this was created on - SourceManager* getSourceManager() const { return m_sourceManager; } - - /// Ctor - SourceFile(SourceManager* sourceManager, const PathInfo& pathInfo, size_t contentSize); - /// Dtor - ~SourceFile(); - - protected: - - SourceManager* m_sourceManager; ///< The source manager this belongs to - PathInfo m_pathInfo; ///< The path The logical file path to report for locations inside this span. - ComPtr m_contentBlob; ///< A blob that owns the storage for the file contents. If nullptr, there is no contents - UnownedStringSlice m_content; ///< The actual contents of the file. - size_t m_contentSize; ///< The size of the actual contents - - // In order to speed up lookup of line number information, - // we will cache the starting offset of each line break in - // the input file: - List m_lineBreakOffsets; -}; - -enum class SourceLocType -{ - Nominal, ///< The normal interpretation which takes into account #line directives - Actual, ///< Ignores #line directives - and is the location as seen in the actual file -}; - -// A source location in a format a human might like to see -struct HumaneSourceLoc -{ - PathInfo pathInfo = PathInfo::makeUnknown(); - Int line = 0; - Int column = 0; -}; - - -/* A SourceView maps to a single span of SourceLoc range and is equivalent to a single include or more precisely use of a source file. -It is distinct from a SourceFile - because a SourceFile may be included multiple times, with different interpretations (depending -on #defines for example). -*/ -class SourceView -{ - public: - - // Each entry represents some contiguous span of locations that - // all map to the same logical file. - struct Entry - { - /// True if this resets the line numbering. It is distinct from a m_lineAdjust being 0, because it also means the path returns to the default. - bool isDefault() const { return m_pathHandle == StringSlicePool::Handle(0); } - - SourceLoc m_startLoc; ///< Where does this entry begin? - StringSlicePool::Handle m_pathHandle; ///< What is the presumed path for this entry. If 0 it means there is no path. - int32_t m_lineAdjust; ///< Adjustment to apply to source line numbers when printing presumed locations. Relative to the line number in the underlying file. - }; - - /// Given a sourceLoc finds the entry associated with it. If returns -1 then no entry is - /// associated with this location, and therefore the location should be interpreted as an offset - /// into the underlying sourceFile. - int findEntryIndex(SourceLoc sourceLoc) const; - - /// Add a line directive for this view. The directiveLoc must of course be in this SourceView - /// The path handle, must have been constructed on the SourceManager associated with the view - /// NOTE! Directives are assumed to be added IN ORDER during parsing such that every directiveLoc > previous - void addLineDirective(SourceLoc directiveLoc, StringSlicePool::Handle pathHandle, int line); - void addLineDirective(SourceLoc directiveLoc, const String& path, int line); - - /// Removes any corrections on line numbers and reverts to the source files path - void addDefaultLineDirective(SourceLoc directiveLoc); - - /// Get the range that this view applies to - const SourceRange& getRange() const { return m_range; } - /// Get the entries - const List& getEntries() const { return m_entries; } - /// Set the entries list - void setEntries(const Entry* entries, UInt numEntries) { m_entries.clear(); m_entries.addRange(entries, numEntries); } - - /// Get the source file holds the contents this view - SourceFile* getSourceFile() const { return m_sourceFile; } - /// Get the source manager - SourceManager* getSourceManager() const { return m_sourceFile->getSourceManager(); } - - /// Get the associated 'content' (the source text) - const UnownedStringSlice& getContent() const { return m_sourceFile->getContent(); } - - /// Get the size of the content - size_t getContentSize() const { return m_sourceFile->getContentSize(); } - - /// Get the humane location - /// Type determines if the location wanted is the original, or the 'normal' (which modifys behavior based on #line directives) - HumaneSourceLoc getHumaneLoc(SourceLoc loc, SourceLocType type = SourceLocType::Nominal); - - /// Get the path associated with a location - PathInfo getPathInfo(SourceLoc loc, SourceLocType type = SourceLocType::Nominal); - - /// Get the initiating source location - that is the source location that caused the this SourceView to be created - /// Can be SourceLoc(0) if there is no initiating location. - /// For example for a #include - the view's initiating source loc for the view that is the contents of the view - /// will be the location of the #include in the source. - /// For the original source file (ie not an include) - the view will have an initiating source loc of SourceLoc(0) - SourceLoc getInitiatingSourceLoc() const { return m_initiatingSourceLoc; } - - /// Ctor - SourceView(SourceFile* sourceFile, SourceRange range, const String* viewPath, SourceLoc initiatingSourceLoc): - m_range(range), - m_sourceFile(sourceFile), - m_initiatingSourceLoc(initiatingSourceLoc) - { - if (viewPath) - { - m_viewPath = *viewPath; - } - } - - protected: - /// Get the pathInfo from a string handle. If it's 0, it will return the _getPathInfo - PathInfo _getPathInfoFromHandle(StringSlicePool::Handle pathHandle) const; - /// Gets the pathInfo for this view. It may be different from the m_sourceFile's if the path has been - /// overridden by m_viewPath - PathInfo _getPathInfo() const; - - String m_viewPath; ///< Path to this view. If empty the path is the path to the SourceView - - SourceLoc m_initiatingSourceLoc; ///< An optional source loc that defines where this view was initiated from. SourceLoc(0) if not defined. - - SourceRange m_range; ///< The range that this SourceView applies to - SourceFile* m_sourceFile; ///< The source file. Can hold the line breaks - List m_entries; ///< An array entries describing how we should interpret a range, starting from the start location. -}; - -struct SourceManager -{ - // Initialize a source manager, with an optional parent - void initialize(SourceManager* parent, ISlangFileSystemExt* fileSystemExt); - - /// Allocate a range of SourceLoc locations, these can be used to identify a specific location in the source - SourceRange allocateSourceRange(UInt size); - - /// Create a SourceFile defined with the specified path, and content held within a blob - SourceFile* createSourceFileWithSize(const PathInfo& pathInfo, size_t contentSize); - SourceFile* createSourceFileWithString(const PathInfo& pathInfo, const String& contents); - SourceFile* createSourceFileWithBlob(const PathInfo& pathInfo, ISlangBlob* blob); - - /// Get the humane source location - HumaneSourceLoc getHumaneLoc(SourceLoc loc, SourceLocType type = SourceLocType::Nominal); - - /// Get the path associated with a location - PathInfo getPathInfo(SourceLoc loc, SourceLocType type = SourceLocType::Nominal); - - /// Create a new source view from a file - /// @param sourceFile is the source file that contains the source - /// @param pathInfo is path used to read the file from - /// @param initiatingSourceLoc the (optional) location in the source that led the the creation of this view. If there isn't an initiating source location pass SourceLoc(0)s - SourceView* createSourceView(SourceFile* sourceFile, const PathInfo* pathInfo, SourceLoc initiatingSourceLoc); - - /// Find a view by a source file location. - /// If not found in this manager will look in the parent SourceManager - /// Returns nullptr if not found. - SourceView* findSourceViewRecursively(SourceLoc loc) const; - - /// Find the SourceView associated with this manager for a specified location - /// Returns nullptr if not found. - SourceView* findSourceView(SourceLoc loc) const; - - /// Searches this manager, and then the parent to see if can find a match for path. - /// If not found returns nullptr. - SourceFile* findSourceFileRecursively(const String& uniqueIdentity) const; - /// Find if the source file is defined on this manager. - SourceFile* findSourceFile(const String& uniqueIdentity) const; - - /// Searches this manager, and then the parent to see if can find a match - SourceFile* findSourceFileByContentRecursively(const char* text); - /// Find the source file that contains *the memory* text points to. - SourceFile* findSourceFileByContent(const char* text) const; - - /// Get the file system associated with this source manager - ISlangFileSystemExt* getFileSystemExt() const { return m_fileSystemExt; } - /// Get the file system associated with this source manager - void setFileSystemExt(ISlangFileSystemExt* fileSystemExt) { m_fileSystemExt = fileSystemExt; } - - /// Add a source file, uniqueIdentity must be unique for this manager AND any parents - void addSourceFile(const String& uniqueIdentity, SourceFile* sourceFile); - - /// Get the slice pool - StringSlicePool& getStringSlicePool() { return m_slicePool; } - - /// Get the source range for just this manager - /// Caution - the range will change if allocations are made to this manager. - SourceRange getSourceRange() const { return SourceRange(m_startLoc, m_nextLoc); } - - /// Get the parent manager to this manager. Returns nullptr if there isn't any. - SourceManager* getParent() const { return m_parent; } - - /// A memory arena to hold allocations that are in scope for the same time as SourceManager - MemoryArena* getMemoryArena() { return &m_memoryArena; } - - /// Allocate a string slice - UnownedStringSlice allocateStringSlice(const UnownedStringSlice& slice); - - /// Get all of the source files - const List& getSourceFiles() const { return m_sourceFiles; } - - /// Get the source views - const List& getSourceViews() const { return m_sourceViews; } - - SourceManager() : - m_memoryArena(2048), - m_slicePool(StringSlicePool::Style::Default) - {} - ~SourceManager(); - - protected: - - // The first location available to this source manager - // (may not be the first location of all, because we might - // have a parent source manager) - SourceLoc m_startLoc; - - // The "parent" source manager that owns locations ahead of `startLoc` - SourceManager* m_parent = nullptr; - - // The location to be used by the next source file to be loaded - SourceLoc m_nextLoc; - - // All of the SourceViews constructed on this SourceManager. These are held in increasing order of range, so can find by doing a binary chop. - List m_sourceViews; - // All of the SourceFiles constructed on this SourceManager. This owns the SourceFile. - List m_sourceFiles; - - StringSlicePool m_slicePool; - - // Memory arena that can be used for holding data to held in scope as long as the Source is - // Can be used for storing the decoded contents of Token. Content for example. - MemoryArena m_memoryArena; - - // Maps uniqueIdentities to source files - Dictionary m_sourceFileMap; - - ComPtr m_fileSystemExt; -}; - -} // namespace Slang - -#endif diff --git a/source/slang/slang-token-defs.h b/source/slang/slang-token-defs.h deleted file mode 100644 index 6cece330e..000000000 --- a/source/slang/slang-token-defs.h +++ /dev/null @@ -1,96 +0,0 @@ -// slang-token-defs.h - -// This file is meant to be included multiple times, to produce different -// pieces of code related to tokens -// -// Each token is declared here with: -// -// TOKEN(id, desc) -// -// where `id` is the identifier that will be used for the token in -// ordinary code, while `desc` is name we should print when -// referring to this token in diagnostic messages. - - -#ifndef TOKEN -#error Need to define TOKEN(ID, DESC) before including "token-defs.h" -#endif - -TOKEN(Unknown, "") -TOKEN(EndOfFile, "end of file") -TOKEN(EndOfDirective, "end of line") -TOKEN(Invalid, "invalid character") -TOKEN(Identifier, "identifier") -TOKEN(IntegerLiteral, "integer literal") -TOKEN(FloatingPointLiteral, "floating-point literal") -TOKEN(StringLiteral, "string literal") -TOKEN(CharLiteral, "character literal") -TOKEN(WhiteSpace, "whitespace") -TOKEN(NewLine, "newline") -TOKEN(LineComment, "line comment") -TOKEN(BlockComment, "block comment") -TOKEN(DirectiveMessage, "user-defined message") - -#define PUNCTUATION(id, text) \ - TOKEN(id, "'" text "'") - -PUNCTUATION(Semicolon, ";") -PUNCTUATION(Comma, ",") -PUNCTUATION(Dot, ".") - -PUNCTUATION(LBrace, "{") -PUNCTUATION(RBrace, "}") -PUNCTUATION(LBracket, "[") -PUNCTUATION(RBracket, "]") -PUNCTUATION(LParent, "(") -PUNCTUATION(RParent, ")") - -PUNCTUATION(OpAssign, "=") -PUNCTUATION(OpAdd, "+") -PUNCTUATION(OpSub, "-") -PUNCTUATION(OpMul, "*") -PUNCTUATION(OpDiv, "/") -PUNCTUATION(OpMod, "%") -PUNCTUATION(OpNot, "!") -PUNCTUATION(OpBitNot, "~") -PUNCTUATION(OpLsh, "<<") -PUNCTUATION(OpRsh, ">>") -PUNCTUATION(OpEql, "==") -PUNCTUATION(OpNeq, "!=") -PUNCTUATION(OpGreater, ">") -PUNCTUATION(OpLess, "<") -PUNCTUATION(OpGeq, ">=") -PUNCTUATION(OpLeq, "<=") -PUNCTUATION(OpAnd, "&&") -PUNCTUATION(OpOr, "||") -PUNCTUATION(OpBitAnd, "&") -PUNCTUATION(OpBitOr, "|") -PUNCTUATION(OpBitXor, "^") -PUNCTUATION(OpInc, "++") -PUNCTUATION(OpDec, "--") - -PUNCTUATION(OpAddAssign, "+=") -PUNCTUATION(OpSubAssign, "-=") -PUNCTUATION(OpMulAssign, "*=") -PUNCTUATION(OpDivAssign, "/=") -PUNCTUATION(OpModAssign, "%=") -PUNCTUATION(OpShlAssign, "<<=") -PUNCTUATION(OpShrAssign, ">>=") -PUNCTUATION(OpAndAssign, "&=") -PUNCTUATION(OpOrAssign, "|=") -PUNCTUATION(OpXorAssign, "^=") - -PUNCTUATION(QuestionMark, "?") -PUNCTUATION(Colon, ":") -PUNCTUATION(RightArrow, "->") -PUNCTUATION(At, "@") -PUNCTUATION(Dollar, "$") -PUNCTUATION(Pound, "#") -PUNCTUATION(PoundPound, "##") - -PUNCTUATION(Scope, "::") - -#undef PUNCTUATION - -// Un-define the `TOKEN` macro so that client doesn't have to -#undef TOKEN diff --git a/source/slang/slang-token.cpp b/source/slang/slang-token.cpp deleted file mode 100644 index a7f6d7d62..000000000 --- a/source/slang/slang-token.cpp +++ /dev/null @@ -1,24 +0,0 @@ -// slang-token.cpp -#include "slang-token.h" - -#include - -namespace Slang { - - - - -char const* TokenTypeToString(TokenType type) -{ - switch( type ) - { - default: - SLANG_ASSERT(!"unexpected"); - return ""; - -#define TOKEN(NAME, DESC) case TokenType::NAME: return DESC; -#include "slang-token-defs.h" - } -} - -} // namespace Slang diff --git a/source/slang/slang-token.h b/source/slang/slang-token.h deleted file mode 100644 index 9697a5c2d..000000000 --- a/source/slang/slang-token.h +++ /dev/null @@ -1,140 +0,0 @@ -// slang-token.h -#ifndef SLANG_TOKEN_H_INCLUDED -#define SLANG_TOKEN_H_INCLUDED - -#include "../core/slang-basic.h" - -#include "slang-source-loc.h" -#include "slang-name.h" - -namespace Slang { - -class Name; - -enum class TokenType : uint8_t -{ -#define TOKEN(NAME, DESC) NAME, -#include "slang-token-defs.h" -}; - -char const* TokenTypeToString(TokenType type); - -typedef uint8_t TokenFlags; -struct TokenFlag -{ - enum Enum : TokenFlags - { - AtStartOfLine = 1 << 0, - AfterWhitespace = 1 << 1, - SuppressMacroExpansion = 1 << 2, - ScrubbingNeeded = 1 << 3, - Name = 1 << 4, ///< Determines if 'name' is set or 'chars' in the charsNameUnion - }; -}; - -class Token -{ -public: - - TokenType type = TokenType::Unknown; - TokenFlags flags = 0; - - SourceLoc loc; - uint32_t charsCount = 0; ///< Amount of characters. Is set if name or not. - - union CharsNameUnion - { - const char* chars; - Name* name; - }; - - CharsNameUnion charsNameUnion; - - bool hasContent() const { return charsCount > 0; } - Index getContentLength() const { return charsCount; } - - UnownedStringSlice getContent() const; - /// Set content - void setContent(const UnownedStringSlice& content); - - Name* getName() const; - - Name* getNameOrNull() const; - - SourceLoc getLoc() const { return loc; } - - /// Set the name - SLANG_FORCE_INLINE void setName(Name* inName); - - Token() - { - charsNameUnion.chars = nullptr; - } - - Token( - TokenType inType, - const UnownedStringSlice& inContent, - SourceLoc inLoc, - TokenFlags inFlags = 0) - : flags(inFlags) - { - SLANG_ASSERT((inFlags & TokenFlag::Name) == 0); - type = inType; - charsNameUnion.chars = inContent.begin(); - charsCount = uint32_t(inContent.getLength()); - loc = inLoc; - } - Token( - TokenType inType, - Name* name, - SourceLoc inLoc, - TokenFlags inFlags = 0) - { - SLANG_ASSERT(name); - type = inType; - flags = inFlags | TokenFlag::Name; - charsNameUnion.name = name; - charsCount = uint32_t(name->text.getLength()); - loc = inLoc; - } -}; - -// --------------------------------------------------------------------------- -SLANG_FORCE_INLINE UnownedStringSlice Token::getContent() const -{ - return (flags & TokenFlag::Name) ? charsNameUnion.name->text.getUnownedSlice() : UnownedStringSlice(charsNameUnion.chars, charsCount); -} - -// --------------------------------------------------------------------------- -SLANG_FORCE_INLINE Name* Token::getName() const -{ - return getNameOrNull(); -} - -// --------------------------------------------------------------------------- -SLANG_FORCE_INLINE Name* Token::getNameOrNull() const -{ - return (flags & TokenFlag::Name) ? charsNameUnion.name : nullptr; -} - -// --------------------------------------------------------------------------- -SLANG_FORCE_INLINE void Token::setContent(const UnownedStringSlice& content) -{ - flags &= ~TokenFlag::Name; - charsNameUnion.chars = content.begin(); - charsCount = uint32_t(content.getLength()); -} - -// --------------------------------------------------------------------------- -SLANG_FORCE_INLINE void Token::setName(Name* inName) -{ - SLANG_ASSERT(inName); - flags |= TokenFlag::Name; - charsNameUnion.name = inName; - charsCount = uint32_t(inName->text.getLength()); -} - - -} // namespace Slang - -#endif diff --git a/source/slang/slang.cpp b/source/slang/slang.cpp index bcc3d1c67..5c04777de 100644 --- a/source/slang/slang.cpp +++ b/source/slang/slang.cpp @@ -18,11 +18,10 @@ #include "slang-repro.h" -#include "slang-file-system.h" - +#include "../core/slang-file-system.h" #include "../core/slang-writer.h" -#include "slang-source-loc.h" +#include "../compiler-core/slang-source-loc.h" #include "slang-ast-dump.h" -- cgit v1.2.3