From eaa8dcfcc9deabb906cc09bf31fc17ab6f343ff4 Mon Sep 17 00:00:00 2001 From: Lauro Oyen <15063951+laurooyen@users.noreply.github.com> Date: Mon, 2 Dec 2024 20:46:43 +0100 Subject: Move c++ parsing code from slang-cpp-extractor to static library (#5675) * Move c++ parsing code from slang-cpp-extractor to static library * Format code * Remove relative includes --------- Co-authored-by: slangbot Co-authored-by: Yong He --- tools/CMakeLists.txt | 10 + tools/slang-cpp-extractor/cpp-extractor-main.cpp | 40 +- tools/slang-cpp-extractor/diagnostic-defs.h | 92 - tools/slang-cpp-extractor/diagnostics.cpp | 15 - tools/slang-cpp-extractor/diagnostics.h | 19 - tools/slang-cpp-extractor/file-util.cpp | 101 - tools/slang-cpp-extractor/file-util.h | 34 - tools/slang-cpp-extractor/identifier-lookup.cpp | 177 -- tools/slang-cpp-extractor/identifier-lookup.h | 124 -- tools/slang-cpp-extractor/macro-writer.cpp | 22 +- tools/slang-cpp-extractor/macro-writer.h | 14 +- tools/slang-cpp-extractor/node-tree.cpp | 176 -- tools/slang-cpp-extractor/node-tree.h | 104 - tools/slang-cpp-extractor/node.cpp | 700 ------- tools/slang-cpp-extractor/node.h | 425 ---- tools/slang-cpp-extractor/options.cpp | 153 -- tools/slang-cpp-extractor/options.h | 67 - tools/slang-cpp-extractor/parser.cpp | 2261 ---------------------- tools/slang-cpp-extractor/parser.h | 115 -- tools/slang-cpp-extractor/unit-test.cpp | 127 -- tools/slang-cpp-extractor/unit-test.h | 17 - tools/slang-cpp-parser/diagnostic-defs.h | 92 + tools/slang-cpp-parser/diagnostics.cpp | 15 + tools/slang-cpp-parser/diagnostics.h | 16 + tools/slang-cpp-parser/file-util.cpp | 101 + tools/slang-cpp-parser/file-util.h | 31 + tools/slang-cpp-parser/identifier-lookup.cpp | 177 ++ tools/slang-cpp-parser/identifier-lookup.h | 121 ++ tools/slang-cpp-parser/node-tree.cpp | 176 ++ tools/slang-cpp-parser/node-tree.h | 101 + tools/slang-cpp-parser/node.cpp | 700 +++++++ tools/slang-cpp-parser/node.h | 422 ++++ tools/slang-cpp-parser/options.cpp | 153 ++ tools/slang-cpp-parser/options.h | 63 + tools/slang-cpp-parser/parser.cpp | 2261 ++++++++++++++++++++++ tools/slang-cpp-parser/parser.h | 112 ++ tools/slang-cpp-parser/unit-test.cpp | 127 ++ tools/slang-cpp-parser/unit-test.h | 14 + 38 files changed, 4728 insertions(+), 4747 deletions(-) delete mode 100644 tools/slang-cpp-extractor/diagnostic-defs.h delete mode 100644 tools/slang-cpp-extractor/diagnostics.cpp delete mode 100644 tools/slang-cpp-extractor/diagnostics.h delete mode 100644 tools/slang-cpp-extractor/file-util.cpp delete mode 100644 tools/slang-cpp-extractor/file-util.h delete mode 100644 tools/slang-cpp-extractor/identifier-lookup.cpp delete mode 100644 tools/slang-cpp-extractor/identifier-lookup.h delete mode 100644 tools/slang-cpp-extractor/node-tree.cpp delete mode 100644 tools/slang-cpp-extractor/node-tree.h delete mode 100644 tools/slang-cpp-extractor/node.cpp delete mode 100644 tools/slang-cpp-extractor/node.h delete mode 100644 tools/slang-cpp-extractor/options.cpp delete mode 100644 tools/slang-cpp-extractor/options.h delete mode 100644 tools/slang-cpp-extractor/parser.cpp delete mode 100644 tools/slang-cpp-extractor/parser.h delete mode 100644 tools/slang-cpp-extractor/unit-test.cpp delete mode 100644 tools/slang-cpp-extractor/unit-test.h create mode 100644 tools/slang-cpp-parser/diagnostic-defs.h create mode 100644 tools/slang-cpp-parser/diagnostics.cpp create mode 100644 tools/slang-cpp-parser/diagnostics.h create mode 100644 tools/slang-cpp-parser/file-util.cpp create mode 100644 tools/slang-cpp-parser/file-util.h create mode 100644 tools/slang-cpp-parser/identifier-lookup.cpp create mode 100644 tools/slang-cpp-parser/identifier-lookup.h create mode 100644 tools/slang-cpp-parser/node-tree.cpp create mode 100644 tools/slang-cpp-parser/node-tree.h create mode 100644 tools/slang-cpp-parser/node.cpp create mode 100644 tools/slang-cpp-parser/node.h create mode 100644 tools/slang-cpp-parser/options.cpp create mode 100644 tools/slang-cpp-parser/options.h create mode 100644 tools/slang-cpp-parser/parser.cpp create mode 100644 tools/slang-cpp-parser/parser.h create mode 100644 tools/slang-cpp-parser/unit-test.cpp create mode 100644 tools/slang-cpp-parser/unit-test.h (limited to 'tools') diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 8438fe6b8..900101a39 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -49,6 +49,7 @@ generator( USE_FEWER_WARNINGS LINK_WITH_PRIVATE compiler-core + slang-cpp-parser ) generator(slang-embed) generator(slang-generate USE_FEWER_WARNINGS) @@ -68,6 +69,15 @@ generator( Threads::Threads ) +slang_add_target( + slang-cpp-parser + STATIC + USE_FEWER_WARNINGS + LINK_WITH_PRIVATE core compiler-core + INCLUDE_DIRECTORIES_PUBLIC . + FOLDER generators +) + # # Language Server # diff --git a/tools/slang-cpp-extractor/cpp-extractor-main.cpp b/tools/slang-cpp-extractor/cpp-extractor-main.cpp index a4573a8e4..bd7787162 100644 --- a/tools/slang-cpp-extractor/cpp-extractor-main.cpp +++ b/tools/slang-cpp-extractor/cpp-extractor-main.cpp @@ -1,27 +1,27 @@ // main.cpp -#include "../../source/compiler-core/slang-diagnostic-sink.h" -#include "../../source/compiler-core/slang-doc-extractor.h" -#include "../../source/compiler-core/slang-lexer.h" -#include "../../source/compiler-core/slang-name-convention-util.h" -#include "../../source/compiler-core/slang-name.h" -#include "../../source/compiler-core/slang-source-loc.h" -#include "../../source/core/slang-file-system.h" -#include "../../source/core/slang-io.h" -#include "../../source/core/slang-list.h" -#include "../../source/core/slang-secure-crt.h" -#include "../../source/core/slang-string-slice-pool.h" -#include "../../source/core/slang-string-util.h" -#include "../../source/core/slang-string.h" -#include "../../source/core/slang-writer.h" -#include "diagnostics.h" -#include "file-util.h" +#include "compiler-core/slang-diagnostic-sink.h" +#include "compiler-core/slang-doc-extractor.h" +#include "compiler-core/slang-lexer.h" +#include "compiler-core/slang-name-convention-util.h" +#include "compiler-core/slang-name.h" +#include "compiler-core/slang-source-loc.h" +#include "core/slang-file-system.h" +#include "core/slang-io.h" +#include "core/slang-list.h" +#include "core/slang-secure-crt.h" +#include "core/slang-string-slice-pool.h" +#include "core/slang-string-util.h" +#include "core/slang-string.h" +#include "core/slang-writer.h" #include "macro-writer.h" -#include "node.h" -#include "options.h" -#include "parser.h" #include "slang-com-helper.h" -#include "unit-test.h" +#include "slang-cpp-parser/diagnostics.h" +#include "slang-cpp-parser/file-util.h" +#include "slang-cpp-parser/node.h" +#include "slang-cpp-parser/options.h" +#include "slang-cpp-parser/parser.h" +#include "slang-cpp-parser/unit-test.h" #include #include diff --git a/tools/slang-cpp-extractor/diagnostic-defs.h b/tools/slang-cpp-extractor/diagnostic-defs.h deleted file mode 100644 index 03ba88dd3..000000000 --- a/tools/slang-cpp-extractor/diagnostic-defs.h +++ /dev/null @@ -1,92 +0,0 @@ -// - -// The file is meant to be included multiple times, to produce different -// pieces of declaration/definition code related to diagnostic messages -// -// Each diagnostic is declared here with: -// -// DIAGNOSTIC(id, severity, name, messageFormat) -// -// Where `id` is the unique diagnostic ID, `severity` is the default -// severity (from the `Severity` enum), `name` is a name used to refer -// to this diagnostic from code, and `messageFormat` is the default -// (non-localized) message for the diagnostic, with placeholders -// for any arguments. - -#ifndef DIAGNOSTIC -#error Need to #define DIAGNOSTIC(...) before including "slang-cpp-extractor-diagnostics-defs.h" -#define DIAGNOSTIC(id, severity, name, messageFormat) /* */ -#endif - -DIAGNOSTIC(-1, Note, seeDeclarationOf, "see declaration of '$0'") -DIAGNOSTIC(-1, Note, seeOpen, "see open $0") -DIAGNOSTIC(-1, Note, commandLine, "Command line: $0") -DIAGNOSTIC(-1, Note, previousLocation, "previous location") - -DIAGNOSTIC(1, Error, cannotOpenFile, "cannot open file '$0'.") -DIAGNOSTIC(1, Error, errorAccessingFile, "error accessing file '$0'.") - -DIAGNOSTIC(1, Error, extractorFailed, "C++ Extractor failed") -DIAGNOSTIC(1, Error, internalError, "Unknown internal error in C++ Extractor, aborted!") - -// Parsing errors -DIAGNOSTIC(100000, Error, expectingToken, "Expecting token $0") -DIAGNOSTIC(100001, Error, typeAlreadyDeclared, "Type '$0' already declared") -DIAGNOSTIC(100002, Error, scopeNotClosed, "Scope not closed") -DIAGNOSTIC(100003, Error, typeNameDoesntMatch, "Type name doesn't match $0") -DIAGNOSTIC(100004, Error, didntFindMatchingBrace, "Didn't find brace matching $0") -DIAGNOSTIC(100005, Error, braceOpenAtEndOfFile, "Brace open at file end") -DIAGNOSTIC(100006, Error, unexpectedTemplateClose, "Unexpected template close") -DIAGNOSTIC(100007, Error, superTypeNotFound, "Super type not found for $0") -DIAGNOSTIC(100008, Error, superTypeNotAType, "Named super type is not a type $0") -DIAGNOSTIC(100009, Error, unexpectedUnbalancedToken, "Unexpected unbalanced token") -DIAGNOSTIC(100010, Error, unexpectedEndOfFile, "Unexpected end of file") -DIAGNOSTIC( - 100011, - Error, - expectingTypeKeyword, - "Expecting type keyword - struct or class, found $0") - -DIAGNOSTIC( - 100012, - Error, - typeInDifferentTypeSet, - "Type $0 in different type set $1 from super class $2") -DIAGNOSTIC(100013, Error, expectingIdentifier, "Expecting an identifier, found $0") -DIAGNOSTIC(100014, Error, cannotDeclareTypeInScope, "Cannot declare types in this scope") -DIAGNOSTIC(100015, Error, identifierAlreadyDefined, "Identifier already defined '$0'") -DIAGNOSTIC(100016, Error, expectingType, "Expecting a type") -DIAGNOSTIC(100017, Error, cannotParseExpression, "Cannot parse expression") -DIAGNOSTIC(100018, Error, cannotOverload, "Cannot overload this declaration"); -DIAGNOSTIC( - 100019, - Error, - classMarkerOutsideOfClass, - "A class/struct marker is found outside of class or struct"); -DIAGNOSTIC( - 100020, - Error, - classMarkerAlreadyFound, - "A class/struct marker already found in type '$0'"); -DIAGNOSTIC(100021, Error, cannotParseType, "Cannot parse type"); -DIAGNOSTIC( - 100022, - Error, - destructorNameDoesntMatch, - "Destructor name doesn't match class name '$0'"); -DIAGNOSTIC(100023, Error, cannotParseCallable, "Cannot parse callable"); -DIAGNOSTIC( - 100024, - Error, - cannoseUseArchDependentType, - "Cannot use architecture dependent type '$0' for serializable data.") - -// Command line errors 100100 - -DIAGNOSTIC(100101, Error, optionAlreadyDefined, "Option '$0' is already defined '$1'") -DIAGNOSTIC(100102, Error, requireValueAfterOption, "Require a value after $0 option") -DIAGNOSTIC(100103, Error, unknownOption, "Unknown option '$0'") -DIAGNOSTIC(100104, Error, noInputPathsSpecified, "No input paths specified") -DIAGNOSTIC(100105, Error, noOutputPathSpecified, "No -o output path specified") - -#undef DIAGNOSTIC diff --git a/tools/slang-cpp-extractor/diagnostics.cpp b/tools/slang-cpp-extractor/diagnostics.cpp deleted file mode 100644 index 782e5e9da..000000000 --- a/tools/slang-cpp-extractor/diagnostics.cpp +++ /dev/null @@ -1,15 +0,0 @@ -#include "diagnostics.h" - -namespace CppExtract -{ - -namespace CPPDiagnostics -{ -using namespace Slang; - -#define DIAGNOSTIC(id, severity, name, messageFormat) \ - const DiagnosticInfo name = {id, Severity::severity, #name, messageFormat}; -#include "diagnostic-defs.h" -} // namespace CPPDiagnostics - -} // namespace CppExtract diff --git a/tools/slang-cpp-extractor/diagnostics.h b/tools/slang-cpp-extractor/diagnostics.h deleted file mode 100644 index cb3bd342b..000000000 --- a/tools/slang-cpp-extractor/diagnostics.h +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef CPP_EXTRACT_DIAGNOSTICS_H -#define CPP_EXTRACT_DIAGNOSTICS_H - -#include "../../source/slang/slang-diagnostics.h" - -namespace CppExtract -{ -using namespace Slang; - -namespace CPPDiagnostics -{ - -#define DIAGNOSTIC(id, severity, name, messageFormat) extern const DiagnosticInfo name; -#include "diagnostic-defs.h" - -} // namespace CPPDiagnostics -} // namespace CppExtract - -#endif diff --git a/tools/slang-cpp-extractor/file-util.cpp b/tools/slang-cpp-extractor/file-util.cpp deleted file mode 100644 index 2980a22ce..000000000 --- a/tools/slang-cpp-extractor/file-util.cpp +++ /dev/null @@ -1,101 +0,0 @@ -#include "file-util.h" - -#include "../../source/core/slang-io.h" - -namespace CppExtract -{ -using namespace Slang; - -namespace -{ // anonymous -struct DiagnosticReporter -{ - SlangResult report(SlangResult res) - { - if (SLANG_FAILED(res)) - { - if (m_sink) - { - if (res == SLANG_E_CANNOT_OPEN) - { - m_sink->diagnose(SourceLoc(), CPPDiagnostics::cannotOpenFile, m_filename); - } - else - { - m_sink->diagnose(SourceLoc(), CPPDiagnostics::errorAccessingFile, m_filename); - } - } - } - return res; - } - - DiagnosticReporter(const String& filename, DiagnosticSink* sink) - : m_filename(filename), m_sink(sink) - { - } - - DiagnosticSink* m_sink; - String m_filename; -}; - -} // namespace - -/* static */ SlangResult FileUtil::readAllText( - const Slang::String& fileName, - DiagnosticSink* sink, - String& outRead) -{ - DiagnosticReporter reporter(fileName, sink); - - RefPtr stream = new FileStream; - SLANG_RETURN_ON_FAIL(reporter.report( - stream->init(fileName, FileMode::Open, FileAccess::Read, FileShare::ReadWrite))); - - StreamReader reader; - SLANG_RETURN_ON_FAIL(reporter.report(reader.init(stream))); - SLANG_RETURN_ON_FAIL(reporter.report(reader.readToEnd(outRead))); - - return SLANG_OK; -} - -/* static */ SlangResult FileUtil::writeAllText( - const Slang::String& fileName, - DiagnosticSink* sink, - const UnownedStringSlice& text) -{ - // TODO(JS): - // There is an optimization/behavior here,that checks if the contents has changed. It only - // writes if it hasn't That might not be what you want (both because of extra work of read, the - // file modified stamp or other reasons, file is write only etc) NOTE! That this also does the - // work of the comparison after it is decoded, but the *bytes* might actually be different. - - if (File::exists(fileName)) - { - String existingText; - if (SLANG_SUCCEEDED(readAllText(fileName, nullptr, existingText))) - { - if (existingText == text) - return SLANG_OK; - } - } - - DiagnosticReporter reporter(fileName, sink); - - RefPtr stream = new FileStream; - SLANG_RETURN_ON_FAIL(reporter.report(stream->init(fileName, FileMode::Create))); - - StreamWriter writer; - SLANG_RETURN_ON_FAIL(reporter.report(writer.init(stream))); - SLANG_RETURN_ON_FAIL(reporter.report(writer.write(text))) - return SLANG_OK; -} - -/* static */ void FileUtil::indent(Index indentCount, StringBuilder& out) -{ - for (Index i = 0; i < indentCount; ++i) - { - out << CPP_EXTRACT_INDENT_STRING; - } -} - -} // namespace CppExtract diff --git a/tools/slang-cpp-extractor/file-util.h b/tools/slang-cpp-extractor/file-util.h deleted file mode 100644 index 201825973..000000000 --- a/tools/slang-cpp-extractor/file-util.h +++ /dev/null @@ -1,34 +0,0 @@ -#ifndef CPP_EXTRACT_FILE_UTIL_H -#define CPP_EXTRACT_FILE_UTIL_H - -#include "diagnostics.h" - -namespace CppExtract -{ -using namespace Slang; - -// A macro to define a single indent as a string -#define CPP_EXTRACT_INDENT_STRING " " - -struct FileUtil -{ - /// Read text into outRead. Any failures written to sink (can be passed as nullptr, for no - /// output) - static SlangResult readAllText( - const Slang::String& fileName, - DiagnosticSink* sink, - String& outRead); - /// Write text to filename. Any failures written to sink. (can be passed as nullptr, for no - /// output) - static SlangResult writeAllText( - const Slang::String& fileName, - DiagnosticSink* sink, - const UnownedStringSlice& text); - - /// Appends CPP_EXTRACT_INDENT_STRING indentCount number of times to out - static void indent(Index indentCount, StringBuilder& out); -}; - -} // namespace CppExtract - -#endif diff --git a/tools/slang-cpp-extractor/identifier-lookup.cpp b/tools/slang-cpp-extractor/identifier-lookup.cpp deleted file mode 100644 index c7255aa1f..000000000 --- a/tools/slang-cpp-extractor/identifier-lookup.cpp +++ /dev/null @@ -1,177 +0,0 @@ -#include "identifier-lookup.h" - -namespace CppExtract -{ -using namespace Slang; - -/* static */ const IdentifierFlags - IdentifierLookup::kIdentifierFlags[Index(IdentifierStyle::CountOf)] = { - 0, /// None - 0, /// Identifier - 0, /// Declare type - 0, /// Type set - IdentifierFlag::Keyword, /// TypeModifier - IdentifierFlag::Keyword, /// Keyword - - IdentifierFlag::Keyword | IdentifierFlag::StartScope | IdentifierFlag::ClassLike, /// Class - IdentifierFlag::Keyword | IdentifierFlag::StartScope | IdentifierFlag::ClassLike, /// Struct - IdentifierFlag::Keyword | IdentifierFlag::StartScope, /// Namespace - IdentifierFlag::Keyword | IdentifierFlag::StartScope, /// Enum - - IdentifierFlag::Keyword, /// Typedef - - IdentifierFlag::Keyword, /// Access - IdentifierFlag::Reflection, /// Reflected - IdentifierFlag::Reflection, /// Unreflected - - IdentifierFlag::Keyword, /// virtual - 0, /// Calling convention - IdentifierFlag::Keyword, /// template - IdentifierFlag::Keyword, /// static - - IdentifierFlag::Keyword, /// unsigned/signed - - IdentifierFlag::Keyword, /// extern - - 0, /// Callable misc - 0, /// IntegerType int, short, char, long - - IdentifierFlag::Keyword, /// default -}; - -void IdentifierLookup::set(const UnownedStringSlice& name, IdentifierStyle style) -{ - StringSlicePool::Handle handle; - if (m_pool.findOrAdd(name, handle)) - { - // Add the extra flags - m_styles[Index(handle)] = style; - } - else - { - Index index = Index(handle); - SLANG_ASSERT(index == m_styles.getCount()); - m_styles.add(style); - } -} - -void IdentifierLookup::set(const char* const* names, size_t namesCount, IdentifierStyle style) -{ - for (size_t i = 0; i < namesCount; ++i) - { - set(UnownedStringSlice(names[i]), style); - } -} - -void IdentifierLookup::set(const Pair* pairs, Index pairsCount) -{ - for (Index i = 0; i < pairsCount; ++i) - { - const auto& pair = pairs[i]; - set(UnownedStringSlice(pair.name), pair.style); - } -} - -void IdentifierLookup::initDefault(const UnownedStringSlice& markPrefix) -{ - reset(); - - // Some keywords - { - const char* names[] = { - "continue", - "if", - "case", - "break", - "catch", - "delete", - "do", - "else", - "for", - "new", - "goto", - "return", - "switch", - "throw", - "using", - "while", - "operator", - "explicit"}; - set(names, SLANG_COUNT_OF(names), IdentifierStyle::Keyword); - } - - // Type modifier keywords - { - const char* names[] = {"const", "volatile"}; - set(names, SLANG_COUNT_OF(names), IdentifierStyle::TypeModifier); - } - - // Special markers - { - const char* names[] = {"PRE_DECLARE", "TYPE_SET", "REFLECTED", "UNREFLECTED"}; - const IdentifierStyle styles[] = { - IdentifierStyle::PreDeclare, - IdentifierStyle::TypeSet, - IdentifierStyle::Reflected, - IdentifierStyle::Unreflected}; - SLANG_COMPILE_TIME_ASSERT(SLANG_COUNT_OF(names) == SLANG_COUNT_OF(styles)); - - StringBuilder buf; - for (Index i = 0; i < SLANG_COUNT_OF(names); ++i) - { - buf.clear(); - buf << markPrefix << names[i]; - set(buf.getUnownedSlice(), styles[i]); - } - } - - { - set("virtual", IdentifierStyle::Virtual); - - set("template", IdentifierStyle::Template); - set("static", IdentifierStyle::Static); - set("extern", IdentifierStyle::Extern); - set("default", IdentifierStyle::Default); - } - - { - const char* names[] = {"char", "short", "int", "long"}; - set(names, SLANG_COUNT_OF(names), IdentifierStyle::IntegerType); - } - - { - const char* names[] = {"SLANG_MCALL"}; - set(names, SLANG_COUNT_OF(names), IdentifierStyle::CallingConvention); - } - - { - const char* names[] = {"SLANG_NO_THROW", "inline"}; - set(names, SLANG_COUNT_OF(names), IdentifierStyle::CallableMisc); - } - - // Keywords which introduce types/scopes - { - const Pair pairs[] = { - {"struct", IdentifierStyle::Struct}, - {"class", IdentifierStyle::Class}, - {"namespace", IdentifierStyle::Namespace}, - {"enum", IdentifierStyle::Enum}, - {"typedef", IdentifierStyle::TypeDef}, - }; - - set(pairs, SLANG_COUNT_OF(pairs)); - } - - // Keywords that control access - { - const char* names[] = {"private", "protected", "public"}; - set(names, SLANG_COUNT_OF(names), IdentifierStyle::Access); - } - { - const char* names[] = {"signed", "unsigned"}; - - set(names, SLANG_COUNT_OF(names), IdentifierStyle::IntegerModifier); - } -} - -} // namespace CppExtract diff --git a/tools/slang-cpp-extractor/identifier-lookup.h b/tools/slang-cpp-extractor/identifier-lookup.h deleted file mode 100644 index f26220f1f..000000000 --- a/tools/slang-cpp-extractor/identifier-lookup.h +++ /dev/null @@ -1,124 +0,0 @@ -#ifndef CPP_EXTRACT_IDENTIFIER_LOOKUP_H -#define CPP_EXTRACT_IDENTIFIER_LOOKUP_H - -#include "diagnostics.h" - -namespace CppExtract -{ -using namespace Slang; - -enum class IdentifierStyle -{ - None, ///< It's not an identifier - - Identifier, ///< Just an identifier - - PreDeclare, ///< Declare a type (not visible in C++ code) - TypeSet, ///< TypeSet - - TypeModifier, ///< const, volatile etc - Keyword, ///< A keyword C/C++ keyword that is not another type - - Class, ///< class - Struct, ///< struct - Namespace, ///< namespace - Enum, ///< enum - - TypeDef, ///< typedef - - Access, ///< public, protected, private - - Reflected, - Unreflected, - - CallingConvention, ///< Used on a method - Virtual, ///< - - Template, - - Static, - - IntegerModifier, - - Extern, - - CallableMisc, ///< For SLANG_NO_THROW etc - - IntegerType, ///< Built in integer type - - Default, /// default - - CountOf, -}; - -typedef uint32_t IdentifierFlags; -struct IdentifierFlag -{ - enum Enum : IdentifierFlags - { - StartScope = 0x1, ///< namespace, struct or class - ClassLike = 0x2, ///< Struct or class - Keyword = 0x4, - Reflection = 0x8, - }; -}; - - -class IdentifierLookup -{ -public: - struct Pair - { - const char* name; - IdentifierStyle style; - }; - - IdentifierStyle get(const UnownedStringSlice& slice) const - { - Index index = m_pool.findIndex(slice); - return (index >= 0) ? m_styles[index] : IdentifierStyle::None; - } - - void set(const char* name, IdentifierStyle style) { set(UnownedStringSlice(name), style); } - - void set(const UnownedStringSlice& name, IdentifierStyle style); - - void set(const char* const* names, size_t namesCount, IdentifierStyle style); - - void set(const Pair* pairs, Index pairsCount); - - void reset() - { - m_styles.clear(); - m_pool.clear(); - } - - void initDefault(const UnownedStringSlice& markPrefix); - - IdentifierLookup() - : m_pool(StringSlicePool::Style::Empty) - { - SLANG_ASSERT(m_pool.getSlicesCount() == 0); - } - - static const IdentifierFlags kIdentifierFlags[Index(IdentifierStyle::CountOf)]; - -protected: - List m_styles; - StringSlicePool m_pool; -}; - - -SLANG_FORCE_INLINE IdentifierFlags getFlags(IdentifierStyle style) -{ - return IdentifierLookup::kIdentifierFlags[Index(style)]; -} - -SLANG_FORCE_INLINE bool hasFlag(IdentifierStyle style, IdentifierFlag::Enum flag) -{ - return (getFlags(style) & flag) != 0; -} - -} // namespace CppExtract - -#endif diff --git a/tools/slang-cpp-extractor/macro-writer.cpp b/tools/slang-cpp-extractor/macro-writer.cpp index 3f0e0a5fa..324a48efd 100644 --- a/tools/slang-cpp-extractor/macro-writer.cpp +++ b/tools/slang-cpp-extractor/macro-writer.cpp @@ -1,22 +1,20 @@ #include "macro-writer.h" -#include "../../source/core/slang-list.h" -#include "../../source/core/slang-string.h" +#include "compiler-core/slang-diagnostic-sink.h" +#include "core/slang-io.h" +#include "core/slang-list.h" +#include "core/slang-string.h" +#include "core/slang-writer.h" #include "slang-com-helper.h" -// #include "../../source/core/slang-string-util.h" -#include "../../source/compiler-core/slang-diagnostic-sink.h" -#include "../../source/core/slang-io.h" -#include "../../source/core/slang-writer.h" -// #include "../../source/compiler-core/slang-name.h" - -#include "diagnostics.h" -#include "file-util.h" -#include "node-tree.h" -#include "options.h" +#include "slang-cpp-parser/diagnostics.h" +#include "slang-cpp-parser/file-util.h" +#include "slang-cpp-parser/node-tree.h" +#include "slang-cpp-parser/options.h" namespace CppExtract { using namespace Slang; +using namespace CppParse; SLANG_FORCE_INLINE static void _indent(Index indentCount, StringBuilder& out) { diff --git a/tools/slang-cpp-extractor/macro-writer.h b/tools/slang-cpp-extractor/macro-writer.h index 0cbf85fb3..aa23c96ef 100644 --- a/tools/slang-cpp-extractor/macro-writer.h +++ b/tools/slang-cpp-extractor/macro-writer.h @@ -1,14 +1,14 @@ -#ifndef CPP_EXTRACT_MACRO_WRITER_H -#define CPP_EXTRACT_MACRO_WRITER_H +#pragma once -#include "../../source/compiler-core/slang-diagnostic-sink.h" -#include "diagnostics.h" -#include "node-tree.h" -#include "options.h" +#include "compiler-core/slang-diagnostic-sink.h" +#include "slang-cpp-parser/diagnostics.h" +#include "slang-cpp-parser/node-tree.h" +#include "slang-cpp-parser/options.h" namespace CppExtract { using namespace Slang; +using namespace CppParse; /* A class that writes out macros that define type hierarchies, as well as fields of types */ class MacroWriter @@ -39,5 +39,3 @@ protected: }; } // namespace CppExtract - -#endif diff --git a/tools/slang-cpp-extractor/node-tree.cpp b/tools/slang-cpp-extractor/node-tree.cpp deleted file mode 100644 index 6b543ad31..000000000 --- a/tools/slang-cpp-extractor/node-tree.cpp +++ /dev/null @@ -1,176 +0,0 @@ -#include "node-tree.h" - -#include "../../source/compiler-core/slang-name-convention-util.h" -#include "../../source/core/slang-io.h" -#include "identifier-lookup.h" -#include "options.h" - -namespace CppExtract -{ -using namespace Slang; - -/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!! NodeTree !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ - -NodeTree::NodeTree( - StringSlicePool* typePool, - NamePool* namePool, - IdentifierLookup* identifierLookup) - : m_typePool(typePool) - , m_namePool(namePool) - , m_identifierLookup(identifierLookup) - , m_typeSetPool(StringSlicePool::Style::Empty) -{ - m_rootNode = new ScopeNode(Node::Kind::Namespace); - m_rootNode->m_reflectionType = ReflectionType::Reflected; -} - -TypeSet* NodeTree::getTypeSet(const UnownedStringSlice& slice) -{ - Index index = m_typeSetPool.findIndex(slice); - if (index < 0) - { - return nullptr; - } - return m_typeSets[index]; -} - -TypeSet* NodeTree::getOrAddTypeSet(const UnownedStringSlice& slice) -{ - const Index index = Index(m_typeSetPool.add(slice)); - if (index >= m_typeSets.getCount()) - { - SLANG_ASSERT(m_typeSets.getCount() == index); - TypeSet* typeSet = new TypeSet; - - m_typeSets.add(typeSet); - typeSet->m_macroName = m_typeSetPool.getSlice(StringSlicePool::Handle(index)); - return typeSet; - } - else - { - return m_typeSets[index]; - } -} - -SourceOrigin* NodeTree::addSourceOrigin(SourceFile* sourceFile, const Options& options) -{ - // Calculate from the path, a 'macro origin' name. - const String macroOrigin = calcMacroOrigin(sourceFile->getPathInfo().foundPath, options); - - SourceOrigin* origin = new SourceOrigin(sourceFile, macroOrigin); - m_sourceOrigins.add(origin); - return origin; -} - -/* static */ String NodeTree::calcMacroOrigin(const String& filePath, const Options& options) -{ - // Get the filename without extension - String fileName = Path::getFileNameWithoutExt(filePath); - - // We can work on just the slice - UnownedStringSlice slice = fileName.getUnownedSlice(); - - // Filename prefix - if (options.m_stripFilePrefix.getLength() && - slice.startsWith(options.m_stripFilePrefix.getUnownedSlice())) - { - const Index len = options.m_stripFilePrefix.getLength(); - slice = UnownedStringSlice(slice.begin() + len, slice.end()); - } - - // Trim - - slice = slice.trim('-'); - - StringBuilder out; - NameConventionUtil::convert(slice, NameConvention::UpperSnake, out); - return out; -} - -SlangResult NodeTree::_calcDerivedTypesRec(ScopeNode* inScopeNode, DiagnosticSink* sink) -{ - if (inScopeNode->isClassLike()) - { - ClassLikeNode* classLikeNode = static_cast(inScopeNode); - - if (classLikeNode->m_super.hasContent()) - { - ScopeNode* parentScope = classLikeNode->m_parentScope; - if (parentScope == nullptr) - { - sink->diagnoseRaw( - Severity::Error, - UnownedStringSlice::fromLiteral("Can't lookup in scope if there is none!")); - return SLANG_FAIL; - } - - Node* superNode = Node::lookup(parentScope, classLikeNode->m_super.getContent()); - - if (!superNode) - { - if (classLikeNode->isReflected()) - { - sink->diagnose( - classLikeNode->m_name, - CPPDiagnostics::superTypeNotFound, - classLikeNode->getAbsoluteName()); - return SLANG_FAIL; - } - } - else - { - ClassLikeNode* superType = as(superNode); - - if (!superType) - { - sink->diagnose( - classLikeNode->m_name, - CPPDiagnostics::superTypeNotAType, - classLikeNode->getAbsoluteName()); - return SLANG_FAIL; - } - - if (superType->m_typeSet != classLikeNode->m_typeSet) - { - sink->diagnose( - classLikeNode->m_name, - CPPDiagnostics::typeInDifferentTypeSet, - classLikeNode->m_name.getContent(), - classLikeNode->m_typeSet->m_macroName, - superType->m_typeSet->m_macroName); - return SLANG_FAIL; - } - - // The base class must be defined in same scope (as we didn't allow different scopes - // for base classes) - superType->addDerived(classLikeNode); - } - } - else - { - // Add to it's own typeset - if (classLikeNode->isReflected() && classLikeNode->m_typeSet) - { - classLikeNode->m_typeSet->m_baseTypes.add(classLikeNode); - } - } - } - - for (Node* child : inScopeNode->m_children) - { - ScopeNode* childScope = as(child); - if (childScope) - { - SLANG_RETURN_ON_FAIL(_calcDerivedTypesRec(childScope, sink)); - } - } - - return SLANG_OK; -} - -SlangResult NodeTree::calcDerivedTypes(DiagnosticSink* sink) -{ - return _calcDerivedTypesRec(m_rootNode, sink); -} - - -} // namespace CppExtract diff --git a/tools/slang-cpp-extractor/node-tree.h b/tools/slang-cpp-extractor/node-tree.h deleted file mode 100644 index f1547dbc1..000000000 --- a/tools/slang-cpp-extractor/node-tree.h +++ /dev/null @@ -1,104 +0,0 @@ -#ifndef CPP_EXTRACT_NODE_TREE_H -#define CPP_EXTRACT_NODE_TREE_H - -#include "../../source/compiler-core/slang-lexer.h" -#include "diagnostics.h" -#include "identifier-lookup.h" -#include "node.h" - -namespace CppExtract -{ -using namespace Slang; - -class TypeSet : public RefObject -{ -public: - /// This is the looked up name. - UnownedStringSlice - m_macroName; ///< The name extracted from the macro SLANG_ABSTRACT_AST_CLASS -> AST - - String m_typeName; ///< The enum type name associated with this type for AST it is ASTNode - String m_fileMark; ///< This 'mark' becomes of the output filename - - List m_baseTypes; ///< The base types for this type set -}; - -class SourceOrigin : public RefObject -{ -public: - void addNode(Node* node) - { - if (auto classLike = as(node)) - { - SLANG_ASSERT(classLike->m_origin == nullptr); - classLike->m_origin = this; - } - - m_nodes.add(node); - } - - SourceOrigin(SourceFile* sourceFile, const String& macroOrigin) - : m_sourceFile(sourceFile), m_macroOrigin(macroOrigin) - { - } - - String m_macroOrigin; ///< The macro text is inserted into the macro to identify the origin. It - ///< is based on the filename - SourceFile* m_sourceFile; ///< The source file - also holds the path information - - /// All of the nodes defined in this file in the order they were defined - /// Note that the same namespace may be listed multiple times. - List> m_nodes; -}; - -struct Options; -class IdentifierLookup; - -/* NodeTree holds nodes that have been parsed into a tree rooted on the 'rootNode'. -Also contains other state associated with or useful to a node tree */ -class NodeTree -{ -public: - friend class Parser; - /// Get all of the parsed source origins - const List>& getSourceOrigins() const { return m_sourceOrigins; } - - TypeSet* getTypeSet(const UnownedStringSlice& slice); - TypeSet* getOrAddTypeSet(const UnownedStringSlice& slice); - - SourceOrigin* addSourceOrigin(SourceFile* sourceFile, const Options& options); - - /// Get all of the type sets - const List>& getTypeSets() const { return m_typeSets; } - - /// Get the root node - Node* getRootNode() const { return m_rootNode; } - - /// When parsing we don't lookup all up super types/add derived types. This is because - /// we allow files to be processed in any order, so we have to do the type lookup as a separate - /// operation - SlangResult calcDerivedTypes(DiagnosticSink* sink); - - NodeTree(StringSlicePool* typePool, NamePool* namePool, IdentifierLookup* identifierLookup); - - static String calcMacroOrigin(const String& filePath, const Options& options); - -protected: - SlangResult _calcDerivedTypesRec(ScopeNode* node, DiagnosticSink* sink); - - StringSlicePool m_typeSetPool; ///< Pool for type set names - List> m_typeSets; ///< The type sets - - IdentifierLookup* m_identifierLookup; - StringSlicePool* m_typePool; ///< Pool for just types - - NamePool* m_namePool; - - RefPtr m_rootNode; ///< The root scope - - List> m_sourceOrigins; -}; - -} // namespace CppExtract - -#endif diff --git a/tools/slang-cpp-extractor/node.cpp b/tools/slang-cpp-extractor/node.cpp deleted file mode 100644 index 4598e34c9..000000000 --- a/tools/slang-cpp-extractor/node.cpp +++ /dev/null @@ -1,700 +0,0 @@ -#include "node.h" - -#include "../../source/core/slang-string-escape-util.h" -#include "../../source/core/slang-string-util.h" -#include "file-util.h" - -namespace CppExtract -{ - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Node Impl -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -SLANG_FORCE_INLINE static void _indent(Index indentCount, StringBuilder& out) -{ - FileUtil::indent(indentCount, out); -} - -void Node::dumpMarkup(int indentCount, StringBuilder& out) -{ - if (m_markup.getLength() <= 0) - { - return; - } - - List lines; - StringUtil::calcLines(m_markup.getUnownedSlice(), lines); - - // Remove empty lines from the end - while (lines.getCount()) - { - auto lastLine = lines.getLast(); - if (lastLine.trim().getLength() == 0) - { - lines.removeLast(); - continue; - } - break; - } - - if (lines.getCount() == 0) - { - return; - } - - for (auto line : lines) - { - _indent(indentCount, out); - out << "// " << line << "\n"; - } -} - -ScopeNode* Node::getRootScope() -{ - if (m_parentScope) - { - ScopeNode* scope = m_parentScope; - while (scope->m_parentScope) - { - scope = scope->m_parentScope; - } - return scope; - } - else - { - return as(this); - } -} - -void Node::calcScopeDepthFirst(List& outNodes) -{ - outNodes.add(this); -} - -void Node::calcAbsoluteName(StringBuilder& outName) const -{ - List path; - calcScopePath(const_cast(this), path); - - // 1 so we skip the global scope - for (Index i = 1; i < path.getCount(); ++i) - { - Node* node = path[i]; - - if (i > 1) - { - outName << "::"; - } - - if (node->m_kind == Kind::AnonymousNamespace) - { - outName << "{Anonymous}"; - } - else - { - outName << node->m_name.getContent(); - } - } -} - -/* static */ void Node::calcScopePath(Node* node, List& outPath) -{ - outPath.clear(); - - while (node) - { - outPath.add(node); - node = node->m_parentScope; - } - - // reverse the order, so we go from root to the node - outPath.reverse(); -} - -/* static */ void Node::filterImpl(Filter inFilter, List& ioNodes) -{ - // Filter out all the unreflected nodes - Index count = ioNodes.getCount(); - for (Index j = 0; j < count;) - { - Node* node = ioNodes[j]; - - if (!inFilter(node)) - { - ioNodes.removeAt(j); - count--; - } - else - { - j++; - } - } -} - -/* static */ Node* Node::lookupNameInScope(ScopeNode* scope, const UnownedStringSlice& name) -{ - // TODO(JS): Doesn't handle 'using namespace'. - - // Must be unqualified name - SLANG_ASSERT(name.indexOf(UnownedStringSlice::fromLiteral("::")) < 0); - - Node* childNode = scope->findChild(name); - if (childNode) - { - return childNode; - } - - // If we have an anonymous namespace in this scope, try looking up in there.. - if (scope->m_anonymousNamespace) - { - Node* childNode = scope->m_anonymousNamespace->findChild(name); - if (childNode) - { - return childNode; - } - } - - // I could have an enum (that's not an enum class) - for (Node* node : scope->m_children) - { - EnumNode* enumNode = as(node); - if (enumNode && enumNode->m_kind == Node::Kind::Enum) - { - Node** nodePtr = enumNode->m_childMap.tryGetValue(name); - if (nodePtr) - { - return *nodePtr; - } - } - } - - return nullptr; -} - -/* static */ Node* Node::lookupFromScope( - ScopeNode* scope, - const UnownedStringSlice* parts, - Index partsCount) -{ - SLANG_ASSERT(partsCount > 0); - if (partsCount == 1) - { - return lookupNameInScope(scope, parts[0]); - } - - for (Index i = 0; i < partsCount; ++i) - { - const UnownedStringSlice& part = parts[i]; - - Node* node = lookupNameInScope(scope, part); - if (node == nullptr) - { - return node; - } - // If at end, then we are done - if (i == partsCount - 1) - { - return node; - } - - // If there are more elements, then node must be some kind of scope, - // if we are going to find it - scope = as(node); - if (scope == nullptr) - { - break; - } - } - - return nullptr; -} - -/* static */ void Node::splitPath( - const UnownedStringSlice& inPath, - List& outParts) -{ - if (inPath.indexOf(UnownedStringSlice::fromLiteral("::")) >= 0) - { - StringUtil::split(inPath, UnownedStringSlice::fromLiteral("::"), outParts); - // Remove any whitespace - for (auto& part : outParts) - { - part = part.trim(); - } - } - else - { - outParts.clear(); - outParts.add(inPath.trim()); - } -} - -/* static */ Node* Node::lookupFromScope(ScopeNode* scope, const UnownedStringSlice& inPath) -{ - if (inPath.indexOf(UnownedStringSlice::fromLiteral("::")) >= 0) - { - List parts; - splitPath(inPath, parts); - - return lookupFromScope(scope, parts.getBuffer(), parts.getCount()); - } - else - { - return lookupNameInScope(scope, inPath); - } -} - -/* static */ Node* Node::lookup(ScopeNode* scope, const UnownedStringSlice& inPath) -{ - if (inPath.indexOf(UnownedStringSlice::fromLiteral("::")) >= 0) - { - List parts; - splitPath(inPath, parts); - - if (parts[0].getLength() == 0) - { - // It's a lookup from global scope - ScopeNode* rootScope = scope->getRootScope(); - return lookupFromScope(rootScope, parts.getBuffer() + 1, parts.getCount() + 1); - } - - // Okay lets try a lookup from each scope up to the global scope - while (scope) - { - Node* node = lookupFromScope(scope, parts.getBuffer(), parts.getCount()); - if (node) - { - return node; - } - - scope = scope->m_parentScope; - } - } - else - { - while (scope) - { - // Lookup in this scope - Node* node = lookupNameInScope(scope, inPath); - if (node) - { - return node; - } - - // Try parent scope - scope = scope->m_parentScope; - } - } - - return nullptr; -} - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ScopeNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -ScopeNode* ScopeNode::getAnonymousNamespace() -{ - if (!m_anonymousNamespace) - { - m_anonymousNamespace = new ScopeNode(Kind::AnonymousNamespace); - m_anonymousNamespace->m_parentScope = this; - m_children.add(m_anonymousNamespace); - } - - return m_anonymousNamespace; -} - -void ScopeNode::addChildIgnoringName(Node* child) -{ - SLANG_ASSERT(child->m_parentScope == nullptr); - // Can't add anonymous namespace this way - should be added via getAnonymousNamespace - SLANG_ASSERT(child->m_kind != Kind::AnonymousNamespace); - - child->m_parentScope = this; - m_children.add(child); -} - -void ScopeNode::addChild(Node* child) -{ - addChildIgnoringName(child); - - if (child->m_name.hasContent()) - { - m_childMap.add(child->m_name.getContent(), child); - } -} - -Node* ScopeNode::findChild(const UnownedStringSlice& name) const -{ - Node* const* nodePtr = m_childMap.tryGetValue(name); - if (nodePtr) - { - return *nodePtr; - } - return nullptr; -} - -void ScopeNode::calcScopeDepthFirst(List& outNodes) -{ - outNodes.add(this); - for (Node* child : m_children) - { - child->calcScopeDepthFirst(outNodes); - } -} - -void ScopeNode::dump(int indentCount, StringBuilder& out) -{ - dumpMarkup(indentCount, out); - - _indent(indentCount, out); - - switch (m_kind) - { - case Kind::AnonymousNamespace: - { - out << "namespace {\n"; - } - case Kind::Namespace: - { - if (m_name.hasContent()) - { - out << "namespace " << m_name.getContent() << " {\n"; - } - else - { - out << "{\n"; - } - break; - } - } - - for (Node* child : m_children) - { - child->dump(indentCount + 1, out); - } - - _indent(indentCount, out); - out << "}\n"; -} - -/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! EnumCaseNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ - -/* Returns true if needs space between the tokens. -It determines this based on the locs, and if they contain something between them. -*/ -static bool _needsSpace(const Token& prevTok, const Token& tok) -{ - auto prevLoc = prevTok.getLoc(); - auto loc = tok.getLoc(); - - auto prevContent = prevTok.getContent(); - - if (prevLoc + prevContent.getLength() == loc) - { - return false; - } - - return true; -} - - -static void _dumpTokens(const Token* toks, Index count, StringBuilder& out) -{ - if (count > 0) - { - out << toks[0].getContent(); - - for (Index i = 1; i < count; ++i) - { - const auto& prevToken = toks[i - 1]; - const auto& token = toks[i]; - - if (_needsSpace(prevToken, token)) - { - out << " "; - } - - out << token.getContent(); - } - } -} - -static void _dumpTokens(const List& toks, StringBuilder& out) -{ - _dumpTokens(toks.getBuffer(), toks.getCount(), out); -} - - -void EnumCaseNode::dump(int indent, StringBuilder& out) -{ - if (isReflected()) - { - dumpMarkup(indent, out); - - _indent(indent, out); - out << m_name.getContent(); - - if (m_valueTokens.getCount()) - { - out << " = "; - _dumpTokens(m_valueTokens, out); - } - - out << ",\n"; - } -} - -/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! EnumNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ - -void TypeDefNode::dump(int indent, StringBuilder& out) -{ - if (isReflected()) - { - dumpMarkup(indent, out); - - _indent(indent, out); - - out << "typedef "; - _dumpTokens(m_targetTypeTokens, out); - out << " " << m_name.getContent() << ";\n"; - } -} - -/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! EnumNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ - -void EnumNode::dump(int indent, StringBuilder& out) -{ - if (!isReflected()) - { - return; - } - - dumpMarkup(indent, out); - - _indent(indent, out); - - out << "enum "; - - if (m_kind == Kind::EnumClass) - { - out << "class "; - } - - if (m_name.type != TokenType::Invalid) - { - out << m_name.getContent(); - } - - if (m_backingTokens.getCount() > 0) - { - out << " : "; - _dumpTokens(m_backingTokens, out); - } - - out << "\n"; - _indent(indent, out); - out << "{\n"; - - for (Node* child : m_children) - { - child->dump(indent + 1, out); - } - - _indent(indent, out); - out << "}\n"; -} - -/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!! CallableNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ - -void CallableNode::dump(int indent, StringBuilder& out) -{ - if (!isReflected()) - { - return; - } - - dumpMarkup(indent, out); - - _indent(indent, out); - - if (m_isStatic) - { - out << "static "; - } - if (m_isVirtual) - { - out << "virtual "; - } - - out << m_returnType << " "; - out << m_name.getContent() << "("; - - const Index count = m_params.getCount(); - for (Index i = 0; i < count; ++i) - { - if (i > 0) - { - out << ", "; - } - - const auto& param = m_params[i]; - out << param.m_type; - if (param.m_name.type == TokenType::Identifier) - { - out << " " << param.m_name.getContent(); - } - } - - out << ")"; - - if (m_isPure) - { - out << " = 0"; - } - - out << "\n"; -} - -/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! FieldNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ - -void FieldNode::dump(int indent, StringBuilder& out) -{ - if (!isReflected()) - { - return; - } - - dumpMarkup(indent, out); - - _indent(indent, out); - - if (m_isStatic) - { - out << "static "; - } - - out << m_fieldType << " " << m_name.getContent() << "\n"; -} - -/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ClassLikeNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ - -/// Add a node that is derived from this -void ClassLikeNode::addDerived(ClassLikeNode* derived) -{ - SLANG_ASSERT(derived->m_superNode == nullptr); - derived->m_superNode = this; - m_derivedTypes.add(derived); -} - -void ClassLikeNode::calcDerivedDepthFirst(List& outNodes) -{ - outNodes.add(this); - for (ClassLikeNode* derivedType : m_derivedTypes) - { - derivedType->calcDerivedDepthFirst(outNodes); - } -} - -void ClassLikeNode::dumpDerived(int indentCount, StringBuilder& out) -{ - if (isClassLike() && isReflected() && m_name.hasContent()) - { - _indent(indentCount, out); - out << m_name.getContent() << "\n"; - } - - for (ClassLikeNode* derivedType : m_derivedTypes) - { - derivedType->dumpDerived(indentCount + 1, out); - } -} - -Index ClassLikeNode::calcDerivedDepth() const -{ - const ClassLikeNode* node = this; - Index count = 0; - - while (node) - { - count++; - node = node->m_superNode; - } - - return count; -} - -ClassLikeNode* ClassLikeNode::findLastDerived() -{ - for (Index i = m_derivedTypes.getCount() - 1; i >= 0; --i) - { - ClassLikeNode* derivedType = m_derivedTypes[i]; - ClassLikeNode* found = derivedType->findLastDerived(); - if (found) - { - return found; - } - } - return this; -} - -bool ClassLikeNode::hasReflectedDerivedType() const -{ - for (ClassLikeNode* type : m_derivedTypes) - { - if (type->isReflected()) - { - return true; - } - } - return false; -} - -void ClassLikeNode::getReflectedDerivedTypes(List& out) const -{ - out.clear(); - for (ClassLikeNode* type : m_derivedTypes) - { - if (type->isReflected()) - { - out.add(type); - } - } -} - -void ClassLikeNode::dump(int indentCount, StringBuilder& out) -{ - dumpMarkup(indentCount, out); - - _indent(indentCount, out); - - const char* typeName = (m_kind == Kind::StructType) ? "struct" : "class"; - - out << typeName << " "; - - if (!isReflected()) - { - out << " ("; - } - out << m_name.getContent(); - if (!isReflected()) - { - out << ") "; - } - - if (m_super.hasContent()) - { - out << " : " << m_super.getContent(); - } - - out << " {\n"; - - for (Node* child : m_children) - { - child->dump(indentCount + 1, out); - } - - _indent(indentCount, out); - out << "}\n"; -} - -} // namespace CppExtract diff --git a/tools/slang-cpp-extractor/node.h b/tools/slang-cpp-extractor/node.h deleted file mode 100644 index 2c15e460d..000000000 --- a/tools/slang-cpp-extractor/node.h +++ /dev/null @@ -1,425 +0,0 @@ -#ifndef CPP_EXTRACT_NODE_H -#define CPP_EXTRACT_NODE_H - -#include "../../source/compiler-core/slang-doc-extractor.h" -#include "diagnostics.h" - -namespace CppExtract -{ -using namespace Slang; - -enum class ReflectionType : uint8_t -{ - NotReflected, - Reflected, -}; - -// Pre-declare -class TypeSet; -class SourceOrigin; - -struct ScopeNode; - -class Node : public RefObject -{ -public: - enum class Kind : uint8_t - { - Invalid, - - StructType, - ClassType, - - Enum, - EnumClass, - - Namespace, - AnonymousNamespace, - - Field, - EnumCase, - - TypeDef, - - Callable, ///< Functions/methods - - Other, ///< Used 'other' parsing like for TYPE - Unknown, ///< Used for marking tokens consumed but usage is not known - - CountOf, - }; - - enum class KindRange - { - ScopeStart = int(Kind::StructType), - ScopeEnd = int(Kind::AnonymousNamespace), - - ClassLikeStart = int(Kind::StructType), - ClassLikeEnd = int(Kind::ClassType), - - ScopeTypeStart = int(Kind::StructType), - ScopeTypeEnd = int(Kind::EnumClass), - - OtherTypeStart = int(Kind::TypeDef), - OtherTypeEnd = int(Kind::TypeDef), - - EnumStart = int(Kind::Enum), - EnumEnd = int(Kind::EnumClass), - }; - - /// Returns true if kind can cast to this type - /// Used for implementing as casting - static bool isOfKind(Kind kind) - { - SLANG_UNUSED(kind); - return true; - } - - static bool isKindScope(Kind kind) - { - return int(kind) >= int(KindRange::ScopeStart) && int(kind) <= int(KindRange::ScopeEnd); - } - static bool isKindClassLike(Kind kind) - { - return int(kind) >= int(KindRange::ClassLikeStart) && - int(kind) <= int(KindRange::ClassLikeEnd); - } - static bool isKindEnumLike(Kind kind) - { - return int(kind) >= int(KindRange::EnumStart) && int(kind) <= int(KindRange::EnumEnd); - } - - /// It a type, but doesn't have a scope - static bool isKindOtherType(Kind kind) - { - return int(kind) >= int(KindRange::OtherTypeStart) && - int(kind) <= int(KindRange::OtherTypeEnd); - } - /// Is a type and has a scope - static bool isKindScopeType(Kind kind) - { - return int(kind) >= int(KindRange::ScopeTypeStart) && - int(kind) <= int(KindRange::ScopeTypeEnd); - } - - /// True if the kind is any type - static bool isKindType(Kind kind) { return isKindOtherType(kind) || isKindScopeType(kind); } - - /// True if the kind can accept contained types - static bool canKindContainTypes(Kind type) - { - switch (type) - { - case Kind::StructType: - case Kind::ClassType: - case Kind::Namespace: - case Kind::AnonymousNamespace: - { - return true; - } - default: - break; - } - return false; - } - - bool isNamespace() const { return m_kind == Kind::Namespace; } - bool isClassLike() const { return isKindClassLike(m_kind); } - bool isScope() const { return isKindScope(m_kind); } - bool isEnumLike() const { return isKindEnumLike(m_kind); } - - /// These are useful for the filter - static bool isClassLikeAndReflected(Node* node) - { - return node->isClassLike() && node->isReflected(); - } - static bool isClassLike(Node* node) { return isKindClassLike(node->m_kind); } - - virtual void dump(int indent, StringBuilder& out) = 0; - - /// Do depth first traversal of nodes in scopes - virtual void calcScopeDepthFirst(List& outNodes); - - /// Calculate the absolute name for this namespace/type - void calcAbsoluteName(StringBuilder& outName) const; - - /// Get the absolute name - String getAbsoluteName() const - { - StringBuilder buf; - calcAbsoluteName(buf); - return buf.produceString(); - } - - /// Calculate the scope path to this node, from the root - void calcScopePath(List& outPath) { calcScopePath(this, outPath); } - - /// True if reflected - bool isReflected() const { return m_reflectionType == ReflectionType::Reflected; } - - SourceLoc getSourceLoc() const { return m_name.getLoc(); } - - ScopeNode* getRootScope(); - - typedef bool (*Filter)(Node* node); - - template - static void filter(Filter filter, List& io) - { - const Node* _isNodeDerived = (T*)nullptr; - SLANG_UNUSED(_isNodeDerived); - filterImpl(filter, reinterpret_cast&>(io)); - } - - static void filterImpl(Filter filter, List& io); - - static void calcScopePath(Node* node, List& outPath); - - /// Lookup a name in just the specified scope - /// Handles anonymous namespaces, or name lookups that are in the parents space - static Node* lookupNameInScope(ScopeNode* scope, const UnownedStringSlice& name); - - /// Lookup from a path - static Node* lookupFromScope(ScopeNode* scope, const UnownedStringSlice* path, Index pathCount); - /// Looks up *just* from the specified scope. - static Node* lookupFromScope(ScopeNode* scope, const UnownedStringSlice& slice); - - /// Look up name (which can contain ::) - static Node* lookup(ScopeNode* scope, const UnownedStringSlice& name); - - static void splitPath(const UnownedStringSlice& slice, List& outSplitPath); - - /// If markup is specified dump it - void dumpMarkup(int indent, StringBuilder& out); - - Node(Kind type) - : m_kind(type), m_parentScope(nullptr), m_reflectionType(ReflectionType::NotReflected) - { - } - - Kind m_kind; ///< The kind of node this is - ReflectionType m_reflectionType; ///< Classes can be traversed, but not reflected. To be - ///< reflected they have to contain the marker - - MarkupVisibility m_markupVisibility = - MarkupVisibility::Public; ///< The visibility of the markup - String m_markup; ///< Documentation associated with this node - - Token m_name; ///< The name of this scope/type - - ScopeNode* m_parentScope; ///< The scope this type/scope is defined in -}; - -struct ScopeNode : public Node -{ - typedef Node Super; - - static bool isOfKind(Kind kind) { return isKindScope(kind); } - - virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; - virtual void calcScopeDepthFirst(List& outNodes) SLANG_OVERRIDE; - - /// True if can contain callable entries - bool canContainCallable() const { return isClassLike() || isNamespace(); } - - /// True if can accept fields (class like types can) - bool canContainFields() const { return isClassLike(); } - - /// True if the scope can accept types - bool canContainTypes() const { return canKindContainTypes(m_kind); } - - /// Gets the reflection for any contained types - ReflectionType getContainedReflectionType() const - { - return m_reflectionType == ReflectionType::NotReflected ? ReflectionType::NotReflected - : m_reflectionOverride; - } - - /// Add a child node to this nodes scope - void addChild(Node* child); - /// Adds the child but does not add the name to the map - void addChildIgnoringName(Node* child); - - /// Find a child node in this scope with the specified name. Return nullptr if not found - Node* findChild(const UnownedStringSlice& name) const; - - /// Gets the anonymous namespace associated with this scope - ScopeNode* getAnonymousNamespace(); - - ScopeNode(Kind kind) - : Super(kind) - , m_reflectionOverride(ReflectionType::Reflected) - , m_anonymousNamespace(nullptr) - { - } - - /// For child types, fields, how reflection is handled. If this type is not reflected - ReflectionType m_reflectionOverride; - - /// All of the types and namespaces in this *scope* - List> m_children; - - /// Map from a name (in this scope) to the Node - Dictionary m_childMap; - - /// There can only be one anonymousNamespace for a scope. If there is one it's held here - ScopeNode* m_anonymousNamespace; -}; - -struct FieldNode : public Node -{ - typedef Node Super; - - static bool isOfKind(Kind kind) { return kind == Kind::Field; } - - virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; - - FieldNode() - : Super(Kind::Field) - { - } - - UnownedStringSlice m_fieldType; - - bool m_isStatic = false; - - /// TODO(JS): We may want to add initializer tokens -}; - -struct ClassLikeNode : public ScopeNode -{ - typedef ScopeNode Super; - - static bool isOfKind(Kind kind) { return isKindClassLike(kind); } - - /// Add a node that is derived from this - void addDerived(ClassLikeNode* derived); - - /// Dump all of the derived types - void dumpDerived(int indentCount, StringBuilder& out); - - /// Calculates the derived depth - Index calcDerivedDepth() const; - - /// Find the last (reflected) derived type - ClassLikeNode* findLastDerived(); - - /// Traverse the hierarchy of derived nodes, in depth first order - void calcDerivedDepthFirst(List& outNodes); - - /// True if has a derived type that is reflected - bool hasReflectedDerivedType() const; - - /// Stores in out any reflected derived types - void getReflectedDerivedTypes(List& out) const; - - // Node Impl - virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; - - ClassLikeNode(Kind kind) - : Super(kind), m_origin(nullptr), m_typeSet(nullptr), m_superNode(nullptr) - { - SLANG_ASSERT(kind == Kind::ClassType || kind == Kind::StructType); - } - - SourceOrigin* m_origin; ///< Defines where this was uniquely defined. - - Token m_marker; ///< The marker associated with this scope (typically the marker is SLANG_CLASS - ///< etc, that is used to identify reflectedType) - - List> m_derivedTypes; ///< All of the types derived from this type - - TypeSet* m_typeSet; ///< The typeset this type belongs to. - - Token m_super; ///< Super class name - ClassLikeNode* m_superNode; ///< If this is a class/struct, the type it is derived from (or - ///< nullptr if base) -}; - -struct CallableNode : public Node -{ - typedef Node Super; - - static bool isOfKind(Kind kind) { return kind == Kind::Callable; } - - virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; - - CallableNode() - : Super(Kind::Callable) - { - } - - struct Param - { - UnownedStringSlice m_type; - Token m_name; - }; - - UnownedStringSlice m_returnType; - - CallableNode* m_nextOverload = nullptr; - - List m_params; - - bool m_isStatic = false; - bool m_isVirtual = false; - bool m_isPure = false; -}; - -struct EnumCaseNode : public Node -{ - typedef Node Super; - - static bool isOfKind(Kind kind) { return kind == Kind::EnumCase; } - - virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; - - EnumCaseNode() - : Super(Kind::EnumCase) - { - } - - // Tokens that make up the value. If not defined will be empty - List m_valueTokens; -}; - -struct EnumNode : public ScopeNode -{ - typedef ScopeNode Super; - static bool isOfKind(Kind kind) { return isKindEnumLike(kind); } - - virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; - - EnumNode(Kind kind) - : Super(kind) - { - SLANG_ASSERT(isKindEnumLike(kind)); - } - - List m_backingTokens; -}; - -struct TypeDefNode : public Node -{ - typedef Node Super; - static bool isOfKind(Kind kind) { return kind == Kind::TypeDef; } - - virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; - - TypeDefNode() - : Super(Kind::TypeDef) - { - } - - List m_targetTypeTokens; -}; - -template -T* as(Node* node) -{ - return (node && T::isOfKind(node->m_kind)) ? static_cast(node) : nullptr; -} - -} // namespace CppExtract - -#endif diff --git a/tools/slang-cpp-extractor/options.cpp b/tools/slang-cpp-extractor/options.cpp deleted file mode 100644 index 17ff5eebc..000000000 --- a/tools/slang-cpp-extractor/options.cpp +++ /dev/null @@ -1,153 +0,0 @@ -#include "options.h" - -#include "diagnostics.h" - -namespace CppExtract -{ - -SlangResult OptionsParser::_parseArgFlag(const char* option, bool& outFlag) -{ - SLANG_ASSERT(UnownedStringSlice(m_args[m_index]) == option); - SLANG_ASSERT(m_index < m_argCount); - - m_index++; - outFlag = true; - return SLANG_OK; -} - -SlangResult OptionsParser::_parseArgWithValue(const char* option, String& ioValue) -{ - SLANG_ASSERT(UnownedStringSlice(m_args[m_index]) == option); - if (m_index + 1 < m_argCount) - { - // Next parameter is the output path, there can only be one - if (ioValue.getLength()) - { - // There already is output - m_sink->diagnose(SourceLoc(), CPPDiagnostics::optionAlreadyDefined, option, ioValue); - return SLANG_FAIL; - } - } - else - { - m_sink->diagnose(SourceLoc(), CPPDiagnostics::requireValueAfterOption, option); - return SLANG_FAIL; - } - - ioValue = m_args[m_index + 1]; - m_index += 2; - return SLANG_OK; -} - -SlangResult OptionsParser::_parseArgReplaceValue(const char* option, String& ioValue) -{ - SLANG_ASSERT(UnownedStringSlice(m_args[m_index]) == option); - if (m_index + 1 >= m_argCount) - { - m_sink->diagnose(SourceLoc(), CPPDiagnostics::requireValueAfterOption, option); - return SLANG_FAIL; - } - - ioValue = m_args[m_index + 1]; - m_index += 2; - return SLANG_OK; -} - -SlangResult OptionsParser::parse( - int argc, - const char* const* argv, - DiagnosticSink* sink, - Options& outOptions) -{ - outOptions.reset(); - - m_index = 0; - m_argCount = argc; - m_args = argv; - m_sink = sink; - - outOptions.reset(); - - while (m_index < m_argCount) - { - const UnownedStringSlice arg = UnownedStringSlice(argv[m_index]); - - if (arg.getLength() > 0 && arg[0] == '-') - { - if (arg == "-d") - { - SLANG_RETURN_ON_FAIL(_parseArgWithValue("-d", outOptions.m_inputDirectory)); - continue; - } - else if (arg == "-o") - { - SLANG_RETURN_ON_FAIL(_parseArgWithValue("-o", outOptions.m_outputPath)); - continue; - } - else if (arg == "-dump") - { - SLANG_RETURN_ON_FAIL(_parseArgFlag("-dump", outOptions.m_dump)); - continue; - } - else if (arg == "-mark-prefix") - { - SLANG_RETURN_ON_FAIL( - _parseArgReplaceValue("-mark-prefix", outOptions.m_markPrefix)); - continue; - } - else if (arg == "-mark-suffix") - { - SLANG_RETURN_ON_FAIL( - _parseArgReplaceValue("-mark-suffix", outOptions.m_markSuffix)); - continue; - } - else if (arg == "-defs") - { - SLANG_RETURN_ON_FAIL(_parseArgFlag("-defs", outOptions.m_defs)); - continue; - } - else if (arg == "-output-fields") - { - SLANG_RETURN_ON_FAIL(_parseArgFlag("-output-fields", outOptions.m_outputFields)); - continue; - } - else if (arg == "-strip-prefix") - { - SLANG_RETURN_ON_FAIL( - _parseArgWithValue("-strip-prefix", outOptions.m_stripFilePrefix)); - continue; - } - else if (arg == "-unit-test") - { - SLANG_RETURN_ON_FAIL(_parseArgFlag("-unit-test", outOptions.m_runUnitTests)); - continue; - } - else if (arg == "-unmarked") - { - bool unmarked; - SLANG_RETURN_ON_FAIL(_parseArgFlag("-unmarked", unmarked)); - outOptions.m_requireMark = !unmarked; - continue; - } - - m_sink->diagnose(SourceLoc(), CPPDiagnostics::unknownOption, arg); - return SLANG_FAIL; - } - else - { - // If it doesn't start with - then it's assumed to be an input path - outOptions.m_inputPaths.add(arg); - m_index++; - } - } - - if (outOptions.m_inputPaths.getCount() < 0) - { - m_sink->diagnose(SourceLoc(), CPPDiagnostics::noInputPathsSpecified); - return SLANG_FAIL; - } - - return SLANG_OK; -} - -} // namespace CppExtract diff --git a/tools/slang-cpp-extractor/options.h b/tools/slang-cpp-extractor/options.h deleted file mode 100644 index 8231d5e3b..000000000 --- a/tools/slang-cpp-extractor/options.h +++ /dev/null @@ -1,67 +0,0 @@ -#ifndef CPP_EXTRACT_OPTIONS_H -#define CPP_EXTRACT_OPTIONS_H - -#include "../../source/slang/slang-diagnostics.h" - -namespace CppExtract -{ -using namespace Slang; - - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Options !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -struct Options -{ - void reset() { *this = Options(); } - - Options() - { - m_markPrefix = "SLANG_"; - m_markSuffix = "_CLASS"; - } - - bool m_defs = false; ///< If set will output a '-defs.h' file for each of the input files, that - ///< corresponds to previous defs files (although doesn't have fields/RAW) - bool m_dump = - false; ///< If true will dump to stderr the types/fields and hierarchy it extracted - bool m_runUnitTests = false; ///< If true will run internal unit tests - bool m_extractDoc = true; ///< If set will try to extract documentation associated with nodes - - bool m_outputFields = false; ///< When dumping macros also dump field definitions - bool m_requireMark = true; - - List m_inputPaths; ///< The input paths to the files to be processed - - String m_outputPath; ///< The output path. Note that the extractor can generate multiple output - ///< files, and this will actually be the 'stem' of several files - - String m_inputDirectory; ///< The input directory that is by default used for reading - ///< m_inputPaths from. - String m_markPrefix; ///< The prefix of the 'marker' used to identify a reflected type - String m_markSuffix; ///< The postfix of the 'marker' used to identify a reflected type - String m_stripFilePrefix; ///< Used for the 'origin' information, this is stripped from the - ///< source filename, and the remainder of the filename (without - ///< extension) is 'macroized' -}; - -struct OptionsParser -{ - /// Parse the parameters. NOTE! Must have the program path removed - SlangResult parse(int argc, const char* const* argv, DiagnosticSink* sink, Options& outOptions); - - SlangResult _parseArgWithValue(const char* option, String& outValue); - SlangResult _parseArgReplaceValue(const char* option, String& outValue); - SlangResult _parseArgFlag(const char* option, bool& outFlag); - - String m_reflectType; - - Index m_index; - Int m_argCount; - const char* const* m_args; - DiagnosticSink* m_sink; -}; - - -} // namespace CppExtract - -#endif diff --git a/tools/slang-cpp-extractor/parser.cpp b/tools/slang-cpp-extractor/parser.cpp deleted file mode 100644 index 3a4891829..000000000 --- a/tools/slang-cpp-extractor/parser.cpp +++ /dev/null @@ -1,2261 +0,0 @@ -#include "parser.h" - -#include "../../source/compiler-core/slang-name-convention-util.h" -#include "../../source/core/slang-io.h" -#include "../../source/core/slang-string-util.h" -#include "identifier-lookup.h" -#include "options.h" - -namespace CppExtract -{ -using namespace Slang; - -// If fails then we need more bits to identify types -SLANG_COMPILE_TIME_ASSERT(int(Node::Kind::CountOf) <= 8 * sizeof(uint32_t)); - -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Parser !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -Parser::Parser(NodeTree* nodeTree, DiagnosticSink* sink) - : m_sink(sink), m_nodeTree(nodeTree), m_nodeTypeEnabled(0) -{ - // Enable types by default - const Node::Kind defaultEnabled[] = { - Node::Kind::ClassType, - Node::Kind::StructType, - Node::Kind::Namespace, - Node::Kind::AnonymousNamespace, - Node::Kind::Field, - - // These are disabled by default because AST uses macro magic to build up the types - // Node::Type::TypeDef, - // Node::Type::Enum, - // Node::Type::EnumClass, - - Node::Kind::Callable, - }; - setKindsEnabled(defaultEnabled, SLANG_COUNT_OF(defaultEnabled)); -} - -void Parser::setKindEnabled(Node::Kind kind, bool isEnabled) -{ - if (isEnabled) - { - m_nodeTypeEnabled |= (NodeTypeBitType(1) << int(kind)); - } - else - { - m_nodeTypeEnabled &= ~(NodeTypeBitType(1) << int(kind)); - } -} - -void Parser::setKindsEnabled(const Node::Kind* kinds, Index kindsCount, bool isEnabled) -{ - for (Index i = 0; i < kindsCount; ++i) - { - setKindEnabled(kinds[i], isEnabled); - } -} - -bool Parser::_isMarker(const UnownedStringSlice& name) -{ - return name.startsWith(m_options->m_markPrefix.getUnownedSlice()) && - name.endsWith(m_options->m_markSuffix.getUnownedSlice()); -} - -SlangResult Parser::expect(TokenType type, Token* outToken) -{ - if (m_reader.peekTokenType() != type) - { - m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::expectingToken, type); - return SLANG_FAIL; - } - - if (outToken) - { - *outToken = m_reader.advanceToken(); - } - else - { - m_reader.advanceToken(); - } - return SLANG_OK; -} - -bool Parser::advanceIfToken(TokenType type, Token* outToken) -{ - if (m_reader.peekTokenType() == type) - { - Token token = m_reader.advanceToken(); - if (outToken) - { - *outToken = token; - } - return true; - } - return false; -} - -bool Parser::advanceIfMarker(Token* outToken) -{ - const Token peekToken = m_reader.peekToken(); - if (peekToken.type == TokenType::Identifier && _isMarker(peekToken.getContent())) - { - m_reader.advanceToken(); - if (outToken) - { - *outToken = peekToken; - } - return true; - } - return false; -} - -bool Parser::advanceIfStyle(IdentifierStyle style, Token* outToken) -{ - if (m_reader.peekTokenType() == TokenType::Identifier) - { - IdentifierStyle readStyle = - m_nodeTree->m_identifierLookup->get(m_reader.peekToken().getContent()); - if (readStyle == style) - { - Token token = m_reader.advanceToken(); - if (outToken) - { - *outToken = token; - } - return true; - } - } - return false; -} - - -SlangResult Parser::pushAnonymousNamespace() -{ - m_currentScope = m_currentScope->getAnonymousNamespace(); - - if (m_sourceOrigin) - { - m_sourceOrigin->addNode(m_currentScope); - } - - // Add the to the scope stack so can pop. - m_scopeStack.add(m_currentScope); - - return SLANG_OK; -} - -SlangResult Parser::pushScope(ScopeNode* scopeNode) -{ - // We can only have one 'special' scope. - SLANG_ASSERT(scopeNode || m_scopeStack.getLast()); - - // We keep to track. - m_scopeStack.add(scopeNode); - - // If we pass nullptr, we don't update the current scope. - if (scopeNode == nullptr) - { - return SLANG_OK; - } - - if (m_sourceOrigin) - { - m_sourceOrigin->addNode(scopeNode); - } - - if (scopeNode->m_name.hasContent()) - { - // For anonymous namespace, we should look if we already have one and just reopen that. - // Doing so will mean will find anonymous namespace clashes - - if (Node* foundNode = m_currentScope->findChild(scopeNode->m_name.getContent())) - { - if (scopeNode->isClassLike()) - { - m_sink->diagnose( - m_reader.peekToken(), - CPPDiagnostics::typeAlreadyDeclared, - scopeNode->m_name.getContent()); - m_sink->diagnose( - foundNode->m_name, - CPPDiagnostics::seeDeclarationOf, - scopeNode->m_name.getContent()); - return SLANG_FAIL; - } - - if (foundNode->m_kind == Node::Kind::Namespace) - { - if (foundNode->m_kind != scopeNode->m_kind) - { - // Different types can't work - m_sink->diagnose( - m_reader.peekToken(), - CPPDiagnostics::typeAlreadyDeclared, - scopeNode->m_name.getContent()); - return SLANG_FAIL; - } - - ScopeNode* foundScopeNode = as(foundNode); - SLANG_ASSERT(foundScopeNode); - - // Make sure the node is empty, as we are *not* going to add it, we are just going - // to use the pre-existing namespace - SLANG_ASSERT(scopeNode->m_children.getCount() == 0); - - // We can just use the pre-existing namespace - m_currentScope = foundScopeNode; - return SLANG_OK; - } - } - } - - m_currentScope->addChild(scopeNode); - m_currentScope = scopeNode; - return SLANG_OK; -} - -SlangResult Parser::popScope() -{ - if (m_scopeStack.getCount() <= 0) - { - m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::scopeNotClosed); - return SLANG_FAIL; - } - - ScopeNode* topScope = m_scopeStack.getLast(); - m_scopeStack.removeLast(); - - // If the top is nullptr, we don't change the current scope - if (topScope == nullptr) - { - return SLANG_OK; - } - - m_currentScope = m_currentScope->m_parentScope; - return SLANG_OK; -} - -SlangResult Parser::_maybeConsumeScope() -{ - // Look for either ; or { to open scope - while (true) - { - const TokenType type = m_reader.peekTokenType(); - if (type == TokenType::Semicolon) - { - m_reader.advanceToken(); - return SLANG_OK; - } - else if (type == TokenType::LBrace) - { - // m_reader.advanceToken(); - return consumeToClosingBrace(); - } - else if (type == TokenType::EndOfFile) - { - return SLANG_OK; - } - - m_reader.advanceToken(); - } -} - -SlangResult Parser::consumeToClosingBrace(const Token* inOpenBraceToken) -{ - Token openToken; - if (inOpenBraceToken) - { - openToken = *inOpenBraceToken; - } - else - { - openToken = m_reader.advanceToken(); - } - SLANG_ASSERT(openToken.type == TokenType::LBrace); - - while (true) - { - switch (m_reader.peekTokenType()) - { - case TokenType::EndOfFile: - { - m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::didntFindMatchingBrace); - m_sink->diagnose(openToken, CPPDiagnostics::seeOpen); - return SLANG_FAIL; - } - case TokenType::LBrace: - { - SLANG_RETURN_ON_FAIL(consumeToClosingBrace()); - break; - } - case TokenType::RBrace: - { - m_reader.advanceToken(); - return SLANG_OK; - } - default: - { - m_reader.advanceToken(); - break; - } - } - } -} - - -SlangResult Parser::_parseEnum() -{ - // We are looking for - // enum ([class name] | [name]) [: base] ( { | ; ) - - Token enumToken; - - // consume enum - SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &enumToken)); - - if (!m_currentScope->canContainTypes()) - { - m_sink->diagnose(enumToken.loc, CPPDiagnostics::cannotDeclareTypeInScope); - return SLANG_FAIL; - } - - Node::Kind kind = Node::Kind::Enum; - - Token nameToken; - if (advanceIfToken(TokenType::Identifier, &nameToken)) - { - const IdentifierStyle style = m_nodeTree->m_identifierLookup->get(nameToken.getContent()); - - if (style == IdentifierStyle::Class) - { - kind = Node::Kind::EnumClass; - SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &nameToken)); - } - else if (style == IdentifierStyle::None) - { - // It holds the name then - } - else - { - m_sink->diagnose( - nameToken.loc, - CPPDiagnostics::expectingIdentifier, - nameToken.getContent()); - return SLANG_FAIL; - } - } - - RefPtr node = new EnumNode(kind); - node->m_name = nameToken; - node->m_reflectionType = m_currentScope->getContainedReflectionType(); - - if (advanceIfToken(TokenType::Colon)) - { - // We may have tokens up to { or ; - List backingTokens; - - while (true) - { - TokenType tokenType = m_reader.peekTokenType(); - if (tokenType == TokenType::Semicolon || tokenType == TokenType::LBrace || - tokenType == TokenType::EndOfFile) - { - break; - } - - backingTokens.add(m_reader.advanceToken()); - } - - // TODO - Look up the backing type. It can only be an integral. We can assume it must be - // defined before lookup for our uses here. If we can't find the type, we could assume it's - // size is undefined - - if (backingTokens.getCount() > 0) - { - node->m_backingTokens.swapWith(backingTokens); - } - } - - pushScope(node); - - if (advanceIfToken(TokenType::Semicolon)) - { - if (nameToken.type != TokenType::Invalid) - { - Node* node = m_currentScope->findChild(nameToken.getContent()); - if (node) - { - // Strictly speaking we should check the backing type etc, match, but for now ignore - // and assume it's ok - - if (node->m_kind == kind) - { - return SLANG_OK; - } - m_sink->diagnose( - nameToken.loc, - CPPDiagnostics::typeAlreadyDeclared, - nameToken.getContent()); - return SLANG_FAIL; - } - return popScope(); - } - } - - SLANG_RETURN_ON_FAIL(expect(TokenType::LBrace)); - - while (true) - { - TokenType tokenType = m_reader.peekTokenType(); - if (tokenType == TokenType::RBrace) - { - break; - } - - RefPtr caseNode(new EnumCaseNode); - - // We could also check if the name is a valid identifier for name, for now just assume. - SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &caseNode->m_name)); - - if (node->findChild(caseNode->m_name.getContent())) - { - m_sink->diagnose( - caseNode->m_name.loc, - CPPDiagnostics::identifierAlreadyDefined, - caseNode->m_name.getContent()); - return SLANG_FAIL; - } - - caseNode->m_reflectionType = m_currentScope->getContainedReflectionType(); - - // Add the value - node->addChild(caseNode); - - if (advanceIfToken(TokenType::OpAssign)) - { - List valueTokens; - SLANG_RETURN_ON_FAIL(_parseExpression(valueTokens)); - - if (valueTokens.getCount() > 0) - { - caseNode->m_valueTokens.swapWith(valueTokens); - } - } - - tokenType = m_reader.peekTokenType(); - if (tokenType == TokenType::Comma) - { - m_reader.advanceToken(); - continue; - } - - break; - } - - SLANG_RETURN_ON_FAIL(expect(TokenType::RBrace)); - SLANG_RETURN_ON_FAIL(expect(TokenType::Semicolon)); - - return popScope(); -} - -SlangResult Parser::_consumeTemplate() -{ - // Skip the current 'template' token. - m_reader.advanceToken(); - - // Consume everything in <> - SLANG_RETURN_ON_FAIL(expect(TokenType::OpLess)); - - { - Index arrowCount = 1; - while (true) - { - auto tokenType = m_reader.peekTokenType(); - - if (tokenType == TokenType::OpLess) - { - m_reader.advanceToken(); - arrowCount++; - } - else if (tokenType == TokenType::OpGreater) - { - m_reader.advanceToken(); - if (arrowCount == 1) - { - break; - } - --arrowCount; - } - else if (tokenType == TokenType::OpRsh) - { - if (arrowCount < 2) - { - m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::unexpectedTemplateClose); - return SLANG_FAIL; - } - m_reader.advanceToken(); - if (arrowCount == 2) - { - break; - } - arrowCount -= 2; - } - else if (tokenType == TokenType::EndOfFile) - { - m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::unexpectedEndOfFile); - return SLANG_FAIL; - } - else - { - m_reader.advanceToken(); - } - } - } - - // Search for { or ; to consume remaining - while (true) - { - auto tokenType = m_reader.peekTokenType(); - - switch (tokenType) - { - case TokenType::EndOfFile: - { - m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::unexpectedEndOfFile); - return SLANG_FAIL; - } - case TokenType::Semicolon: - { - // Ends with semicolon if it's a template pre-declaration - m_reader.advanceToken(); - return SLANG_OK; - } - case TokenType::LBrace: - { - // If ends with {, means could be body of a struct/class or a body of a - // function/method. Consume it - SLANG_RETURN_ON_FAIL(consumeToClosingBrace()); - // If we hit a ; just consume and ignore - advanceIfToken(TokenType::Semicolon); - return SLANG_OK; - } - default: - { - // Consume - m_reader.advanceToken(); - break; - } - } - } -} - -SlangResult Parser::_maybeParseNode(Node::Kind kind) -{ - // We are looking for - // struct/class identifier [: [public|private|protected] Identifier ] { - // [public|private|proctected:]* marker ( identifier ); - - if (kind == Node::Kind::Namespace) - { - // consume namespace - SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier)); - - Token name; - if (advanceIfToken(TokenType::LBrace)) - { - return pushAnonymousNamespace(); - } - else if (advanceIfToken(TokenType::Identifier, &name)) - { - if (advanceIfToken(TokenType::LBrace)) - { - // Okay looks like we are opening a namespace - RefPtr node(new ScopeNode(Node::Kind::Namespace)); - node->m_name = name; - - node->m_reflectionType = m_currentScope->getContainedReflectionType(); - // Push the node - return pushScope(node); - } - } - - // Just ignore it then - return SLANG_OK; - } - else if (Node::isKindEnumLike(kind)) - { - return _parseEnum(); - } - - // Must be class | struct - - SLANG_ASSERT(kind == Node::Kind::ClassType || kind == Node::Kind::StructType); - - Token name; - - // consume class | struct - SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier)); - // Next is the class name - SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &name)); - - - if (m_reader.peekTokenType() == TokenType::Semicolon) - { - // pre declaration; - return SLANG_OK; - } - - RefPtr node(new ClassLikeNode(kind)); - node->m_name = name; - - // We default to the containing scope for reflection type. - if (!m_options->m_requireMark) - { - node->m_reflectionType = m_currentScope->getContainedReflectionType(); - } - else - { - // Defaults to not reflected - SLANG_ASSERT(!node->isReflected()); - } - - if (advanceIfToken(TokenType::Colon)) - { - // Could have public - advanceIfStyle(IdentifierStyle::Access); - - if (!advanceIfToken(TokenType::Identifier, &node->m_super)) - { - return SLANG_OK; - } - } - - // We only accept a single super class. Consume everything afterwards until we hit the { brace - - if (m_reader.peekTokenType() != TokenType::LBrace) - { - // Consume up until we see a brace else it's an error - while (true) - { - const TokenType peekTokenType = m_reader.peekTokenType(); - if (peekTokenType == TokenType::EndOfFile) - { - // Expecting brace - m_sink->diagnose( - m_reader.peekToken(), - CPPDiagnostics::expectingToken, - TokenType::LBrace); - return SLANG_FAIL; - } - else if (peekTokenType == TokenType::LBrace) - { - break; - } - m_reader.advanceToken(); - } - - return pushScope(node); - } - - const Token braceToken = m_reader.advanceToken(); - - // Push the class scope - return pushScope(node); -} - -SlangResult Parser::_consumeToSync() -{ - while (true) - { - TokenType type = m_reader.peekTokenType(); - - switch (type) - { - case TokenType::Semicolon: - { - m_reader.advanceToken(); - return SLANG_OK; - } - case TokenType::Pound: - case TokenType::EndOfFile: - case TokenType::LBrace: - case TokenType::RBrace: - { - return SLANG_OK; - } - } - - m_reader.advanceToken(); - } -} - -SlangResult Parser::_maybeParseTemplateArg(Index& ioTemplateDepth) -{ - switch (m_reader.peekTokenType()) - { - case TokenType::Identifier: - { - TokenReader::ParsingCursor nameCursor; - SLANG_RETURN_ON_FAIL(_maybeParseType(ioTemplateDepth, nameCursor)); - return SLANG_OK; - } - case TokenType::IntegerLiteral: - { - m_reader.advanceToken(); - return SLANG_OK; - } - default: - break; - } - return SLANG_FAIL; -} - -SlangResult Parser::_maybeParseTemplateArgs(Index& ioTemplateDepth) -{ - if (!advanceIfToken(TokenType::OpLess)) - { - return SLANG_FAIL; - } - - ioTemplateDepth++; - - while (true) - { - if (ioTemplateDepth == 0) - { - return SLANG_OK; - } - - switch (m_reader.peekTokenType()) - { - case TokenType::OpGreater: - { - if (ioTemplateDepth <= 0) - { - m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::unexpectedTemplateClose); - return SLANG_FAIL; - } - ioTemplateDepth--; - m_reader.advanceToken(); - return SLANG_OK; - } - case TokenType::OpRsh: - { - if (ioTemplateDepth <= 1) - { - m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::unexpectedTemplateClose); - return SLANG_FAIL; - } - ioTemplateDepth -= 2; - m_reader.advanceToken(); - return SLANG_OK; - } - default: - { - while (true) - { - SLANG_RETURN_ON_FAIL(_maybeParseTemplateArg(ioTemplateDepth)); - - if (m_reader.peekTokenType() == TokenType::Comma) - { - m_reader.advanceToken(); - // If there is a comma parse another arg - continue; - } - break; - } - break; - } - } - } -} - -SlangResult Parser::_maybeConsume(IdentifierStyle style) -{ - while (advanceIfStyle(style)) - ; - return SLANG_OK; -} - -// True if two of these token types of the same type placed immediately after one another -// produce a different token. Can be conservative, as if not strictly required -// it will just mean more spacing in the output -static bool _canRepeatTokenType(TokenType type) -{ - switch (type) - { - case TokenType::OpAdd: - case TokenType::OpSub: - case TokenType::OpAnd: - case TokenType::OpOr: - case TokenType::OpGreater: - case TokenType::OpLess: - case TokenType::Identifier: - case TokenType::OpAssign: - case TokenType::Colon: - { - return false; - } - default: - break; - } - return true; -} - -// Returns true if there needs to be a space between the previous token type, and the current token -// type for correct output. It is assumed that the token stream is appropriate. -// The implementation might need more sophistication, but this at least avoids Blah const * -> -// Blahconst* -static bool _tokenConcatNeedsSpace(TokenType prev, TokenType cur) -{ - if ((cur == TokenType::OpAssign) || (prev == cur && !_canRepeatTokenType(cur))) - { - return true; - } - return false; -} - -void Parser::_getTypeTokens( - TokenReader::ParsingCursor start, - TokenReader::ParsingCursor nameCursor, - List& outToks) -{ - auto endCursor = m_reader.getCursor(); - m_reader.setCursor(start); - - while (!m_reader.isAtCursor(endCursor)) - { - if (m_reader.getCursor() == nameCursor) - { - m_reader.advanceToken(); - } - else - { - outToks.add(m_reader.advanceToken()); - } - } -} - -UnownedStringSlice Parser::_concatType( - TokenReader::ParsingCursor start, - TokenReader::ParsingCursor nameCursor) -{ - List toks; - _getTypeTokens(start, nameCursor, toks); - return _concatTokens(toks.getBuffer(), toks.getCount()); -} - -UnownedStringSlice Parser::_concatTokens(const Token* toks, Index toksCount) -{ - StringBuilder buf; - - TokenType prevTokenType = TokenType::Unknown; - for (Index i = 0; i < toksCount; ++i) - { - const auto token = toks[i]; - - // Check if we need a space between tokens - if (_tokenConcatNeedsSpace(prevTokenType, token.type)) - { - buf << " "; - } - - buf << token.getContent(); - - prevTokenType = token.type; - } - - StringSlicePool* typePool = m_nodeTree->m_typePool; - return typePool->getSlice(typePool->add(buf)); -} - -UnownedStringSlice Parser::_concatTokens(TokenReader::ParsingCursor start) -{ - auto endCursor = m_reader.getCursor(); - - m_reader.setCursor(start); - - TokenType prevTokenType = TokenType::Unknown; - - StringBuilder buf; - while (!m_reader.isAtCursor(endCursor)) - { - const Token token = m_reader.advanceToken(); - // Check if we need a space between tokens - if (_tokenConcatNeedsSpace(prevTokenType, token.type)) - { - buf << " "; - } - buf << token.getContent(); - - prevTokenType = token.type; - } - - StringSlicePool* typePool = m_nodeTree->m_typePool; - return typePool->getSlice(typePool->add(buf)); -} - -SlangResult Parser::_maybeParseType( - Index& ioTemplateDepth, - TokenReader::ParsingCursor& outNameCursor) -{ - outNameCursor = TokenReader::ParsingCursor(); - - while (true) - { - if (m_reader.peekTokenType() == TokenType::Identifier) - { - const IdentifierStyle style = - m_nodeTree->m_identifierLookup->get(m_reader.peekToken().getContent()); - - if (style == IdentifierStyle::TypeModifier || - style == IdentifierStyle::IntegerModifier || style == IdentifierStyle::Class || - style == IdentifierStyle::Struct) - { - // These are ok keywords in this context - } - else if (hasFlag(style, IdentifierFlag::Keyword)) - { - return SLANG_FAIL; - } - } - - _maybeConsume(IdentifierStyle::TypeModifier); - - if (advanceIfStyle(IdentifierStyle::IntegerModifier)) - { - // Consume the integer typename (if there is one) - const Token peekToken = m_reader.peekToken(); - if (peekToken.type == TokenType::Identifier) - { - const IdentifierStyle style = - m_nodeTree->m_identifierLookup->get(peekToken.getContent()); - if (style == IdentifierStyle::IntegerType) - { - m_reader.advanceToken(); - } - } - break; - } - - advanceIfToken(TokenType::Scope); - while (true) - { - // if we have a struct/class prefix in front of a name just consume it. - if (m_reader.peekTokenType() == TokenType::Identifier) - { - const IdentifierStyle style = - m_nodeTree->m_identifierLookup->get(m_reader.peekToken().getContent()); - if (style == IdentifierStyle::Class || style == IdentifierStyle::Struct) - { - m_reader.advanceToken(); - } - } - - Token identifierToken; - if (!advanceIfToken(TokenType::Identifier, &identifierToken)) - { - return SLANG_FAIL; - } - - const IdentifierStyle style = - m_nodeTree->m_identifierLookup->get(identifierToken.getContent()); - if (hasFlag(style, IdentifierFlag::Keyword)) - { - return SLANG_FAIL; - } - - if (advanceIfToken(TokenType::Scope)) - { - continue; - } - break; - } - - if (m_reader.peekTokenType() == TokenType::OpLess) - { - SLANG_RETURN_ON_FAIL(_maybeParseTemplateArgs(ioTemplateDepth)); - } - - if (m_reader.peekTokenType() == TokenType::Scope) - { - // Skip the scope and repeat - m_reader.advanceToken(); - continue; - } - - break; - } - - // Strip all the consts etc modifiers - _maybeConsume(IdentifierStyle::TypeModifier); - - // It's a reference and we are done - if (advanceIfToken(TokenType::OpBitAnd)) - { - return SLANG_OK; - } - - while (true) - { - if (advanceIfToken(TokenType::OpMul)) - { - // Strip all the consts - _maybeConsume(IdentifierStyle::TypeModifier); - continue; - } - break; - } - - if (advanceIfToken(TokenType::LParent)) - { - // TODO(JS): - // Doesn't handle all the modifiers just (*SomeName) - - SLANG_RETURN_ON_FAIL(expect(TokenType::OpMul)); - outNameCursor = m_reader.getCursor(); - SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier)); - - SLANG_RETURN_ON_FAIL(expect(TokenType::RParent)); - - // We need to parse and add the params - if (m_reader.peekTokenType() != TokenType::LParent) - { - m_sink->diagnose( - m_reader.peekToken(), - CPPDiagnostics::expectingToken, - TokenType::LParent); - return SLANG_FAIL; - } - - // Consume the params - SLANG_RETURN_ON_FAIL(_consumeBalancedParens()); - } - else if (m_reader.peekTokenType() == TokenType::Identifier) - { - auto potentialNameCursor = m_reader.getCursor(); - m_reader.advanceToken(); - if (m_reader.peekTokenType() == TokenType::LBracket) - { - outNameCursor = potentialNameCursor; - while (advanceIfToken(TokenType::LBracket)) - { - List exprToks; - SLANG_RETURN_ON_FAIL(_parseExpression(exprToks)); - SLANG_RETURN_ON_FAIL(expect(TokenType::RBracket)); - } - } - else - { - // Wasn't an array type..., so rewind - m_reader.setCursor(potentialNameCursor); - } - } - - return SLANG_OK; -} - -SlangResult Parser::_maybeParseType(List& outToks, Token& outName) -{ - // Set to unknown - outName = Token(); - - auto startCursor = m_reader.getCursor(); - - TokenReader::ParsingCursor nameCursor; - - Index templateDepth = 0; - SlangResult res = _maybeParseType(templateDepth, nameCursor); - if (SLANG_FAILED(res) && m_sink->getErrorCount()) - { - return res; - } - - if (templateDepth != 0) - { - m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::unexpectedTemplateClose); - return SLANG_FAIL; - } - - auto endCursor = m_reader.getCursor(); - m_reader.setCursor(startCursor); - - if (nameCursor.isValid()) - { - while (!m_reader.isAtCursor(endCursor)) - { - if (m_reader.getCursor() == nameCursor) - { - outName = m_reader.advanceToken(); - } - else - { - outToks.add(m_reader.advanceToken()); - } - } - } - else - { - while (!m_reader.isAtCursor(endCursor)) - { - outToks.add(m_reader.advanceToken()); - } - } - - return SLANG_OK; -} - -SlangResult Parser::_parseSpecialMacro() -{ - Token name; - SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &name)); - - List params; - - if (m_reader.peekTokenType() == TokenType::LParent) - { - // Mark the start - auto startCursor = m_reader.getCursor(); - - // Consume the params - SLANG_RETURN_ON_FAIL(_consumeBalancedParens()); - - auto endCursor = m_reader.getCursor(); - m_reader.setCursor(startCursor); - - while (!m_reader.isAtCursor(endCursor)) - { - params.add(m_reader.advanceToken()); - } - } - - // Can do special handling here - const UnownedStringSlice suffix = name.getContent().tail(m_options->m_markPrefix.getLength()); - - if (suffix == "COM_INTERFACE") - { - // TODO(JS): It's a com interface. Extact the GUID - } - - return SLANG_OK; -} - -SlangResult Parser::_parseMarker() -{ - SLANG_ASSERT( - m_reader.peekTokenType() == TokenType::Identifier && - _isMarker(m_reader.peekToken().getContent()) && m_currentScope->isClassLike()); - - ClassLikeNode* node = as(m_currentScope); - - if (node->m_marker.type != TokenType::Unknown) - { - m_sink->diagnose( - m_reader.peekToken(), - CPPDiagnostics::classMarkerAlreadyFound, - node->m_name.getContent()); - m_sink->diagnose(node->m_marker, CPPDiagnostics::previousLocation); - return SLANG_FAIL; - } - - // Set the marker token. - node->m_marker = m_reader.advanceToken(); - - // Looks like it's a marker - UnownedStringSlice slice(node->m_marker.getContent()); - - // Strip the prefix and suffix - slice = UnownedStringSlice( - slice.begin() + m_options->m_markPrefix.getLength(), - slice.end() - m_options->m_markSuffix.getLength()); - - // Strip ABSTRACT_ if it's there - UnownedStringSlice abstractSlice("ABSTRACT_"); - if (slice.startsWith(abstractSlice)) - { - slice = UnownedStringSlice(slice.begin() + abstractSlice.getLength(), slice.end()); - } - - // TODO: We could strip other stuff or have other heuristics there, but this is - // probably okay for now - - // Set the typeSet - node->m_typeSet = m_nodeTree->getOrAddTypeSet(slice); - - // Okay now looking for ( identifier) - Token typeNameToken; - - SLANG_RETURN_ON_FAIL(expect(TokenType::LParent)); - SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &typeNameToken)); - SLANG_RETURN_ON_FAIL(expect(TokenType::RParent)); - - if (typeNameToken.getContent() != node->m_name.getContent()) - { - m_sink->diagnose( - typeNameToken, - CPPDiagnostics::typeNameDoesntMatch, - node->m_name.getContent()); - return SLANG_FAIL; - } - - // If has the marker it is assumed reflected - node->m_reflectionType = ReflectionType::Reflected; - return SLANG_OK; -} - -SlangResult Parser::_maybeParseType(UnownedStringSlice& outType, Token& outName) -{ - auto startCursor = m_reader.getCursor(); - - Index templateDepth = 0; - - TokenReader::ParsingCursor nameCursor; - - SlangResult res = _maybeParseType(templateDepth, nameCursor); - if (SLANG_FAILED(res) && m_sink->getErrorCount()) - { - return res; - } - - if (templateDepth != 0) - { - m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::unexpectedTemplateClose); - return SLANG_FAIL; - } - - if (nameCursor.isValid()) - { - const auto cursor = m_reader.getCursor(); - m_reader.setCursor(nameCursor); - outName = m_reader.peekToken(); - m_reader.setCursor(cursor); - - // Extract the contents - List toks; - _getTypeTokens(startCursor, nameCursor, toks); - outType = _concatTokens(toks.getBuffer(), toks.getCount()); - } - else - { - // We can build up the out type, from the tokens we found - outType = _concatTokens(startCursor); - } - return SLANG_OK; -} - -static bool _isBalancedOpen(TokenType tokenType) -{ - return tokenType == TokenType::LBrace || tokenType == TokenType::LParent || - tokenType == TokenType::LBracket; -} - -static bool _isBalancedClose(TokenType tokenType) -{ - return tokenType == TokenType::RBrace || tokenType == TokenType::RParent || - tokenType == TokenType::RBracket; -} - -static TokenType _getBalancedClose(TokenType tokenType) -{ - SLANG_ASSERT(_isBalancedOpen(tokenType)); - switch (tokenType) - { - case TokenType::LBrace: - return TokenType::RBrace; - case TokenType::LParent: - return TokenType::RParent; - case TokenType::LBracket: - return TokenType::RBracket; - default: - return TokenType::Unknown; - } -} - -SlangResult Parser::_parseBalanced(DiagnosticSink* sink) -{ - const TokenType openTokenType = m_reader.peekTokenType(); - if (!_isBalancedOpen(openTokenType)) - { - return SLANG_FAIL; - } - - // Save the start token - const Token startToken = m_reader.advanceToken(); - // Get the token type that would close the open - const TokenType closeTokenType = _getBalancedClose(openTokenType); - - while (true) - { - const TokenType tokenType = m_reader.peekTokenType(); - - // If we hit the closing token, we are done - if (tokenType == closeTokenType) - { - m_reader.advanceToken(); - return SLANG_OK; - } - - // If we hit a balanced open, recurse - if (_isBalancedOpen(tokenType)) - { - SLANG_RETURN_ON_FAIL(_parseBalanced(sink)); - continue; - } - - // If we hit a close token that doesn't match, then the balancing has gone wrong - if (_isBalancedClose(tokenType)) - { - // Only diagnose if required - if (sink) - { - sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::unexpectedUnbalancedToken); - sink->diagnose(startToken, CPPDiagnostics::seeOpen); - } - return SLANG_FAIL; - } - - // If we hit the end of the file and have not hit the closing token, then - // somethings gone wrong - if (tokenType == TokenType::EndOfFile) - { - if (sink) - { - sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::unexpectedEndOfFile); - sink->diagnose(startToken, CPPDiagnostics::seeOpen); - } - - return SLANG_FAIL; - } - - // Skip the token - m_reader.advanceToken(); - } -} - -SlangResult Parser::_consumeBalancedParens() -{ - SLANG_ASSERT(m_reader.peekTokenType() == TokenType::LParent); - - Index parenCount = 0; - - while (true) - { - const TokenType tokenType = m_reader.peekTokenType(); - - switch (tokenType) - { - case TokenType::LParent: - { - parenCount++; - break; - } - case TokenType::RParent: - { - --parenCount; - // If no more parens then we are done - if (parenCount == 0) - { - m_reader.advanceToken(); - return SLANG_OK; - } - break; - } - case TokenType::EndOfFile: - { - // If we hit the end of the file, then not balanced - return SLANG_FAIL; - } - default: - break; - } - - m_reader.advanceToken(); - } -} - -SlangResult Parser::_parseExpression(List& outExprTokens) -{ - Index parenCount = 0; - Index bracketCount = 0; - - // TODO(JS): NOTE! This doesn't handle an expression that contains a template params in - // Something, because without knowing what Something is, it's not known if < is a - // comparison or or a 'template' bracket - // - // This can be worked around in the originating source by placing in parens - - while (true) - { - TokenType tokenType = m_reader.peekTokenType(); - - switch (tokenType) - { - case TokenType::LParent: - { - parenCount++; - break; - } - case TokenType::RParent: - { - // If no parens, and nothing else is open then we are done - if (parenCount == 0) - { - if (bracketCount) - { - m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::cannotParseExpression); - return SLANG_FAIL; - } - - return SLANG_OK; - } - --parenCount; - break; - } - case TokenType::LBracket: - { - bracketCount++; - break; - } - case TokenType::RBracket: - { - // If no brackets are open we are done - if (bracketCount == 0) - { - if (parenCount) - { - m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::cannotParseExpression); - return SLANG_FAIL; - } - return SLANG_OK; - } - --bracketCount; - break; - } - case TokenType::EndOfFile: - { - if ((bracketCount | parenCount) == 0) - { - return SLANG_OK; - } - m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::cannotParseExpression); - return SLANG_FAIL; - } - case TokenType::RBrace: - case TokenType::Semicolon: - case TokenType::Comma: - { - if ((bracketCount | parenCount) == 0) - { - return SLANG_OK; - } - break; - } - - default: - break; - } - - outExprTokens.add(m_reader.advanceToken()); - } -} - -SlangResult Parser::_parseTypeDef() -{ - if (!m_currentScope->canContainTypes()) - { - m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::cannotDeclareTypeInScope); - return SLANG_FAIL; - } - - // Consume the typedef - SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier)); - - Token nameToken; - // Parse the type - List toks; - SLANG_RETURN_ON_FAIL(_maybeParseType(toks, nameToken)); - - // Followed by the name - if (nameToken.type != TokenType::Identifier) - { - SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &nameToken)); - } - - if (Node::lookupNameInScope(m_currentScope, nameToken.getContent())) - { - m_sink->diagnose( - nameToken.loc, - CPPDiagnostics::identifierAlreadyDefined, - nameToken.getContent()); - return SLANG_FAIL; - } - - SLANG_RETURN_ON_FAIL(expect(TokenType::Semicolon)); - - RefPtr node = new TypeDefNode; - node->m_name = nameToken; - node->m_reflectionType = m_currentScope->getContainedReflectionType(); - - // Set what aliases too - node->m_targetTypeTokens.swapWith(toks); - - m_currentScope->addChild(node); - - return SLANG_OK; -} - - -bool Parser::_isCtor() -{ - bool isCtor = false; - // It's a constructor - if (m_currentScope->isClassLike() && m_reader.peekTokenType() == TokenType::Identifier && - m_reader.peekToken().getContent() == m_currentScope->m_name.getContent()) - { - // We need to check it's followed immediately by ( to be sure it's a ctor - - auto cursor = m_reader.getCursor(); - m_reader.advanceToken(); - isCtor = (m_reader.peekTokenType() == TokenType::LParent); - m_reader.setCursor(cursor); - } - - return isCtor; -} - -bool isAlphaNumeric(char c) -{ - return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9'); -} - -SlangResult Parser::_maybeParseContained(Node** outNode) -{ - *outNode = nullptr; - - _maybeConsume(IdentifierStyle::CallableMisc); - - bool isStatic = false; - bool isVirtual = false; - - while (m_reader.peekTokenType() == TokenType::Identifier) - { - const IdentifierStyle style = - m_nodeTree->m_identifierLookup->get(m_reader.peekToken().getContent()); - - // Check for virtualness - if (style == IdentifierStyle::Virtual) - { - isVirtual = true; - m_reader.advanceToken(); - continue; - } - - // Check if static - if (style == IdentifierStyle::Static) - { - isStatic = true; - m_reader.advanceToken(); - continue; - } - - break; - } - - _maybeConsume(IdentifierStyle::CallableMisc); - - UnownedStringSlice typeName; - Token nameToken; - - bool isConstructor = false; - - if (m_currentScope->isClassLike()) - { - // If it's a dtor - if (advanceIfToken(TokenType::OpBitNot, &nameToken)) - { - // Dtor - // For Dtor we don't hold the full name just the ~ - Token tok; - SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &tok)); - - if (tok.getContent() != m_currentScope->m_name.getContent()) - { - m_sink->diagnose( - m_reader.peekLoc(), - CPPDiagnostics::destructorNameDoesntMatch, - m_currentScope->m_name.getContent()); - return SLANG_FAIL; - } - } - else if (_isCtor()) - { - nameToken = m_reader.advanceToken(); - isConstructor = true; - } - } - - // If don't have a name it's not a dtor or ctor, so see if it's a type - if (nameToken.type == TokenType::Unknown) - { - if (SLANG_FAILED(_maybeParseType(typeName, nameToken))) - { - if (m_sink->getErrorCount()) - { - return SLANG_FAIL; - } - - _consumeToSync(); - return SLANG_OK; - } - } - - if (nameToken.type == TokenType::Unknown) - { - // Has a calling convention (must be a function/method) - Token callingConventionToken; - advanceIfStyle(IdentifierStyle::CallingConvention, &callingConventionToken); - - // Expecting a name - if (!advanceIfToken(TokenType::Identifier, &nameToken)) - { - _consumeToSync(); - return SLANG_OK; - } - } - - // Handles other scenarios, but here for catching operator overloading - if (nameToken.type == TokenType::Identifier) - { - const auto style = m_nodeTree->m_identifierLookup->get(nameToken.getContent()); - if (style != IdentifierStyle::None) - { - _consumeToSync(); - return SLANG_OK; - } - } - - if (m_reader.peekTokenType() == TokenType::LParent) - { - if (!m_currentScope->canContainCallable()) - { - SLANG_RETURN_ON_FAIL(_consumeBalancedParens()); - // Consume everything up to ; or { - SLANG_RETURN_ON_FAIL(_consumeToSync()); - - return SLANG_OK; - } - - // Looks like it's a callable - m_reader.advanceToken(); - - List params; - - if (m_reader.peekTokenType() != TokenType::RParent) - { - while (true) - { - Token paramName; - UnownedStringSlice type; - SlangResult res = _maybeParseType(type, paramName); - - if (SLANG_FAILED(res)) - { - m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::expectingType); - return res; - } - - if (paramName.type != TokenType::Identifier) - { - if (m_reader.peekTokenType() == TokenType::Identifier) - { - paramName = m_reader.advanceToken(); - } - } - - // If we have a name check for default value - if (paramName.type == TokenType::Identifier && advanceIfToken(TokenType::OpAssign)) - { - // Check if we have a default value - List exprTokens; - SLANG_RETURN_ON_FAIL(_parseExpression(exprTokens)); - } - - CallableNode::Param param; - param.m_name = paramName; - param.m_type = type; - - params.add(param); - - { - const auto peekType = m_reader.peekTokenType(); - if (peekType == TokenType::RParent) - { - break; - } - if (peekType == TokenType::Comma) - { - m_reader.advanceToken(); - continue; - } - } - - m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::expectingToken, ", or ) or ="); - return SLANG_FAIL; - } - } - - // Skip ) - m_reader.advanceToken(); - - // Parse suffix - bool isPure = false; - - // const? - _maybeConsume(IdentifierStyle::TypeModifier); - - if (isConstructor) - { - // Initializer list - if (advanceIfToken(TokenType::Colon)) - { - while (true) - { - auto peekType = m_reader.peekTokenType(); - if (peekType == TokenType::Semicolon || peekType == TokenType::LBrace || - peekType == TokenType::EndOfFile) - { - break; - } - // Consume - m_reader.advanceToken(); - } - } - } - - // = 0 ? or = default - if (advanceIfToken(TokenType::OpAssign)) - { - if (m_reader.peekTokenType() == TokenType::IntegerLiteral) - { - Int value = -1; - if (SLANG_SUCCEEDED( - StringUtil::parseInt(m_reader.peekToken().getContent(), value)) && - value == 0) - { - isPure = true; - m_reader.advanceToken(); - } - else - { - m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::expectingToken, "0"); - return SLANG_FAIL; - } - } - else if (advanceIfStyle(IdentifierStyle::Default)) - { - } - else - { - m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::cannotParseCallable); - return SLANG_FAIL; - } - } - - if (m_reader.peekTokenType() == TokenType::Semicolon) - { - m_reader.advanceToken(); - } - else if (m_reader.peekTokenType() == TokenType::LBrace) - { - SLANG_RETURN_ON_FAIL(consumeToClosingBrace()); - } - else - { - m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::expectingToken, "; or {"); - return SLANG_FAIL; - } - - RefPtr callableNode = new CallableNode; - - callableNode->m_returnType = typeName; - callableNode->m_name = nameToken; - callableNode->m_reflectionType = m_currentScope->getContainedReflectionType(); - - callableNode->m_isVirtual = isVirtual; - callableNode->m_isPure = isPure; - callableNode->m_isStatic = isStatic; - - callableNode->m_params.swapWith(params); - - Node* nodeWithName = m_currentScope->findChild(nameToken.getContent()); - - if (nodeWithName) - { - CallableNode* initialOverload = as(nodeWithName); - if (!initialOverload) - { - m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::cannotOverload); - m_sink->diagnose(nodeWithName->getSourceLoc(), CPPDiagnostics::seeDeclarationOf); - return SLANG_FAIL; - } - - callableNode->m_nextOverload = initialOverload->m_nextOverload; - initialOverload->m_nextOverload = initialOverload; - - m_currentScope->addChildIgnoringName(callableNode); - } - else - { - m_currentScope->addChild(callableNode); - } - - *outNode = callableNode; - return SLANG_OK; - } - else - { - // Looks like variable - if (!m_currentScope->canContainFields() || nameToken.type != TokenType::Identifier) - { - _consumeToSync(); - return SLANG_OK; - } - - // Check if has a default value - if (advanceIfToken(TokenType::OpAssign)) - { - List exprTokens; - SLANG_RETURN_ON_FAIL(_parseExpression(exprTokens)); - } - - // Hit end of field/variable - if (m_reader.peekTokenType() == TokenType::Semicolon) - { - RefPtr fieldNode = new FieldNode; - - fieldNode->m_fieldType = typeName; - fieldNode->m_name = nameToken; - fieldNode->m_reflectionType = m_currentScope->getContainedReflectionType(); - fieldNode->m_isStatic = isStatic; - if (fieldNode->m_reflectionType == ReflectionType::Reflected) - { - static const char* illegalTypes[] = { - "size_t", - "Int", - "UInt", - "Index", - "Count", - "UIndex", - "UCount", - "PtrInt", - "intptr_t", - "uintptr_t"}; - for (const auto& illegalType : illegalTypes) - { - int index = typeName.indexOf(UnownedStringSlice(illegalType)); - if (index != -1) - { - index += UnownedStringSlice(illegalType).getLength(); - if (index >= typeName.getLength() || !isAlphaNumeric(typeName[index])) - { - // Cannot use this type in a field (as it's arch dependent - m_sink->diagnose( - nameToken, - CPPDiagnostics::cannoseUseArchDependentType, - illegalType); - return SLANG_FAIL; - } - } - } - } - m_currentScope->addChild(fieldNode); - - *outNode = fieldNode; - return SLANG_OK; - } - } - - _consumeToSync(); - return SLANG_OK; -} - -/* static */ Node::Kind Parser::_toNodeKind(IdentifierStyle style) -{ - switch (style) - { - case IdentifierStyle::Class: - return Node::Kind::ClassType; - case IdentifierStyle::Struct: - return Node::Kind::StructType; - case IdentifierStyle::Namespace: - return Node::Kind::Namespace; - case IdentifierStyle::Enum: - return Node::Kind::Enum; - case IdentifierStyle::TypeDef: - return Node::Kind::TypeDef; - default: - return Node::Kind::Invalid; - } -} - -static UnownedStringSlice _trimUnderscorePrefix(const UnownedStringSlice& slice) -{ - if (slice.getLength() && slice[0] == '_') - { - return UnownedStringSlice(slice.begin() + 1, slice.end()); - } - else - { - return slice; - } -} - -SlangResult Parser::_parsePreDeclare() -{ - // Skip the declare type token - m_reader.advanceToken(); - - SLANG_RETURN_ON_FAIL(expect(TokenType::LParent)); - - // Get the typeSet - Token typeSetToken; - SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &typeSetToken)); - TypeSet* typeSet = m_nodeTree->getOrAddTypeSet(typeSetToken.getContent()); - - SLANG_RETURN_ON_FAIL(expect(TokenType::Comma)); - - // Get the type of type - Node::Kind nodeKind; - { - Token typeToken; - SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &typeToken)); - - const IdentifierStyle style = m_nodeTree->m_identifierLookup->get(typeToken.getContent()); - - if (style != IdentifierStyle::Struct && style != IdentifierStyle::Class) - { - m_sink->diagnose( - typeToken, - CPPDiagnostics::expectingTypeKeyword, - typeToken.getContent()); - return SLANG_FAIL; - } - nodeKind = _toNodeKind(style); - } - - Token name; - Token super; - - SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &name)); - - if (advanceIfToken(TokenType::Colon)) - { - SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &super)); - } - - SLANG_RETURN_ON_FAIL(expect(TokenType::RParent)); - - switch (nodeKind) - { - case Node::Kind::ClassType: - case Node::Kind::StructType: - { - RefPtr node(new ClassLikeNode(nodeKind)); - - node->m_name = name; - node->m_super = super; - node->m_typeSet = typeSet; - - // Assume it is reflected - node->m_reflectionType = ReflectionType::Reflected; - - SLANG_RETURN_ON_FAIL(pushScope(node)); - // Pop out of the node - popScope(); - break; - } - default: - { - return SLANG_FAIL; - } - } - - - return SLANG_OK; -} - -SlangResult Parser::_parseTypeSet() -{ - // Skip the declare type token - m_reader.advanceToken(); - - SLANG_RETURN_ON_FAIL(expect(TokenType::LParent)); - - Token typeSetToken; - SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &typeSetToken)); - - TypeSet* typeSet = m_nodeTree->getOrAddTypeSet(typeSetToken.getContent()); - - SLANG_RETURN_ON_FAIL(expect(TokenType::Comma)); - - // Get the type of type - Token typeToken; - SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &typeToken)); - - SLANG_RETURN_ON_FAIL(expect(TokenType::RParent)); - - // Set the typename - typeSet->m_typeName = typeToken.getContent(); - - return SLANG_OK; -} - -SlangResult Parser::parse(SourceOrigin* sourceOrigin, const Options* options) -{ - SLANG_ASSERT(options); - m_options = options; - - // Set the current origin - m_sourceOrigin = sourceOrigin; - - SourceFile* sourceFile = sourceOrigin->m_sourceFile; - - SourceManager* manager = sourceFile->getSourceManager(); - - SourceView* sourceView = manager->createSourceView(sourceFile, nullptr, SourceLoc::fromRaw(0)); - - Lexer lexer; - - // Set up the scope stack - m_scopeStack.clear(); - - m_currentScope = m_nodeTree->m_rootNode; - m_scopeStack.add(m_currentScope); - - if (!options->m_requireMark) - { - m_currentScope->m_reflectionOverride = ReflectionType::Reflected; - } - - lexer.initialize(sourceView, m_sink, m_nodeTree->m_namePool, manager->getMemoryArena()); - m_tokenList = lexer.lexAllSemanticTokens(); - // See if there were any errors - if (m_sink->getErrorCount()) - { - return SLANG_FAIL; - } - - m_reader = TokenReader(m_tokenList); - - while (true) - { - switch (m_reader.peekTokenType()) - { - case TokenType::OpBitNot: - { - // Handle dtor - if (m_currentScope->isClassLike()) - { - Node* containedNode = nullptr; - SLANG_RETURN_ON_FAIL(_maybeParseContained(&containedNode)); - } - else - { - // consume - m_reader.advanceToken(); - } - break; - } - case TokenType::Identifier: - { - const IdentifierStyle style = - m_nodeTree->m_identifierLookup->get(m_reader.peekToken().getContent()); - - switch (style) - { - case IdentifierStyle::Extern: - { - m_reader.advanceToken(); - - Token externType; - SLANG_RETURN_ON_FAIL(expect(TokenType::StringLiteral, &externType)); - - if (advanceIfToken(TokenType::LBrace)) - { - // Push a 'special' scope (which is basically transparent) - pushScope(nullptr); - } - break; - } - case IdentifierStyle::Template: - { - SLANG_RETURN_ON_FAIL(_consumeTemplate()); - break; - } - case IdentifierStyle::PreDeclare: - { - SLANG_RETURN_ON_FAIL(_parsePreDeclare()); - break; - } - case IdentifierStyle::TypeSet: - { - SLANG_RETURN_ON_FAIL(_parseTypeSet()); - break; - } - case IdentifierStyle::Reflected: - { - m_reader.advanceToken(); - if (m_currentScope) - { - m_currentScope->m_reflectionOverride = ReflectionType::Reflected; - } - break; - } - case IdentifierStyle::Unreflected: - { - m_reader.advanceToken(); - if (m_currentScope) - { - m_currentScope->m_reflectionOverride = ReflectionType::NotReflected; - } - break; - } - case IdentifierStyle::Access: - { - m_reader.advanceToken(); - SLANG_RETURN_ON_FAIL(expect(TokenType::Colon)); - break; - } - case IdentifierStyle::TypeDef: - { - if (isTypeEnabled(Node::Kind::TypeDef)) - { - SLANG_RETURN_ON_FAIL(_parseTypeDef()); - } - else - { - m_reader.advanceToken(); - SLANG_RETURN_ON_FAIL(_consumeToSync()); - } - break; - } - default: - { - IdentifierFlags flags = getFlags(style); - - if (flags & IdentifierFlag::StartScope) - { - Node::Kind kind = _toNodeKind(style); - SLANG_ASSERT(kind != Node::Kind::Invalid); - - if (isTypeEnabled(kind)) - { - SLANG_RETURN_ON_FAIL(_maybeParseNode(kind)); - } - else - { - SLANG_RETURN_ON_FAIL(_maybeConsumeScope()); - } - } - else - { - UnownedStringSlice content = m_reader.peekToken().getContent(); - - // If it's a marker handle it - if (_isMarker(content)) - { - if (!m_currentScope->isClassLike()) - { - m_sink->diagnose( - m_reader.peekLoc(), - CPPDiagnostics::classMarkerOutsideOfClass); - return SLANG_FAIL; - } - - SLANG_RETURN_ON_FAIL(_parseMarker()); - break; - } - - if (m_options->m_markPrefix.getLength() > 0 && - content.startsWith(m_options->m_markPrefix.getUnownedSlice())) - { - SLANG_RETURN_ON_FAIL(_parseSpecialMacro()); - break; - } - - - // Special case the node that's the root of the hierarchy (as far as - // reflection is concerned) This could be a field - if (m_currentScope->canContainFields() || - m_currentScope->canContainCallable()) - { - Node* containedNode = nullptr; - SLANG_RETURN_ON_FAIL(_maybeParseContained(&containedNode)); - } - else - { - m_reader.advanceToken(); - } - } - break; - } - } - break; - } - case TokenType::LBrace: - { - SLANG_RETURN_ON_FAIL(consumeToClosingBrace()); - break; - } - case TokenType::RBrace: - { - SLANG_RETURN_ON_FAIL(popScope()); - m_reader.advanceToken(); - break; - } - case TokenType::EndOfFile: - { - // Okay we need to confirm that we are in the root node, and with no open braces - if (m_currentScope != m_nodeTree->getRootNode()) - { - m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::braceOpenAtEndOfFile); - return SLANG_FAIL; - } - if (m_sink->getErrorCount()) - return SLANG_FAIL; - return SLANG_OK; - } - case TokenType::Pound: - { - Token token = m_reader.peekToken(); - if (token.flags & TokenFlag::AtStartOfLine) - { - // We are just going to ignore all of these for now.... - m_reader.advanceToken(); - for (;;) - { - auto t = m_reader.peekToken(); - if (t.type == TokenType::EndOfFile || (t.flags & TokenFlag::AtStartOfLine)) - { - break; - } - m_reader.advanceToken(); - } - break; - } - // Skip it then - m_reader.advanceToken(); - break; - } - default: - { - // Skip it then - m_reader.advanceToken(); - break; - } - } - } -} - -} // namespace CppExtract diff --git a/tools/slang-cpp-extractor/parser.h b/tools/slang-cpp-extractor/parser.h deleted file mode 100644 index 605a0d8be..000000000 --- a/tools/slang-cpp-extractor/parser.h +++ /dev/null @@ -1,115 +0,0 @@ -#ifndef CPP_EXTRACT_PARSER_H -#define CPP_EXTRACT_PARSER_H - -#include "../../source/compiler-core/slang-lexer.h" -#include "diagnostics.h" -#include "identifier-lookup.h" -#include "node-tree.h" -#include "node.h" - -namespace CppExtract -{ -using namespace Slang; - -class Parser -{ -public: - typedef uint32_t NodeTypeBitType; - - SlangResult expect(TokenType type, Token* outToken = nullptr); - - bool advanceIfMarker(Token* outToken = nullptr); - bool advanceIfToken(TokenType type, Token* outToken = nullptr); - bool advanceIfStyle(IdentifierStyle style, Token* outToken = nullptr); - - SlangResult pushAnonymousNamespace(); - SlangResult pushScope(ScopeNode* node); - SlangResult consumeToClosingBrace(const Token* openBraceToken = nullptr); - SlangResult popScope(); - - /// Parse the contents of the source file - SlangResult parse(SourceOrigin* sourceOrigin, const Options* options); - - void setKindEnabled(Node::Kind kind, bool isEnabled = true); - bool isTypeEnabled(Node::Kind kind) - { - return (m_nodeTypeEnabled & (NodeTypeBitType(1) << int(kind))) != 0; - } - - void setKindsEnabled(const Node::Kind* kinds, Index kindsCount, bool isEnabled = true); - - Parser(NodeTree* nodeTree, DiagnosticSink* sink); - -protected: - static Node::Kind _toNodeKind(IdentifierStyle style); - - bool _isMarker(const UnownedStringSlice& name); - - SlangResult _maybeConsumeScope(); - - SlangResult _parsePreDeclare(); - SlangResult _parseTypeSet(); - - SlangResult _maybeParseNode(Node::Kind kind); - SlangResult _maybeParseContained(Node** outNode); - - SlangResult _parseTypeDef(); - SlangResult _parseEnum(); - SlangResult _parseMarker(); - SlangResult _parseSpecialMacro(); - - SlangResult _maybeParseType(List& outToks, Token& outName); - SlangResult _maybeParseType(UnownedStringSlice& outType, Token& outName); - SlangResult _maybeParseType(Index& ioTemplateDepth, TokenReader::ParsingCursor& outCursor); - - SlangResult _parseExpression(List& outExprTokens); - - SlangResult _maybeParseTemplateArgs(Index& ioTemplateDepth); - SlangResult _maybeParseTemplateArg(Index& ioTemplateDepth); - - /// Parse balanced - if a sink is set will report to that sink - SlangResult _parseBalanced(DiagnosticSink* sink); - - bool _isCtor(); - - /// Concatenate all tokens from start to the current position - UnownedStringSlice _concatTokens(TokenReader::ParsingCursor start); - UnownedStringSlice _concatTokens(const Token* toks, Index toksCount); - - UnownedStringSlice _concatType( - TokenReader::ParsingCursor start, - TokenReader::ParsingCursor nameCursor); - - void _getTypeTokens( - TokenReader::ParsingCursor start, - TokenReader::ParsingCursor nameCursor, - List& outToks); - - /// Consume what looks like a template definition - SlangResult _consumeTemplate(); - SlangResult _maybeConsume(IdentifierStyle style); - - SlangResult _consumeToSync(); - /// Consumes balanced parens. Will return an error if not matched. Assumes starts on opening ( - SlangResult _consumeBalancedParens(); - - NodeTypeBitType m_nodeTypeEnabled; - - TokenList m_tokenList; - TokenReader m_reader; - - List m_scopeStack; - - ScopeNode* m_currentScope; ///< The current scope being processed - SourceOrigin* m_sourceOrigin; ///< The source origin that all tokens are in - - DiagnosticSink* m_sink; ///< Diagnostic sink - - NodeTree* m_nodeTree; ///< Shared state between parses. Nodes will be added to this - - const Options* m_options; -}; - -} // namespace CppExtract - -#endif diff --git a/tools/slang-cpp-extractor/unit-test.cpp b/tools/slang-cpp-extractor/unit-test.cpp deleted file mode 100644 index 3c0edac3c..000000000 --- a/tools/slang-cpp-extractor/unit-test.cpp +++ /dev/null @@ -1,127 +0,0 @@ -#include "unit-test.h" - -#include "../../source/compiler-core/slang-lexer.h" -#include "../../source/compiler-core/slang-name-convention-util.h" -#include "../../source/compiler-core/slang-source-loc.h" -#include "../../source/core/slang-io.h" -#include "identifier-lookup.h" -#include "node-tree.h" -#include "options.h" -#include "parser.h" - -namespace CppExtract -{ -using namespace Slang; - - -struct TestState -{ - TestState() - : m_slicePool(StringSlicePool::Style::Default) - { - m_identifierLookup.initDefault(UnownedStringSlice::fromLiteral("SLANG_")); - - m_sourceManager.initialize(nullptr, nullptr); - - m_sink.init(&m_sourceManager, Lexer::sourceLocationLexer); - - m_namePool.setRootNamePool(&m_rootNamePool); - - // We don't require marker - m_options.m_requireMark = false; - } - - RootNamePool m_rootNamePool; - Options m_options; - SourceManager m_sourceManager; - DiagnosticSink m_sink; - NamePool m_namePool; - StringSlicePool m_slicePool; - IdentifierLookup m_identifierLookup; -}; - -static const char someSource[] = "class ISomeInterface\n" - "{\n" - " public:\n" - " virtual int SLANG_MCALL someMethod(int a, int b) const = 0;\n" - " virtual float SLANG_MCALL anotherMethod(float a) = 0;\n" - "};\n" - "\n" - "struct SomeStruct\n" - "{\n" - " SomeStruct() = default;\n" - " SomeStruct(float v = 0.0f):b(v) {}\n" - " ~SomeStruct() {}\n" - " int a = 10; \n" - " float b; \n" - " int another[10];\n" - " const char* yetAnother = nullptr;\n" - "};\n" - "\n" - "enum SomeEnum\n" - "{\n" - " Value,\n" - " Another = 10,\n" - "};\n" - "\n" - "typedef int (*SomeFunc)(int a);\n" - "\n" - "typedef SomeEnum AliasEnum;\n" - "void someFunc(int a, float b) { }\n" - "namespace Blah {\n" - "int add(int a, int b) { return a + b; }\n" - "unsigned add(unsigned a, unsigned b) { return a + b; }\n" - "}\n"; - - -/* static */ SlangResult UnitTestUtil::run() -{ - { - TestState state; - - NodeTree tree(&state.m_slicePool, &state.m_namePool, &state.m_identifierLookup); - - UnownedStringSlice contents = UnownedStringSlice::fromLiteral(someSource); - PathInfo pathInfo = PathInfo::makeFromString("source.h"); - - SourceManager* sourceManager = &state.m_sourceManager; - - SourceFile* sourceFile = sourceManager->createSourceFileWithString(pathInfo, contents); - SourceOrigin* sourceOrigin = tree.addSourceOrigin(sourceFile, state.m_options); - - Parser parser(&tree, &state.m_sink); - - - { - const Node::Kind enableKinds[] = { - Node::Kind::Enum, - Node::Kind::EnumClass, - Node::Kind::EnumCase, - Node::Kind::TypeDef}; - parser.setKindsEnabled(enableKinds, SLANG_COUNT_OF(enableKinds)); - } - - SlangResult res = parser.parse(sourceOrigin, &state.m_options); - - if (state.m_sink.outputBuffer.getLength()) - { - printf("%s\n", state.m_sink.outputBuffer.getBuffer()); - } - - if (SLANG_FAILED(res)) - { - return res; - } - - { - StringBuilder buf; - tree.getRootNode()->dump(0, buf); - - SLANG_UNUSED(buf); - } - } - - return SLANG_OK; -} - -} // namespace CppExtract diff --git a/tools/slang-cpp-extractor/unit-test.h b/tools/slang-cpp-extractor/unit-test.h deleted file mode 100644 index fd3ab1328..000000000 --- a/tools/slang-cpp-extractor/unit-test.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef CPP_EXTRACT_UNIT_TEST_H -#define CPP_EXTRACT_UNIT_TEST_H - -#include "diagnostics.h" - -namespace CppExtract -{ -using namespace Slang; - -struct UnitTestUtil -{ - static SlangResult run(); -}; - -} // namespace CppExtract - -#endif diff --git a/tools/slang-cpp-parser/diagnostic-defs.h b/tools/slang-cpp-parser/diagnostic-defs.h new file mode 100644 index 000000000..cba5a8e6f --- /dev/null +++ b/tools/slang-cpp-parser/diagnostic-defs.h @@ -0,0 +1,92 @@ +// + +// The file is meant to be included multiple times, to produce different +// pieces of declaration/definition code related to diagnostic messages +// +// Each diagnostic is declared here with: +// +// DIAGNOSTIC(id, severity, name, messageFormat) +// +// Where `id` is the unique diagnostic ID, `severity` is the default +// severity (from the `Severity` enum), `name` is a name used to refer +// to this diagnostic from code, and `messageFormat` is the default +// (non-localized) message for the diagnostic, with placeholders +// for any arguments. + +#ifndef DIAGNOSTIC +#error Need to #define DIAGNOSTIC(...) before including "slang-cpp-parser/diagnostics-defs.h" +#define DIAGNOSTIC(id, severity, name, messageFormat) /* */ +#endif + +DIAGNOSTIC(-1, Note, seeDeclarationOf, "see declaration of '$0'") +DIAGNOSTIC(-1, Note, seeOpen, "see open $0") +DIAGNOSTIC(-1, Note, commandLine, "Command line: $0") +DIAGNOSTIC(-1, Note, previousLocation, "previous location") + +DIAGNOSTIC(1, Error, cannotOpenFile, "cannot open file '$0'.") +DIAGNOSTIC(1, Error, errorAccessingFile, "error accessing file '$0'.") + +DIAGNOSTIC(1, Error, extractorFailed, "C++ Extractor failed") +DIAGNOSTIC(1, Error, internalError, "Unknown internal error in C++ Extractor, aborted!") + +// Parsing errors +DIAGNOSTIC(100000, Error, expectingToken, "Expecting token $0") +DIAGNOSTIC(100001, Error, typeAlreadyDeclared, "Type '$0' already declared") +DIAGNOSTIC(100002, Error, scopeNotClosed, "Scope not closed") +DIAGNOSTIC(100003, Error, typeNameDoesntMatch, "Type name doesn't match $0") +DIAGNOSTIC(100004, Error, didntFindMatchingBrace, "Didn't find brace matching $0") +DIAGNOSTIC(100005, Error, braceOpenAtEndOfFile, "Brace open at file end") +DIAGNOSTIC(100006, Error, unexpectedTemplateClose, "Unexpected template close") +DIAGNOSTIC(100007, Error, superTypeNotFound, "Super type not found for $0") +DIAGNOSTIC(100008, Error, superTypeNotAType, "Named super type is not a type $0") +DIAGNOSTIC(100009, Error, unexpectedUnbalancedToken, "Unexpected unbalanced token") +DIAGNOSTIC(100010, Error, unexpectedEndOfFile, "Unexpected end of file") +DIAGNOSTIC( + 100011, + Error, + expectingTypeKeyword, + "Expecting type keyword - struct or class, found $0") + +DIAGNOSTIC( + 100012, + Error, + typeInDifferentTypeSet, + "Type $0 in different type set $1 from super class $2") +DIAGNOSTIC(100013, Error, expectingIdentifier, "Expecting an identifier, found $0") +DIAGNOSTIC(100014, Error, cannotDeclareTypeInScope, "Cannot declare types in this scope") +DIAGNOSTIC(100015, Error, identifierAlreadyDefined, "Identifier already defined '$0'") +DIAGNOSTIC(100016, Error, expectingType, "Expecting a type") +DIAGNOSTIC(100017, Error, cannotParseExpression, "Cannot parse expression") +DIAGNOSTIC(100018, Error, cannotOverload, "Cannot overload this declaration"); +DIAGNOSTIC( + 100019, + Error, + classMarkerOutsideOfClass, + "A class/struct marker is found outside of class or struct"); +DIAGNOSTIC( + 100020, + Error, + classMarkerAlreadyFound, + "A class/struct marker already found in type '$0'"); +DIAGNOSTIC(100021, Error, cannotParseType, "Cannot parse type"); +DIAGNOSTIC( + 100022, + Error, + destructorNameDoesntMatch, + "Destructor name doesn't match class name '$0'"); +DIAGNOSTIC(100023, Error, cannotParseCallable, "Cannot parse callable"); +DIAGNOSTIC( + 100024, + Error, + cannoseUseArchDependentType, + "Cannot use architecture dependent type '$0' for serializable data.") + +// Command line errors 100100 + +DIAGNOSTIC(100101, Error, optionAlreadyDefined, "Option '$0' is already defined '$1'") +DIAGNOSTIC(100102, Error, requireValueAfterOption, "Require a value after $0 option") +DIAGNOSTIC(100103, Error, unknownOption, "Unknown option '$0'") +DIAGNOSTIC(100104, Error, noInputPathsSpecified, "No input paths specified") +DIAGNOSTIC(100105, Error, noOutputPathSpecified, "No -o output path specified") + +#undef DIAGNOSTIC diff --git a/tools/slang-cpp-parser/diagnostics.cpp b/tools/slang-cpp-parser/diagnostics.cpp new file mode 100644 index 000000000..10455a9ff --- /dev/null +++ b/tools/slang-cpp-parser/diagnostics.cpp @@ -0,0 +1,15 @@ +#include "diagnostics.h" + +namespace CppParse +{ + +namespace CPPDiagnostics +{ +using namespace Slang; + +#define DIAGNOSTIC(id, severity, name, messageFormat) \ + const DiagnosticInfo name = {id, Severity::severity, #name, messageFormat}; +#include "diagnostic-defs.h" +} // namespace CPPDiagnostics + +} // namespace CppParse diff --git a/tools/slang-cpp-parser/diagnostics.h b/tools/slang-cpp-parser/diagnostics.h new file mode 100644 index 000000000..4db70fbca --- /dev/null +++ b/tools/slang-cpp-parser/diagnostics.h @@ -0,0 +1,16 @@ +#pragma once + +#include "slang/slang-diagnostics.h" + +namespace CppParse +{ +using namespace Slang; + +namespace CPPDiagnostics +{ + +#define DIAGNOSTIC(id, severity, name, messageFormat) extern const DiagnosticInfo name; +#include "diagnostic-defs.h" + +} // namespace CPPDiagnostics +} // namespace CppParse diff --git a/tools/slang-cpp-parser/file-util.cpp b/tools/slang-cpp-parser/file-util.cpp new file mode 100644 index 000000000..9e7c60755 --- /dev/null +++ b/tools/slang-cpp-parser/file-util.cpp @@ -0,0 +1,101 @@ +#include "file-util.h" + +#include "core/slang-io.h" + +namespace CppParse +{ +using namespace Slang; + +namespace +{ // anonymous +struct DiagnosticReporter +{ + SlangResult report(SlangResult res) + { + if (SLANG_FAILED(res)) + { + if (m_sink) + { + if (res == SLANG_E_CANNOT_OPEN) + { + m_sink->diagnose(SourceLoc(), CPPDiagnostics::cannotOpenFile, m_filename); + } + else + { + m_sink->diagnose(SourceLoc(), CPPDiagnostics::errorAccessingFile, m_filename); + } + } + } + return res; + } + + DiagnosticReporter(const String& filename, DiagnosticSink* sink) + : m_filename(filename), m_sink(sink) + { + } + + DiagnosticSink* m_sink; + String m_filename; +}; + +} // namespace + +/* static */ SlangResult FileUtil::readAllText( + const Slang::String& fileName, + DiagnosticSink* sink, + String& outRead) +{ + DiagnosticReporter reporter(fileName, sink); + + RefPtr stream = new FileStream; + SLANG_RETURN_ON_FAIL(reporter.report( + stream->init(fileName, FileMode::Open, FileAccess::Read, FileShare::ReadWrite))); + + StreamReader reader; + SLANG_RETURN_ON_FAIL(reporter.report(reader.init(stream))); + SLANG_RETURN_ON_FAIL(reporter.report(reader.readToEnd(outRead))); + + return SLANG_OK; +} + +/* static */ SlangResult FileUtil::writeAllText( + const Slang::String& fileName, + DiagnosticSink* sink, + const UnownedStringSlice& text) +{ + // TODO(JS): + // There is an optimization/behavior here,that checks if the contents has changed. It only + // writes if it hasn't That might not be what you want (both because of extra work of read, the + // file modified stamp or other reasons, file is write only etc) NOTE! That this also does the + // work of the comparison after it is decoded, but the *bytes* might actually be different. + + if (File::exists(fileName)) + { + String existingText; + if (SLANG_SUCCEEDED(readAllText(fileName, nullptr, existingText))) + { + if (existingText == text) + return SLANG_OK; + } + } + + DiagnosticReporter reporter(fileName, sink); + + RefPtr stream = new FileStream; + SLANG_RETURN_ON_FAIL(reporter.report(stream->init(fileName, FileMode::Create))); + + StreamWriter writer; + SLANG_RETURN_ON_FAIL(reporter.report(writer.init(stream))); + SLANG_RETURN_ON_FAIL(reporter.report(writer.write(text))) + return SLANG_OK; +} + +/* static */ void FileUtil::indent(Index indentCount, StringBuilder& out) +{ + for (Index i = 0; i < indentCount; ++i) + { + out << CPP_EXTRACT_INDENT_STRING; + } +} + +} // namespace CppParse diff --git a/tools/slang-cpp-parser/file-util.h b/tools/slang-cpp-parser/file-util.h new file mode 100644 index 000000000..735758384 --- /dev/null +++ b/tools/slang-cpp-parser/file-util.h @@ -0,0 +1,31 @@ +#pragma once + +#include "diagnostics.h" + +namespace CppParse +{ +using namespace Slang; + +// A macro to define a single indent as a string +#define CPP_EXTRACT_INDENT_STRING " " + +struct FileUtil +{ + /// Read text into outRead. Any failures written to sink (can be passed as nullptr, for no + /// output) + static SlangResult readAllText( + const Slang::String& fileName, + DiagnosticSink* sink, + String& outRead); + /// Write text to filename. Any failures written to sink. (can be passed as nullptr, for no + /// output) + static SlangResult writeAllText( + const Slang::String& fileName, + DiagnosticSink* sink, + const UnownedStringSlice& text); + + /// Appends CPP_EXTRACT_INDENT_STRING indentCount number of times to out + static void indent(Index indentCount, StringBuilder& out); +}; + +} // namespace CppParse diff --git a/tools/slang-cpp-parser/identifier-lookup.cpp b/tools/slang-cpp-parser/identifier-lookup.cpp new file mode 100644 index 000000000..3a704a454 --- /dev/null +++ b/tools/slang-cpp-parser/identifier-lookup.cpp @@ -0,0 +1,177 @@ +#include "identifier-lookup.h" + +namespace CppParse +{ +using namespace Slang; + +/* static */ const IdentifierFlags + IdentifierLookup::kIdentifierFlags[Index(IdentifierStyle::CountOf)] = { + 0, /// None + 0, /// Identifier + 0, /// Declare type + 0, /// Type set + IdentifierFlag::Keyword, /// TypeModifier + IdentifierFlag::Keyword, /// Keyword + + IdentifierFlag::Keyword | IdentifierFlag::StartScope | IdentifierFlag::ClassLike, /// Class + IdentifierFlag::Keyword | IdentifierFlag::StartScope | IdentifierFlag::ClassLike, /// Struct + IdentifierFlag::Keyword | IdentifierFlag::StartScope, /// Namespace + IdentifierFlag::Keyword | IdentifierFlag::StartScope, /// Enum + + IdentifierFlag::Keyword, /// Typedef + + IdentifierFlag::Keyword, /// Access + IdentifierFlag::Reflection, /// Reflected + IdentifierFlag::Reflection, /// Unreflected + + IdentifierFlag::Keyword, /// virtual + 0, /// Calling convention + IdentifierFlag::Keyword, /// template + IdentifierFlag::Keyword, /// static + + IdentifierFlag::Keyword, /// unsigned/signed + + IdentifierFlag::Keyword, /// extern + + 0, /// Callable misc + 0, /// IntegerType int, short, char, long + + IdentifierFlag::Keyword, /// default +}; + +void IdentifierLookup::set(const UnownedStringSlice& name, IdentifierStyle style) +{ + StringSlicePool::Handle handle; + if (m_pool.findOrAdd(name, handle)) + { + // Add the extra flags + m_styles[Index(handle)] = style; + } + else + { + Index index = Index(handle); + SLANG_ASSERT(index == m_styles.getCount()); + m_styles.add(style); + } +} + +void IdentifierLookup::set(const char* const* names, size_t namesCount, IdentifierStyle style) +{ + for (size_t i = 0; i < namesCount; ++i) + { + set(UnownedStringSlice(names[i]), style); + } +} + +void IdentifierLookup::set(const Pair* pairs, Index pairsCount) +{ + for (Index i = 0; i < pairsCount; ++i) + { + const auto& pair = pairs[i]; + set(UnownedStringSlice(pair.name), pair.style); + } +} + +void IdentifierLookup::initDefault(const UnownedStringSlice& markPrefix) +{ + reset(); + + // Some keywords + { + const char* names[] = { + "continue", + "if", + "case", + "break", + "catch", + "delete", + "do", + "else", + "for", + "new", + "goto", + "return", + "switch", + "throw", + "using", + "while", + "operator", + "explicit"}; + set(names, SLANG_COUNT_OF(names), IdentifierStyle::Keyword); + } + + // Type modifier keywords + { + const char* names[] = {"const", "volatile"}; + set(names, SLANG_COUNT_OF(names), IdentifierStyle::TypeModifier); + } + + // Special markers + { + const char* names[] = {"PRE_DECLARE", "TYPE_SET", "REFLECTED", "UNREFLECTED"}; + const IdentifierStyle styles[] = { + IdentifierStyle::PreDeclare, + IdentifierStyle::TypeSet, + IdentifierStyle::Reflected, + IdentifierStyle::Unreflected}; + SLANG_COMPILE_TIME_ASSERT(SLANG_COUNT_OF(names) == SLANG_COUNT_OF(styles)); + + StringBuilder buf; + for (Index i = 0; i < SLANG_COUNT_OF(names); ++i) + { + buf.clear(); + buf << markPrefix << names[i]; + set(buf.getUnownedSlice(), styles[i]); + } + } + + { + set("virtual", IdentifierStyle::Virtual); + + set("template", IdentifierStyle::Template); + set("static", IdentifierStyle::Static); + set("extern", IdentifierStyle::Extern); + set("default", IdentifierStyle::Default); + } + + { + const char* names[] = {"char", "short", "int", "long"}; + set(names, SLANG_COUNT_OF(names), IdentifierStyle::IntegerType); + } + + { + const char* names[] = {"SLANG_MCALL"}; + set(names, SLANG_COUNT_OF(names), IdentifierStyle::CallingConvention); + } + + { + const char* names[] = {"SLANG_NO_THROW", "inline"}; + set(names, SLANG_COUNT_OF(names), IdentifierStyle::CallableMisc); + } + + // Keywords which introduce types/scopes + { + const Pair pairs[] = { + {"struct", IdentifierStyle::Struct}, + {"class", IdentifierStyle::Class}, + {"namespace", IdentifierStyle::Namespace}, + {"enum", IdentifierStyle::Enum}, + {"typedef", IdentifierStyle::TypeDef}, + }; + + set(pairs, SLANG_COUNT_OF(pairs)); + } + + // Keywords that control access + { + const char* names[] = {"private", "protected", "public"}; + set(names, SLANG_COUNT_OF(names), IdentifierStyle::Access); + } + { + const char* names[] = {"signed", "unsigned"}; + + set(names, SLANG_COUNT_OF(names), IdentifierStyle::IntegerModifier); + } +} + +} // namespace CppParse diff --git a/tools/slang-cpp-parser/identifier-lookup.h b/tools/slang-cpp-parser/identifier-lookup.h new file mode 100644 index 000000000..226d00667 --- /dev/null +++ b/tools/slang-cpp-parser/identifier-lookup.h @@ -0,0 +1,121 @@ +#pragma once + +#include "diagnostics.h" + +namespace CppParse +{ +using namespace Slang; + +enum class IdentifierStyle +{ + None, ///< It's not an identifier + + Identifier, ///< Just an identifier + + PreDeclare, ///< Declare a type (not visible in C++ code) + TypeSet, ///< TypeSet + + TypeModifier, ///< const, volatile etc + Keyword, ///< A keyword C/C++ keyword that is not another type + + Class, ///< class + Struct, ///< struct + Namespace, ///< namespace + Enum, ///< enum + + TypeDef, ///< typedef + + Access, ///< public, protected, private + + Reflected, + Unreflected, + + CallingConvention, ///< Used on a method + Virtual, ///< + + Template, + + Static, + + IntegerModifier, + + Extern, + + CallableMisc, ///< For SLANG_NO_THROW etc + + IntegerType, ///< Built in integer type + + Default, /// default + + CountOf, +}; + +typedef uint32_t IdentifierFlags; +struct IdentifierFlag +{ + enum Enum : IdentifierFlags + { + StartScope = 0x1, ///< namespace, struct or class + ClassLike = 0x2, ///< Struct or class + Keyword = 0x4, + Reflection = 0x8, + }; +}; + + +class IdentifierLookup +{ +public: + struct Pair + { + const char* name; + IdentifierStyle style; + }; + + IdentifierStyle get(const UnownedStringSlice& slice) const + { + Index index = m_pool.findIndex(slice); + return (index >= 0) ? m_styles[index] : IdentifierStyle::None; + } + + void set(const char* name, IdentifierStyle style) { set(UnownedStringSlice(name), style); } + + void set(const UnownedStringSlice& name, IdentifierStyle style); + + void set(const char* const* names, size_t namesCount, IdentifierStyle style); + + void set(const Pair* pairs, Index pairsCount); + + void reset() + { + m_styles.clear(); + m_pool.clear(); + } + + void initDefault(const UnownedStringSlice& markPrefix); + + IdentifierLookup() + : m_pool(StringSlicePool::Style::Empty) + { + SLANG_ASSERT(m_pool.getSlicesCount() == 0); + } + + static const IdentifierFlags kIdentifierFlags[Index(IdentifierStyle::CountOf)]; + +protected: + List m_styles; + StringSlicePool m_pool; +}; + + +SLANG_FORCE_INLINE IdentifierFlags getFlags(IdentifierStyle style) +{ + return IdentifierLookup::kIdentifierFlags[Index(style)]; +} + +SLANG_FORCE_INLINE bool hasFlag(IdentifierStyle style, IdentifierFlag::Enum flag) +{ + return (getFlags(style) & flag) != 0; +} + +} // namespace CppParse diff --git a/tools/slang-cpp-parser/node-tree.cpp b/tools/slang-cpp-parser/node-tree.cpp new file mode 100644 index 000000000..e48ce9779 --- /dev/null +++ b/tools/slang-cpp-parser/node-tree.cpp @@ -0,0 +1,176 @@ +#include "node-tree.h" + +#include "compiler-core/slang-name-convention-util.h" +#include "core/slang-io.h" +#include "identifier-lookup.h" +#include "options.h" + +namespace CppParse +{ +using namespace Slang; + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!! NodeTree !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +NodeTree::NodeTree( + StringSlicePool* typePool, + NamePool* namePool, + IdentifierLookup* identifierLookup) + : m_typePool(typePool) + , m_namePool(namePool) + , m_identifierLookup(identifierLookup) + , m_typeSetPool(StringSlicePool::Style::Empty) +{ + m_rootNode = new ScopeNode(Node::Kind::Namespace); + m_rootNode->m_reflectionType = ReflectionType::Reflected; +} + +TypeSet* NodeTree::getTypeSet(const UnownedStringSlice& slice) +{ + Index index = m_typeSetPool.findIndex(slice); + if (index < 0) + { + return nullptr; + } + return m_typeSets[index]; +} + +TypeSet* NodeTree::getOrAddTypeSet(const UnownedStringSlice& slice) +{ + const Index index = Index(m_typeSetPool.add(slice)); + if (index >= m_typeSets.getCount()) + { + SLANG_ASSERT(m_typeSets.getCount() == index); + TypeSet* typeSet = new TypeSet; + + m_typeSets.add(typeSet); + typeSet->m_macroName = m_typeSetPool.getSlice(StringSlicePool::Handle(index)); + return typeSet; + } + else + { + return m_typeSets[index]; + } +} + +SourceOrigin* NodeTree::addSourceOrigin(SourceFile* sourceFile, const Options& options) +{ + // Calculate from the path, a 'macro origin' name. + const String macroOrigin = calcMacroOrigin(sourceFile->getPathInfo().foundPath, options); + + SourceOrigin* origin = new SourceOrigin(sourceFile, macroOrigin); + m_sourceOrigins.add(origin); + return origin; +} + +/* static */ String NodeTree::calcMacroOrigin(const String& filePath, const Options& options) +{ + // Get the filename without extension + String fileName = Path::getFileNameWithoutExt(filePath); + + // We can work on just the slice + UnownedStringSlice slice = fileName.getUnownedSlice(); + + // Filename prefix + if (options.m_stripFilePrefix.getLength() && + slice.startsWith(options.m_stripFilePrefix.getUnownedSlice())) + { + const Index len = options.m_stripFilePrefix.getLength(); + slice = UnownedStringSlice(slice.begin() + len, slice.end()); + } + + // Trim - + slice = slice.trim('-'); + + StringBuilder out; + NameConventionUtil::convert(slice, NameConvention::UpperSnake, out); + return out; +} + +SlangResult NodeTree::_calcDerivedTypesRec(ScopeNode* inScopeNode, DiagnosticSink* sink) +{ + if (inScopeNode->isClassLike()) + { + ClassLikeNode* classLikeNode = static_cast(inScopeNode); + + if (classLikeNode->m_super.hasContent()) + { + ScopeNode* parentScope = classLikeNode->m_parentScope; + if (parentScope == nullptr) + { + sink->diagnoseRaw( + Severity::Error, + UnownedStringSlice::fromLiteral("Can't lookup in scope if there is none!")); + return SLANG_FAIL; + } + + Node* superNode = Node::lookup(parentScope, classLikeNode->m_super.getContent()); + + if (!superNode) + { + if (classLikeNode->isReflected()) + { + sink->diagnose( + classLikeNode->m_name, + CPPDiagnostics::superTypeNotFound, + classLikeNode->getAbsoluteName()); + return SLANG_FAIL; + } + } + else + { + ClassLikeNode* superType = as(superNode); + + if (!superType) + { + sink->diagnose( + classLikeNode->m_name, + CPPDiagnostics::superTypeNotAType, + classLikeNode->getAbsoluteName()); + return SLANG_FAIL; + } + + if (superType->m_typeSet != classLikeNode->m_typeSet) + { + sink->diagnose( + classLikeNode->m_name, + CPPDiagnostics::typeInDifferentTypeSet, + classLikeNode->m_name.getContent(), + classLikeNode->m_typeSet->m_macroName, + superType->m_typeSet->m_macroName); + return SLANG_FAIL; + } + + // The base class must be defined in same scope (as we didn't allow different scopes + // for base classes) + superType->addDerived(classLikeNode); + } + } + else + { + // Add to it's own typeset + if (classLikeNode->isReflected() && classLikeNode->m_typeSet) + { + classLikeNode->m_typeSet->m_baseTypes.add(classLikeNode); + } + } + } + + for (Node* child : inScopeNode->m_children) + { + ScopeNode* childScope = as(child); + if (childScope) + { + SLANG_RETURN_ON_FAIL(_calcDerivedTypesRec(childScope, sink)); + } + } + + return SLANG_OK; +} + +SlangResult NodeTree::calcDerivedTypes(DiagnosticSink* sink) +{ + return _calcDerivedTypesRec(m_rootNode, sink); +} + + +} // namespace CppParse diff --git a/tools/slang-cpp-parser/node-tree.h b/tools/slang-cpp-parser/node-tree.h new file mode 100644 index 000000000..8911d0d71 --- /dev/null +++ b/tools/slang-cpp-parser/node-tree.h @@ -0,0 +1,101 @@ +#pragma once + +#include "compiler-core/slang-lexer.h" +#include "diagnostics.h" +#include "identifier-lookup.h" +#include "node.h" + +namespace CppParse +{ +using namespace Slang; + +class TypeSet : public RefObject +{ +public: + /// This is the looked up name. + UnownedStringSlice + m_macroName; ///< The name extracted from the macro SLANG_ABSTRACT_AST_CLASS -> AST + + String m_typeName; ///< The enum type name associated with this type for AST it is ASTNode + String m_fileMark; ///< This 'mark' becomes of the output filename + + List m_baseTypes; ///< The base types for this type set +}; + +class SourceOrigin : public RefObject +{ +public: + void addNode(Node* node) + { + if (auto classLike = as(node)) + { + SLANG_ASSERT(classLike->m_origin == nullptr); + classLike->m_origin = this; + } + + m_nodes.add(node); + } + + SourceOrigin(SourceFile* sourceFile, const String& macroOrigin) + : m_sourceFile(sourceFile), m_macroOrigin(macroOrigin) + { + } + + String m_macroOrigin; ///< The macro text is inserted into the macro to identify the origin. It + ///< is based on the filename + SourceFile* m_sourceFile; ///< The source file - also holds the path information + + /// All of the nodes defined in this file in the order they were defined + /// Note that the same namespace may be listed multiple times. + List> m_nodes; +}; + +struct Options; +class IdentifierLookup; + +/* NodeTree holds nodes that have been parsed into a tree rooted on the 'rootNode'. +Also contains other state associated with or useful to a node tree */ +class NodeTree +{ +public: + friend class Parser; + /// Get all of the parsed source origins + const List>& getSourceOrigins() const { return m_sourceOrigins; } + + TypeSet* getTypeSet(const UnownedStringSlice& slice); + TypeSet* getOrAddTypeSet(const UnownedStringSlice& slice); + + SourceOrigin* addSourceOrigin(SourceFile* sourceFile, const Options& options); + + /// Get all of the type sets + const List>& getTypeSets() const { return m_typeSets; } + + /// Get the root node + Node* getRootNode() const { return m_rootNode; } + + /// When parsing we don't lookup all up super types/add derived types. This is because + /// we allow files to be processed in any order, so we have to do the type lookup as a separate + /// operation + SlangResult calcDerivedTypes(DiagnosticSink* sink); + + NodeTree(StringSlicePool* typePool, NamePool* namePool, IdentifierLookup* identifierLookup); + + static String calcMacroOrigin(const String& filePath, const Options& options); + +protected: + SlangResult _calcDerivedTypesRec(ScopeNode* node, DiagnosticSink* sink); + + StringSlicePool m_typeSetPool; ///< Pool for type set names + List> m_typeSets; ///< The type sets + + IdentifierLookup* m_identifierLookup; + StringSlicePool* m_typePool; ///< Pool for just types + + NamePool* m_namePool; + + RefPtr m_rootNode; ///< The root scope + + List> m_sourceOrigins; +}; + +} // namespace CppParse diff --git a/tools/slang-cpp-parser/node.cpp b/tools/slang-cpp-parser/node.cpp new file mode 100644 index 000000000..16484ead3 --- /dev/null +++ b/tools/slang-cpp-parser/node.cpp @@ -0,0 +1,700 @@ +#include "node.h" + +#include "core/slang-string-escape-util.h" +#include "core/slang-string-util.h" +#include "file-util.h" + +namespace CppParse +{ + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Node Impl +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +SLANG_FORCE_INLINE static void _indent(Index indentCount, StringBuilder& out) +{ + FileUtil::indent(indentCount, out); +} + +void Node::dumpMarkup(int indentCount, StringBuilder& out) +{ + if (m_markup.getLength() <= 0) + { + return; + } + + List lines; + StringUtil::calcLines(m_markup.getUnownedSlice(), lines); + + // Remove empty lines from the end + while (lines.getCount()) + { + auto lastLine = lines.getLast(); + if (lastLine.trim().getLength() == 0) + { + lines.removeLast(); + continue; + } + break; + } + + if (lines.getCount() == 0) + { + return; + } + + for (auto line : lines) + { + _indent(indentCount, out); + out << "// " << line << "\n"; + } +} + +ScopeNode* Node::getRootScope() +{ + if (m_parentScope) + { + ScopeNode* scope = m_parentScope; + while (scope->m_parentScope) + { + scope = scope->m_parentScope; + } + return scope; + } + else + { + return as(this); + } +} + +void Node::calcScopeDepthFirst(List& outNodes) +{ + outNodes.add(this); +} + +void Node::calcAbsoluteName(StringBuilder& outName) const +{ + List path; + calcScopePath(const_cast(this), path); + + // 1 so we skip the global scope + for (Index i = 1; i < path.getCount(); ++i) + { + Node* node = path[i]; + + if (i > 1) + { + outName << "::"; + } + + if (node->m_kind == Kind::AnonymousNamespace) + { + outName << "{Anonymous}"; + } + else + { + outName << node->m_name.getContent(); + } + } +} + +/* static */ void Node::calcScopePath(Node* node, List& outPath) +{ + outPath.clear(); + + while (node) + { + outPath.add(node); + node = node->m_parentScope; + } + + // reverse the order, so we go from root to the node + outPath.reverse(); +} + +/* static */ void Node::filterImpl(Filter inFilter, List& ioNodes) +{ + // Filter out all the unreflected nodes + Index count = ioNodes.getCount(); + for (Index j = 0; j < count;) + { + Node* node = ioNodes[j]; + + if (!inFilter(node)) + { + ioNodes.removeAt(j); + count--; + } + else + { + j++; + } + } +} + +/* static */ Node* Node::lookupNameInScope(ScopeNode* scope, const UnownedStringSlice& name) +{ + // TODO(JS): Doesn't handle 'using namespace'. + + // Must be unqualified name + SLANG_ASSERT(name.indexOf(UnownedStringSlice::fromLiteral("::")) < 0); + + Node* childNode = scope->findChild(name); + if (childNode) + { + return childNode; + } + + // If we have an anonymous namespace in this scope, try looking up in there.. + if (scope->m_anonymousNamespace) + { + Node* childNode = scope->m_anonymousNamespace->findChild(name); + if (childNode) + { + return childNode; + } + } + + // I could have an enum (that's not an enum class) + for (Node* node : scope->m_children) + { + EnumNode* enumNode = as(node); + if (enumNode && enumNode->m_kind == Node::Kind::Enum) + { + Node** nodePtr = enumNode->m_childMap.tryGetValue(name); + if (nodePtr) + { + return *nodePtr; + } + } + } + + return nullptr; +} + +/* static */ Node* Node::lookupFromScope( + ScopeNode* scope, + const UnownedStringSlice* parts, + Index partsCount) +{ + SLANG_ASSERT(partsCount > 0); + if (partsCount == 1) + { + return lookupNameInScope(scope, parts[0]); + } + + for (Index i = 0; i < partsCount; ++i) + { + const UnownedStringSlice& part = parts[i]; + + Node* node = lookupNameInScope(scope, part); + if (node == nullptr) + { + return node; + } + // If at end, then we are done + if (i == partsCount - 1) + { + return node; + } + + // If there are more elements, then node must be some kind of scope, + // if we are going to find it + scope = as(node); + if (scope == nullptr) + { + break; + } + } + + return nullptr; +} + +/* static */ void Node::splitPath( + const UnownedStringSlice& inPath, + List& outParts) +{ + if (inPath.indexOf(UnownedStringSlice::fromLiteral("::")) >= 0) + { + StringUtil::split(inPath, UnownedStringSlice::fromLiteral("::"), outParts); + // Remove any whitespace + for (auto& part : outParts) + { + part = part.trim(); + } + } + else + { + outParts.clear(); + outParts.add(inPath.trim()); + } +} + +/* static */ Node* Node::lookupFromScope(ScopeNode* scope, const UnownedStringSlice& inPath) +{ + if (inPath.indexOf(UnownedStringSlice::fromLiteral("::")) >= 0) + { + List parts; + splitPath(inPath, parts); + + return lookupFromScope(scope, parts.getBuffer(), parts.getCount()); + } + else + { + return lookupNameInScope(scope, inPath); + } +} + +/* static */ Node* Node::lookup(ScopeNode* scope, const UnownedStringSlice& inPath) +{ + if (inPath.indexOf(UnownedStringSlice::fromLiteral("::")) >= 0) + { + List parts; + splitPath(inPath, parts); + + if (parts[0].getLength() == 0) + { + // It's a lookup from global scope + ScopeNode* rootScope = scope->getRootScope(); + return lookupFromScope(rootScope, parts.getBuffer() + 1, parts.getCount() + 1); + } + + // Okay lets try a lookup from each scope up to the global scope + while (scope) + { + Node* node = lookupFromScope(scope, parts.getBuffer(), parts.getCount()); + if (node) + { + return node; + } + + scope = scope->m_parentScope; + } + } + else + { + while (scope) + { + // Lookup in this scope + Node* node = lookupNameInScope(scope, inPath); + if (node) + { + return node; + } + + // Try parent scope + scope = scope->m_parentScope; + } + } + + return nullptr; +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ScopeNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +ScopeNode* ScopeNode::getAnonymousNamespace() +{ + if (!m_anonymousNamespace) + { + m_anonymousNamespace = new ScopeNode(Kind::AnonymousNamespace); + m_anonymousNamespace->m_parentScope = this; + m_children.add(m_anonymousNamespace); + } + + return m_anonymousNamespace; +} + +void ScopeNode::addChildIgnoringName(Node* child) +{ + SLANG_ASSERT(child->m_parentScope == nullptr); + // Can't add anonymous namespace this way - should be added via getAnonymousNamespace + SLANG_ASSERT(child->m_kind != Kind::AnonymousNamespace); + + child->m_parentScope = this; + m_children.add(child); +} + +void ScopeNode::addChild(Node* child) +{ + addChildIgnoringName(child); + + if (child->m_name.hasContent()) + { + m_childMap.add(child->m_name.getContent(), child); + } +} + +Node* ScopeNode::findChild(const UnownedStringSlice& name) const +{ + Node* const* nodePtr = m_childMap.tryGetValue(name); + if (nodePtr) + { + return *nodePtr; + } + return nullptr; +} + +void ScopeNode::calcScopeDepthFirst(List& outNodes) +{ + outNodes.add(this); + for (Node* child : m_children) + { + child->calcScopeDepthFirst(outNodes); + } +} + +void ScopeNode::dump(int indentCount, StringBuilder& out) +{ + dumpMarkup(indentCount, out); + + _indent(indentCount, out); + + switch (m_kind) + { + case Kind::AnonymousNamespace: + { + out << "namespace {\n"; + } + case Kind::Namespace: + { + if (m_name.hasContent()) + { + out << "namespace " << m_name.getContent() << " {\n"; + } + else + { + out << "{\n"; + } + break; + } + } + + for (Node* child : m_children) + { + child->dump(indentCount + 1, out); + } + + _indent(indentCount, out); + out << "}\n"; +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! EnumCaseNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +/* Returns true if needs space between the tokens. +It determines this based on the locs, and if they contain something between them. +*/ +static bool _needsSpace(const Token& prevTok, const Token& tok) +{ + auto prevLoc = prevTok.getLoc(); + auto loc = tok.getLoc(); + + auto prevContent = prevTok.getContent(); + + if (prevLoc + prevContent.getLength() == loc) + { + return false; + } + + return true; +} + + +static void _dumpTokens(const Token* toks, Index count, StringBuilder& out) +{ + if (count > 0) + { + out << toks[0].getContent(); + + for (Index i = 1; i < count; ++i) + { + const auto& prevToken = toks[i - 1]; + const auto& token = toks[i]; + + if (_needsSpace(prevToken, token)) + { + out << " "; + } + + out << token.getContent(); + } + } +} + +static void _dumpTokens(const List& toks, StringBuilder& out) +{ + _dumpTokens(toks.getBuffer(), toks.getCount(), out); +} + + +void EnumCaseNode::dump(int indent, StringBuilder& out) +{ + if (isReflected()) + { + dumpMarkup(indent, out); + + _indent(indent, out); + out << m_name.getContent(); + + if (m_valueTokens.getCount()) + { + out << " = "; + _dumpTokens(m_valueTokens, out); + } + + out << ",\n"; + } +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! EnumNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +void TypeDefNode::dump(int indent, StringBuilder& out) +{ + if (isReflected()) + { + dumpMarkup(indent, out); + + _indent(indent, out); + + out << "typedef "; + _dumpTokens(m_targetTypeTokens, out); + out << " " << m_name.getContent() << ";\n"; + } +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! EnumNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +void EnumNode::dump(int indent, StringBuilder& out) +{ + if (!isReflected()) + { + return; + } + + dumpMarkup(indent, out); + + _indent(indent, out); + + out << "enum "; + + if (m_kind == Kind::EnumClass) + { + out << "class "; + } + + if (m_name.type != TokenType::Invalid) + { + out << m_name.getContent(); + } + + if (m_backingTokens.getCount() > 0) + { + out << " : "; + _dumpTokens(m_backingTokens, out); + } + + out << "\n"; + _indent(indent, out); + out << "{\n"; + + for (Node* child : m_children) + { + child->dump(indent + 1, out); + } + + _indent(indent, out); + out << "}\n"; +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!! CallableNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +void CallableNode::dump(int indent, StringBuilder& out) +{ + if (!isReflected()) + { + return; + } + + dumpMarkup(indent, out); + + _indent(indent, out); + + if (m_isStatic) + { + out << "static "; + } + if (m_isVirtual) + { + out << "virtual "; + } + + out << m_returnType << " "; + out << m_name.getContent() << "("; + + const Index count = m_params.getCount(); + for (Index i = 0; i < count; ++i) + { + if (i > 0) + { + out << ", "; + } + + const auto& param = m_params[i]; + out << param.m_type; + if (param.m_name.type == TokenType::Identifier) + { + out << " " << param.m_name.getContent(); + } + } + + out << ")"; + + if (m_isPure) + { + out << " = 0"; + } + + out << "\n"; +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! FieldNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +void FieldNode::dump(int indent, StringBuilder& out) +{ + if (!isReflected()) + { + return; + } + + dumpMarkup(indent, out); + + _indent(indent, out); + + if (m_isStatic) + { + out << "static "; + } + + out << m_fieldType << " " << m_name.getContent() << "\n"; +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ClassLikeNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +/// Add a node that is derived from this +void ClassLikeNode::addDerived(ClassLikeNode* derived) +{ + SLANG_ASSERT(derived->m_superNode == nullptr); + derived->m_superNode = this; + m_derivedTypes.add(derived); +} + +void ClassLikeNode::calcDerivedDepthFirst(List& outNodes) +{ + outNodes.add(this); + for (ClassLikeNode* derivedType : m_derivedTypes) + { + derivedType->calcDerivedDepthFirst(outNodes); + } +} + +void ClassLikeNode::dumpDerived(int indentCount, StringBuilder& out) +{ + if (isClassLike() && isReflected() && m_name.hasContent()) + { + _indent(indentCount, out); + out << m_name.getContent() << "\n"; + } + + for (ClassLikeNode* derivedType : m_derivedTypes) + { + derivedType->dumpDerived(indentCount + 1, out); + } +} + +Index ClassLikeNode::calcDerivedDepth() const +{ + const ClassLikeNode* node = this; + Index count = 0; + + while (node) + { + count++; + node = node->m_superNode; + } + + return count; +} + +ClassLikeNode* ClassLikeNode::findLastDerived() +{ + for (Index i = m_derivedTypes.getCount() - 1; i >= 0; --i) + { + ClassLikeNode* derivedType = m_derivedTypes[i]; + ClassLikeNode* found = derivedType->findLastDerived(); + if (found) + { + return found; + } + } + return this; +} + +bool ClassLikeNode::hasReflectedDerivedType() const +{ + for (ClassLikeNode* type : m_derivedTypes) + { + if (type->isReflected()) + { + return true; + } + } + return false; +} + +void ClassLikeNode::getReflectedDerivedTypes(List& out) const +{ + out.clear(); + for (ClassLikeNode* type : m_derivedTypes) + { + if (type->isReflected()) + { + out.add(type); + } + } +} + +void ClassLikeNode::dump(int indentCount, StringBuilder& out) +{ + dumpMarkup(indentCount, out); + + _indent(indentCount, out); + + const char* typeName = (m_kind == Kind::StructType) ? "struct" : "class"; + + out << typeName << " "; + + if (!isReflected()) + { + out << " ("; + } + out << m_name.getContent(); + if (!isReflected()) + { + out << ") "; + } + + if (m_super.hasContent()) + { + out << " : " << m_super.getContent(); + } + + out << " {\n"; + + for (Node* child : m_children) + { + child->dump(indentCount + 1, out); + } + + _indent(indentCount, out); + out << "}\n"; +} + +} // namespace CppParse diff --git a/tools/slang-cpp-parser/node.h b/tools/slang-cpp-parser/node.h new file mode 100644 index 000000000..4dad9473e --- /dev/null +++ b/tools/slang-cpp-parser/node.h @@ -0,0 +1,422 @@ +#pragma once + +#include "compiler-core/slang-doc-extractor.h" +#include "diagnostics.h" + +namespace CppParse +{ +using namespace Slang; + +enum class ReflectionType : uint8_t +{ + NotReflected, + Reflected, +}; + +// Pre-declare +class TypeSet; +class SourceOrigin; + +struct ScopeNode; + +class Node : public RefObject +{ +public: + enum class Kind : uint8_t + { + Invalid, + + StructType, + ClassType, + + Enum, + EnumClass, + + Namespace, + AnonymousNamespace, + + Field, + EnumCase, + + TypeDef, + + Callable, ///< Functions/methods + + Other, ///< Used 'other' parsing like for TYPE + Unknown, ///< Used for marking tokens consumed but usage is not known + + CountOf, + }; + + enum class KindRange + { + ScopeStart = int(Kind::StructType), + ScopeEnd = int(Kind::AnonymousNamespace), + + ClassLikeStart = int(Kind::StructType), + ClassLikeEnd = int(Kind::ClassType), + + ScopeTypeStart = int(Kind::StructType), + ScopeTypeEnd = int(Kind::EnumClass), + + OtherTypeStart = int(Kind::TypeDef), + OtherTypeEnd = int(Kind::TypeDef), + + EnumStart = int(Kind::Enum), + EnumEnd = int(Kind::EnumClass), + }; + + /// Returns true if kind can cast to this type + /// Used for implementing as casting + static bool isOfKind(Kind kind) + { + SLANG_UNUSED(kind); + return true; + } + + static bool isKindScope(Kind kind) + { + return int(kind) >= int(KindRange::ScopeStart) && int(kind) <= int(KindRange::ScopeEnd); + } + static bool isKindClassLike(Kind kind) + { + return int(kind) >= int(KindRange::ClassLikeStart) && + int(kind) <= int(KindRange::ClassLikeEnd); + } + static bool isKindEnumLike(Kind kind) + { + return int(kind) >= int(KindRange::EnumStart) && int(kind) <= int(KindRange::EnumEnd); + } + + /// It a type, but doesn't have a scope + static bool isKindOtherType(Kind kind) + { + return int(kind) >= int(KindRange::OtherTypeStart) && + int(kind) <= int(KindRange::OtherTypeEnd); + } + /// Is a type and has a scope + static bool isKindScopeType(Kind kind) + { + return int(kind) >= int(KindRange::ScopeTypeStart) && + int(kind) <= int(KindRange::ScopeTypeEnd); + } + + /// True if the kind is any type + static bool isKindType(Kind kind) { return isKindOtherType(kind) || isKindScopeType(kind); } + + /// True if the kind can accept contained types + static bool canKindContainTypes(Kind type) + { + switch (type) + { + case Kind::StructType: + case Kind::ClassType: + case Kind::Namespace: + case Kind::AnonymousNamespace: + { + return true; + } + default: + break; + } + return false; + } + + bool isNamespace() const { return m_kind == Kind::Namespace; } + bool isClassLike() const { return isKindClassLike(m_kind); } + bool isScope() const { return isKindScope(m_kind); } + bool isEnumLike() const { return isKindEnumLike(m_kind); } + + /// These are useful for the filter + static bool isClassLikeAndReflected(Node* node) + { + return node->isClassLike() && node->isReflected(); + } + static bool isClassLike(Node* node) { return isKindClassLike(node->m_kind); } + + virtual void dump(int indent, StringBuilder& out) = 0; + + /// Do depth first traversal of nodes in scopes + virtual void calcScopeDepthFirst(List& outNodes); + + /// Calculate the absolute name for this namespace/type + void calcAbsoluteName(StringBuilder& outName) const; + + /// Get the absolute name + String getAbsoluteName() const + { + StringBuilder buf; + calcAbsoluteName(buf); + return buf.produceString(); + } + + /// Calculate the scope path to this node, from the root + void calcScopePath(List& outPath) { calcScopePath(this, outPath); } + + /// True if reflected + bool isReflected() const { return m_reflectionType == ReflectionType::Reflected; } + + SourceLoc getSourceLoc() const { return m_name.getLoc(); } + + ScopeNode* getRootScope(); + + typedef bool (*Filter)(Node* node); + + template + static void filter(Filter filter, List& io) + { + const Node* _isNodeDerived = (T*)nullptr; + SLANG_UNUSED(_isNodeDerived); + filterImpl(filter, reinterpret_cast&>(io)); + } + + static void filterImpl(Filter filter, List& io); + + static void calcScopePath(Node* node, List& outPath); + + /// Lookup a name in just the specified scope + /// Handles anonymous namespaces, or name lookups that are in the parents space + static Node* lookupNameInScope(ScopeNode* scope, const UnownedStringSlice& name); + + /// Lookup from a path + static Node* lookupFromScope(ScopeNode* scope, const UnownedStringSlice* path, Index pathCount); + /// Looks up *just* from the specified scope. + static Node* lookupFromScope(ScopeNode* scope, const UnownedStringSlice& slice); + + /// Look up name (which can contain ::) + static Node* lookup(ScopeNode* scope, const UnownedStringSlice& name); + + static void splitPath(const UnownedStringSlice& slice, List& outSplitPath); + + /// If markup is specified dump it + void dumpMarkup(int indent, StringBuilder& out); + + Node(Kind type) + : m_kind(type), m_parentScope(nullptr), m_reflectionType(ReflectionType::NotReflected) + { + } + + Kind m_kind; ///< The kind of node this is + ReflectionType m_reflectionType; ///< Classes can be traversed, but not reflected. To be + ///< reflected they have to contain the marker + + MarkupVisibility m_markupVisibility = + MarkupVisibility::Public; ///< The visibility of the markup + String m_markup; ///< Documentation associated with this node + + Token m_name; ///< The name of this scope/type + + ScopeNode* m_parentScope; ///< The scope this type/scope is defined in +}; + +struct ScopeNode : public Node +{ + typedef Node Super; + + static bool isOfKind(Kind kind) { return isKindScope(kind); } + + virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; + virtual void calcScopeDepthFirst(List& outNodes) SLANG_OVERRIDE; + + /// True if can contain callable entries + bool canContainCallable() const { return isClassLike() || isNamespace(); } + + /// True if can accept fields (class like types can) + bool canContainFields() const { return isClassLike(); } + + /// True if the scope can accept types + bool canContainTypes() const { return canKindContainTypes(m_kind); } + + /// Gets the reflection for any contained types + ReflectionType getContainedReflectionType() const + { + return m_reflectionType == ReflectionType::NotReflected ? ReflectionType::NotReflected + : m_reflectionOverride; + } + + /// Add a child node to this nodes scope + void addChild(Node* child); + /// Adds the child but does not add the name to the map + void addChildIgnoringName(Node* child); + + /// Find a child node in this scope with the specified name. Return nullptr if not found + Node* findChild(const UnownedStringSlice& name) const; + + /// Gets the anonymous namespace associated with this scope + ScopeNode* getAnonymousNamespace(); + + ScopeNode(Kind kind) + : Super(kind) + , m_reflectionOverride(ReflectionType::Reflected) + , m_anonymousNamespace(nullptr) + { + } + + /// For child types, fields, how reflection is handled. If this type is not reflected + ReflectionType m_reflectionOverride; + + /// All of the types and namespaces in this *scope* + List> m_children; + + /// Map from a name (in this scope) to the Node + Dictionary m_childMap; + + /// There can only be one anonymousNamespace for a scope. If there is one it's held here + ScopeNode* m_anonymousNamespace; +}; + +struct FieldNode : public Node +{ + typedef Node Super; + + static bool isOfKind(Kind kind) { return kind == Kind::Field; } + + virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; + + FieldNode() + : Super(Kind::Field) + { + } + + UnownedStringSlice m_fieldType; + + bool m_isStatic = false; + + /// TODO(JS): We may want to add initializer tokens +}; + +struct ClassLikeNode : public ScopeNode +{ + typedef ScopeNode Super; + + static bool isOfKind(Kind kind) { return isKindClassLike(kind); } + + /// Add a node that is derived from this + void addDerived(ClassLikeNode* derived); + + /// Dump all of the derived types + void dumpDerived(int indentCount, StringBuilder& out); + + /// Calculates the derived depth + Index calcDerivedDepth() const; + + /// Find the last (reflected) derived type + ClassLikeNode* findLastDerived(); + + /// Traverse the hierarchy of derived nodes, in depth first order + void calcDerivedDepthFirst(List& outNodes); + + /// True if has a derived type that is reflected + bool hasReflectedDerivedType() const; + + /// Stores in out any reflected derived types + void getReflectedDerivedTypes(List& out) const; + + // Node Impl + virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; + + ClassLikeNode(Kind kind) + : Super(kind), m_origin(nullptr), m_typeSet(nullptr), m_superNode(nullptr) + { + SLANG_ASSERT(kind == Kind::ClassType || kind == Kind::StructType); + } + + SourceOrigin* m_origin; ///< Defines where this was uniquely defined. + + Token m_marker; ///< The marker associated with this scope (typically the marker is SLANG_CLASS + ///< etc, that is used to identify reflectedType) + + List> m_derivedTypes; ///< All of the types derived from this type + + TypeSet* m_typeSet; ///< The typeset this type belongs to. + + Token m_super; ///< Super class name + ClassLikeNode* m_superNode; ///< If this is a class/struct, the type it is derived from (or + ///< nullptr if base) +}; + +struct CallableNode : public Node +{ + typedef Node Super; + + static bool isOfKind(Kind kind) { return kind == Kind::Callable; } + + virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; + + CallableNode() + : Super(Kind::Callable) + { + } + + struct Param + { + UnownedStringSlice m_type; + Token m_name; + }; + + UnownedStringSlice m_returnType; + + CallableNode* m_nextOverload = nullptr; + + List m_params; + + bool m_isStatic = false; + bool m_isVirtual = false; + bool m_isPure = false; +}; + +struct EnumCaseNode : public Node +{ + typedef Node Super; + + static bool isOfKind(Kind kind) { return kind == Kind::EnumCase; } + + virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; + + EnumCaseNode() + : Super(Kind::EnumCase) + { + } + + // Tokens that make up the value. If not defined will be empty + List m_valueTokens; +}; + +struct EnumNode : public ScopeNode +{ + typedef ScopeNode Super; + static bool isOfKind(Kind kind) { return isKindEnumLike(kind); } + + virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; + + EnumNode(Kind kind) + : Super(kind) + { + SLANG_ASSERT(isKindEnumLike(kind)); + } + + List m_backingTokens; +}; + +struct TypeDefNode : public Node +{ + typedef Node Super; + static bool isOfKind(Kind kind) { return kind == Kind::TypeDef; } + + virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; + + TypeDefNode() + : Super(Kind::TypeDef) + { + } + + List m_targetTypeTokens; +}; + +template +T* as(Node* node) +{ + return (node && T::isOfKind(node->m_kind)) ? static_cast(node) : nullptr; +} + +} // namespace CppParse diff --git a/tools/slang-cpp-parser/options.cpp b/tools/slang-cpp-parser/options.cpp new file mode 100644 index 000000000..8be319e7b --- /dev/null +++ b/tools/slang-cpp-parser/options.cpp @@ -0,0 +1,153 @@ +#include "options.h" + +#include "diagnostics.h" + +namespace CppParse +{ + +SlangResult OptionsParser::_parseArgFlag(const char* option, bool& outFlag) +{ + SLANG_ASSERT(UnownedStringSlice(m_args[m_index]) == option); + SLANG_ASSERT(m_index < m_argCount); + + m_index++; + outFlag = true; + return SLANG_OK; +} + +SlangResult OptionsParser::_parseArgWithValue(const char* option, String& ioValue) +{ + SLANG_ASSERT(UnownedStringSlice(m_args[m_index]) == option); + if (m_index + 1 < m_argCount) + { + // Next parameter is the output path, there can only be one + if (ioValue.getLength()) + { + // There already is output + m_sink->diagnose(SourceLoc(), CPPDiagnostics::optionAlreadyDefined, option, ioValue); + return SLANG_FAIL; + } + } + else + { + m_sink->diagnose(SourceLoc(), CPPDiagnostics::requireValueAfterOption, option); + return SLANG_FAIL; + } + + ioValue = m_args[m_index + 1]; + m_index += 2; + return SLANG_OK; +} + +SlangResult OptionsParser::_parseArgReplaceValue(const char* option, String& ioValue) +{ + SLANG_ASSERT(UnownedStringSlice(m_args[m_index]) == option); + if (m_index + 1 >= m_argCount) + { + m_sink->diagnose(SourceLoc(), CPPDiagnostics::requireValueAfterOption, option); + return SLANG_FAIL; + } + + ioValue = m_args[m_index + 1]; + m_index += 2; + return SLANG_OK; +} + +SlangResult OptionsParser::parse( + int argc, + const char* const* argv, + DiagnosticSink* sink, + Options& outOptions) +{ + outOptions.reset(); + + m_index = 0; + m_argCount = argc; + m_args = argv; + m_sink = sink; + + outOptions.reset(); + + while (m_index < m_argCount) + { + const UnownedStringSlice arg = UnownedStringSlice(argv[m_index]); + + if (arg.getLength() > 0 && arg[0] == '-') + { + if (arg == "-d") + { + SLANG_RETURN_ON_FAIL(_parseArgWithValue("-d", outOptions.m_inputDirectory)); + continue; + } + else if (arg == "-o") + { + SLANG_RETURN_ON_FAIL(_parseArgWithValue("-o", outOptions.m_outputPath)); + continue; + } + else if (arg == "-dump") + { + SLANG_RETURN_ON_FAIL(_parseArgFlag("-dump", outOptions.m_dump)); + continue; + } + else if (arg == "-mark-prefix") + { + SLANG_RETURN_ON_FAIL( + _parseArgReplaceValue("-mark-prefix", outOptions.m_markPrefix)); + continue; + } + else if (arg == "-mark-suffix") + { + SLANG_RETURN_ON_FAIL( + _parseArgReplaceValue("-mark-suffix", outOptions.m_markSuffix)); + continue; + } + else if (arg == "-defs") + { + SLANG_RETURN_ON_FAIL(_parseArgFlag("-defs", outOptions.m_defs)); + continue; + } + else if (arg == "-output-fields") + { + SLANG_RETURN_ON_FAIL(_parseArgFlag("-output-fields", outOptions.m_outputFields)); + continue; + } + else if (arg == "-strip-prefix") + { + SLANG_RETURN_ON_FAIL( + _parseArgWithValue("-strip-prefix", outOptions.m_stripFilePrefix)); + continue; + } + else if (arg == "-unit-test") + { + SLANG_RETURN_ON_FAIL(_parseArgFlag("-unit-test", outOptions.m_runUnitTests)); + continue; + } + else if (arg == "-unmarked") + { + bool unmarked; + SLANG_RETURN_ON_FAIL(_parseArgFlag("-unmarked", unmarked)); + outOptions.m_requireMark = !unmarked; + continue; + } + + m_sink->diagnose(SourceLoc(), CPPDiagnostics::unknownOption, arg); + return SLANG_FAIL; + } + else + { + // If it doesn't start with - then it's assumed to be an input path + outOptions.m_inputPaths.add(arg); + m_index++; + } + } + + if (outOptions.m_inputPaths.getCount() < 0) + { + m_sink->diagnose(SourceLoc(), CPPDiagnostics::noInputPathsSpecified); + return SLANG_FAIL; + } + + return SLANG_OK; +} + +} // namespace CppParse diff --git a/tools/slang-cpp-parser/options.h b/tools/slang-cpp-parser/options.h new file mode 100644 index 000000000..c1cbb1a27 --- /dev/null +++ b/tools/slang-cpp-parser/options.h @@ -0,0 +1,63 @@ +#pragma once + +#include "slang/slang-diagnostics.h" + +namespace CppParse +{ +using namespace Slang; + + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Options !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +struct Options +{ + void reset() { *this = Options(); } + + Options() + { + m_markPrefix = "SLANG_"; + m_markSuffix = "_CLASS"; + } + + bool m_defs = false; ///< If set will output a '-defs.h' file for each of the input files, that + ///< corresponds to previous defs files (although doesn't have fields/RAW) + bool m_dump = + false; ///< If true will dump to stderr the types/fields and hierarchy it extracted + bool m_runUnitTests = false; ///< If true will run internal unit tests + bool m_extractDoc = true; ///< If set will try to extract documentation associated with nodes + + bool m_outputFields = false; ///< When dumping macros also dump field definitions + bool m_requireMark = true; + + List m_inputPaths; ///< The input paths to the files to be processed + + String m_outputPath; ///< The output path. Note that the extractor can generate multiple output + ///< files, and this will actually be the 'stem' of several files + + String m_inputDirectory; ///< The input directory that is by default used for reading + ///< m_inputPaths from. + String m_markPrefix; ///< The prefix of the 'marker' used to identify a reflected type + String m_markSuffix; ///< The postfix of the 'marker' used to identify a reflected type + String m_stripFilePrefix; ///< Used for the 'origin' information, this is stripped from the + ///< source filename, and the remainder of the filename (without + ///< extension) is 'macroized' +}; + +struct OptionsParser +{ + /// Parse the parameters. NOTE! Must have the program path removed + SlangResult parse(int argc, const char* const* argv, DiagnosticSink* sink, Options& outOptions); + + SlangResult _parseArgWithValue(const char* option, String& outValue); + SlangResult _parseArgReplaceValue(const char* option, String& outValue); + SlangResult _parseArgFlag(const char* option, bool& outFlag); + + String m_reflectType; + + Index m_index; + Int m_argCount; + const char* const* m_args; + DiagnosticSink* m_sink; +}; + +} // namespace CppParse diff --git a/tools/slang-cpp-parser/parser.cpp b/tools/slang-cpp-parser/parser.cpp new file mode 100644 index 000000000..1edc38ac7 --- /dev/null +++ b/tools/slang-cpp-parser/parser.cpp @@ -0,0 +1,2261 @@ +#include "parser.h" + +#include "compiler-core/slang-name-convention-util.h" +#include "core/slang-io.h" +#include "core/slang-string-util.h" +#include "identifier-lookup.h" +#include "options.h" + +namespace CppParse +{ +using namespace Slang; + +// If fails then we need more bits to identify types +SLANG_COMPILE_TIME_ASSERT(int(Node::Kind::CountOf) <= 8 * sizeof(uint32_t)); + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Parser !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +Parser::Parser(NodeTree* nodeTree, DiagnosticSink* sink) + : m_sink(sink), m_nodeTree(nodeTree), m_nodeTypeEnabled(0) +{ + // Enable types by default + const Node::Kind defaultEnabled[] = { + Node::Kind::ClassType, + Node::Kind::StructType, + Node::Kind::Namespace, + Node::Kind::AnonymousNamespace, + Node::Kind::Field, + + // These are disabled by default because AST uses macro magic to build up the types + // Node::Type::TypeDef, + // Node::Type::Enum, + // Node::Type::EnumClass, + + Node::Kind::Callable, + }; + setKindsEnabled(defaultEnabled, SLANG_COUNT_OF(defaultEnabled)); +} + +void Parser::setKindEnabled(Node::Kind kind, bool isEnabled) +{ + if (isEnabled) + { + m_nodeTypeEnabled |= (NodeTypeBitType(1) << int(kind)); + } + else + { + m_nodeTypeEnabled &= ~(NodeTypeBitType(1) << int(kind)); + } +} + +void Parser::setKindsEnabled(const Node::Kind* kinds, Index kindsCount, bool isEnabled) +{ + for (Index i = 0; i < kindsCount; ++i) + { + setKindEnabled(kinds[i], isEnabled); + } +} + +bool Parser::_isMarker(const UnownedStringSlice& name) +{ + return name.startsWith(m_options->m_markPrefix.getUnownedSlice()) && + name.endsWith(m_options->m_markSuffix.getUnownedSlice()); +} + +SlangResult Parser::expect(TokenType type, Token* outToken) +{ + if (m_reader.peekTokenType() != type) + { + m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::expectingToken, type); + return SLANG_FAIL; + } + + if (outToken) + { + *outToken = m_reader.advanceToken(); + } + else + { + m_reader.advanceToken(); + } + return SLANG_OK; +} + +bool Parser::advanceIfToken(TokenType type, Token* outToken) +{ + if (m_reader.peekTokenType() == type) + { + Token token = m_reader.advanceToken(); + if (outToken) + { + *outToken = token; + } + return true; + } + return false; +} + +bool Parser::advanceIfMarker(Token* outToken) +{ + const Token peekToken = m_reader.peekToken(); + if (peekToken.type == TokenType::Identifier && _isMarker(peekToken.getContent())) + { + m_reader.advanceToken(); + if (outToken) + { + *outToken = peekToken; + } + return true; + } + return false; +} + +bool Parser::advanceIfStyle(IdentifierStyle style, Token* outToken) +{ + if (m_reader.peekTokenType() == TokenType::Identifier) + { + IdentifierStyle readStyle = + m_nodeTree->m_identifierLookup->get(m_reader.peekToken().getContent()); + if (readStyle == style) + { + Token token = m_reader.advanceToken(); + if (outToken) + { + *outToken = token; + } + return true; + } + } + return false; +} + + +SlangResult Parser::pushAnonymousNamespace() +{ + m_currentScope = m_currentScope->getAnonymousNamespace(); + + if (m_sourceOrigin) + { + m_sourceOrigin->addNode(m_currentScope); + } + + // Add the to the scope stack so can pop. + m_scopeStack.add(m_currentScope); + + return SLANG_OK; +} + +SlangResult Parser::pushScope(ScopeNode* scopeNode) +{ + // We can only have one 'special' scope. + SLANG_ASSERT(scopeNode || m_scopeStack.getLast()); + + // We keep to track. + m_scopeStack.add(scopeNode); + + // If we pass nullptr, we don't update the current scope. + if (scopeNode == nullptr) + { + return SLANG_OK; + } + + if (m_sourceOrigin) + { + m_sourceOrigin->addNode(scopeNode); + } + + if (scopeNode->m_name.hasContent()) + { + // For anonymous namespace, we should look if we already have one and just reopen that. + // Doing so will mean will find anonymous namespace clashes + + if (Node* foundNode = m_currentScope->findChild(scopeNode->m_name.getContent())) + { + if (scopeNode->isClassLike()) + { + m_sink->diagnose( + m_reader.peekToken(), + CPPDiagnostics::typeAlreadyDeclared, + scopeNode->m_name.getContent()); + m_sink->diagnose( + foundNode->m_name, + CPPDiagnostics::seeDeclarationOf, + scopeNode->m_name.getContent()); + return SLANG_FAIL; + } + + if (foundNode->m_kind == Node::Kind::Namespace) + { + if (foundNode->m_kind != scopeNode->m_kind) + { + // Different types can't work + m_sink->diagnose( + m_reader.peekToken(), + CPPDiagnostics::typeAlreadyDeclared, + scopeNode->m_name.getContent()); + return SLANG_FAIL; + } + + ScopeNode* foundScopeNode = as(foundNode); + SLANG_ASSERT(foundScopeNode); + + // Make sure the node is empty, as we are *not* going to add it, we are just going + // to use the pre-existing namespace + SLANG_ASSERT(scopeNode->m_children.getCount() == 0); + + // We can just use the pre-existing namespace + m_currentScope = foundScopeNode; + return SLANG_OK; + } + } + } + + m_currentScope->addChild(scopeNode); + m_currentScope = scopeNode; + return SLANG_OK; +} + +SlangResult Parser::popScope() +{ + if (m_scopeStack.getCount() <= 0) + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::scopeNotClosed); + return SLANG_FAIL; + } + + ScopeNode* topScope = m_scopeStack.getLast(); + m_scopeStack.removeLast(); + + // If the top is nullptr, we don't change the current scope + if (topScope == nullptr) + { + return SLANG_OK; + } + + m_currentScope = m_currentScope->m_parentScope; + return SLANG_OK; +} + +SlangResult Parser::_maybeConsumeScope() +{ + // Look for either ; or { to open scope + while (true) + { + const TokenType type = m_reader.peekTokenType(); + if (type == TokenType::Semicolon) + { + m_reader.advanceToken(); + return SLANG_OK; + } + else if (type == TokenType::LBrace) + { + // m_reader.advanceToken(); + return consumeToClosingBrace(); + } + else if (type == TokenType::EndOfFile) + { + return SLANG_OK; + } + + m_reader.advanceToken(); + } +} + +SlangResult Parser::consumeToClosingBrace(const Token* inOpenBraceToken) +{ + Token openToken; + if (inOpenBraceToken) + { + openToken = *inOpenBraceToken; + } + else + { + openToken = m_reader.advanceToken(); + } + SLANG_ASSERT(openToken.type == TokenType::LBrace); + + while (true) + { + switch (m_reader.peekTokenType()) + { + case TokenType::EndOfFile: + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::didntFindMatchingBrace); + m_sink->diagnose(openToken, CPPDiagnostics::seeOpen); + return SLANG_FAIL; + } + case TokenType::LBrace: + { + SLANG_RETURN_ON_FAIL(consumeToClosingBrace()); + break; + } + case TokenType::RBrace: + { + m_reader.advanceToken(); + return SLANG_OK; + } + default: + { + m_reader.advanceToken(); + break; + } + } + } +} + + +SlangResult Parser::_parseEnum() +{ + // We are looking for + // enum ([class name] | [name]) [: base] ( { | ; ) + + Token enumToken; + + // consume enum + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &enumToken)); + + if (!m_currentScope->canContainTypes()) + { + m_sink->diagnose(enumToken.loc, CPPDiagnostics::cannotDeclareTypeInScope); + return SLANG_FAIL; + } + + Node::Kind kind = Node::Kind::Enum; + + Token nameToken; + if (advanceIfToken(TokenType::Identifier, &nameToken)) + { + const IdentifierStyle style = m_nodeTree->m_identifierLookup->get(nameToken.getContent()); + + if (style == IdentifierStyle::Class) + { + kind = Node::Kind::EnumClass; + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &nameToken)); + } + else if (style == IdentifierStyle::None) + { + // It holds the name then + } + else + { + m_sink->diagnose( + nameToken.loc, + CPPDiagnostics::expectingIdentifier, + nameToken.getContent()); + return SLANG_FAIL; + } + } + + RefPtr node = new EnumNode(kind); + node->m_name = nameToken; + node->m_reflectionType = m_currentScope->getContainedReflectionType(); + + if (advanceIfToken(TokenType::Colon)) + { + // We may have tokens up to { or ; + List backingTokens; + + while (true) + { + TokenType tokenType = m_reader.peekTokenType(); + if (tokenType == TokenType::Semicolon || tokenType == TokenType::LBrace || + tokenType == TokenType::EndOfFile) + { + break; + } + + backingTokens.add(m_reader.advanceToken()); + } + + // TODO - Look up the backing type. It can only be an integral. We can assume it must be + // defined before lookup for our uses here. If we can't find the type, we could assume it's + // size is undefined + + if (backingTokens.getCount() > 0) + { + node->m_backingTokens.swapWith(backingTokens); + } + } + + pushScope(node); + + if (advanceIfToken(TokenType::Semicolon)) + { + if (nameToken.type != TokenType::Invalid) + { + Node* node = m_currentScope->findChild(nameToken.getContent()); + if (node) + { + // Strictly speaking we should check the backing type etc, match, but for now ignore + // and assume it's ok + + if (node->m_kind == kind) + { + return SLANG_OK; + } + m_sink->diagnose( + nameToken.loc, + CPPDiagnostics::typeAlreadyDeclared, + nameToken.getContent()); + return SLANG_FAIL; + } + return popScope(); + } + } + + SLANG_RETURN_ON_FAIL(expect(TokenType::LBrace)); + + while (true) + { + TokenType tokenType = m_reader.peekTokenType(); + if (tokenType == TokenType::RBrace) + { + break; + } + + RefPtr caseNode(new EnumCaseNode); + + // We could also check if the name is a valid identifier for name, for now just assume. + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &caseNode->m_name)); + + if (node->findChild(caseNode->m_name.getContent())) + { + m_sink->diagnose( + caseNode->m_name.loc, + CPPDiagnostics::identifierAlreadyDefined, + caseNode->m_name.getContent()); + return SLANG_FAIL; + } + + caseNode->m_reflectionType = m_currentScope->getContainedReflectionType(); + + // Add the value + node->addChild(caseNode); + + if (advanceIfToken(TokenType::OpAssign)) + { + List valueTokens; + SLANG_RETURN_ON_FAIL(_parseExpression(valueTokens)); + + if (valueTokens.getCount() > 0) + { + caseNode->m_valueTokens.swapWith(valueTokens); + } + } + + tokenType = m_reader.peekTokenType(); + if (tokenType == TokenType::Comma) + { + m_reader.advanceToken(); + continue; + } + + break; + } + + SLANG_RETURN_ON_FAIL(expect(TokenType::RBrace)); + SLANG_RETURN_ON_FAIL(expect(TokenType::Semicolon)); + + return popScope(); +} + +SlangResult Parser::_consumeTemplate() +{ + // Skip the current 'template' token. + m_reader.advanceToken(); + + // Consume everything in <> + SLANG_RETURN_ON_FAIL(expect(TokenType::OpLess)); + + { + Index arrowCount = 1; + while (true) + { + auto tokenType = m_reader.peekTokenType(); + + if (tokenType == TokenType::OpLess) + { + m_reader.advanceToken(); + arrowCount++; + } + else if (tokenType == TokenType::OpGreater) + { + m_reader.advanceToken(); + if (arrowCount == 1) + { + break; + } + --arrowCount; + } + else if (tokenType == TokenType::OpRsh) + { + if (arrowCount < 2) + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::unexpectedTemplateClose); + return SLANG_FAIL; + } + m_reader.advanceToken(); + if (arrowCount == 2) + { + break; + } + arrowCount -= 2; + } + else if (tokenType == TokenType::EndOfFile) + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::unexpectedEndOfFile); + return SLANG_FAIL; + } + else + { + m_reader.advanceToken(); + } + } + } + + // Search for { or ; to consume remaining + while (true) + { + auto tokenType = m_reader.peekTokenType(); + + switch (tokenType) + { + case TokenType::EndOfFile: + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::unexpectedEndOfFile); + return SLANG_FAIL; + } + case TokenType::Semicolon: + { + // Ends with semicolon if it's a template pre-declaration + m_reader.advanceToken(); + return SLANG_OK; + } + case TokenType::LBrace: + { + // If ends with {, means could be body of a struct/class or a body of a + // function/method. Consume it + SLANG_RETURN_ON_FAIL(consumeToClosingBrace()); + // If we hit a ; just consume and ignore + advanceIfToken(TokenType::Semicolon); + return SLANG_OK; + } + default: + { + // Consume + m_reader.advanceToken(); + break; + } + } + } +} + +SlangResult Parser::_maybeParseNode(Node::Kind kind) +{ + // We are looking for + // struct/class identifier [: [public|private|protected] Identifier ] { + // [public|private|proctected:]* marker ( identifier ); + + if (kind == Node::Kind::Namespace) + { + // consume namespace + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier)); + + Token name; + if (advanceIfToken(TokenType::LBrace)) + { + return pushAnonymousNamespace(); + } + else if (advanceIfToken(TokenType::Identifier, &name)) + { + if (advanceIfToken(TokenType::LBrace)) + { + // Okay looks like we are opening a namespace + RefPtr node(new ScopeNode(Node::Kind::Namespace)); + node->m_name = name; + + node->m_reflectionType = m_currentScope->getContainedReflectionType(); + // Push the node + return pushScope(node); + } + } + + // Just ignore it then + return SLANG_OK; + } + else if (Node::isKindEnumLike(kind)) + { + return _parseEnum(); + } + + // Must be class | struct + + SLANG_ASSERT(kind == Node::Kind::ClassType || kind == Node::Kind::StructType); + + Token name; + + // consume class | struct + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier)); + // Next is the class name + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &name)); + + + if (m_reader.peekTokenType() == TokenType::Semicolon) + { + // pre declaration; + return SLANG_OK; + } + + RefPtr node(new ClassLikeNode(kind)); + node->m_name = name; + + // We default to the containing scope for reflection type. + if (!m_options->m_requireMark) + { + node->m_reflectionType = m_currentScope->getContainedReflectionType(); + } + else + { + // Defaults to not reflected + SLANG_ASSERT(!node->isReflected()); + } + + if (advanceIfToken(TokenType::Colon)) + { + // Could have public + advanceIfStyle(IdentifierStyle::Access); + + if (!advanceIfToken(TokenType::Identifier, &node->m_super)) + { + return SLANG_OK; + } + } + + // We only accept a single super class. Consume everything afterwards until we hit the { brace + + if (m_reader.peekTokenType() != TokenType::LBrace) + { + // Consume up until we see a brace else it's an error + while (true) + { + const TokenType peekTokenType = m_reader.peekTokenType(); + if (peekTokenType == TokenType::EndOfFile) + { + // Expecting brace + m_sink->diagnose( + m_reader.peekToken(), + CPPDiagnostics::expectingToken, + TokenType::LBrace); + return SLANG_FAIL; + } + else if (peekTokenType == TokenType::LBrace) + { + break; + } + m_reader.advanceToken(); + } + + return pushScope(node); + } + + const Token braceToken = m_reader.advanceToken(); + + // Push the class scope + return pushScope(node); +} + +SlangResult Parser::_consumeToSync() +{ + while (true) + { + TokenType type = m_reader.peekTokenType(); + + switch (type) + { + case TokenType::Semicolon: + { + m_reader.advanceToken(); + return SLANG_OK; + } + case TokenType::Pound: + case TokenType::EndOfFile: + case TokenType::LBrace: + case TokenType::RBrace: + { + return SLANG_OK; + } + } + + m_reader.advanceToken(); + } +} + +SlangResult Parser::_maybeParseTemplateArg(Index& ioTemplateDepth) +{ + switch (m_reader.peekTokenType()) + { + case TokenType::Identifier: + { + TokenReader::ParsingCursor nameCursor; + SLANG_RETURN_ON_FAIL(_maybeParseType(ioTemplateDepth, nameCursor)); + return SLANG_OK; + } + case TokenType::IntegerLiteral: + { + m_reader.advanceToken(); + return SLANG_OK; + } + default: + break; + } + return SLANG_FAIL; +} + +SlangResult Parser::_maybeParseTemplateArgs(Index& ioTemplateDepth) +{ + if (!advanceIfToken(TokenType::OpLess)) + { + return SLANG_FAIL; + } + + ioTemplateDepth++; + + while (true) + { + if (ioTemplateDepth == 0) + { + return SLANG_OK; + } + + switch (m_reader.peekTokenType()) + { + case TokenType::OpGreater: + { + if (ioTemplateDepth <= 0) + { + m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::unexpectedTemplateClose); + return SLANG_FAIL; + } + ioTemplateDepth--; + m_reader.advanceToken(); + return SLANG_OK; + } + case TokenType::OpRsh: + { + if (ioTemplateDepth <= 1) + { + m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::unexpectedTemplateClose); + return SLANG_FAIL; + } + ioTemplateDepth -= 2; + m_reader.advanceToken(); + return SLANG_OK; + } + default: + { + while (true) + { + SLANG_RETURN_ON_FAIL(_maybeParseTemplateArg(ioTemplateDepth)); + + if (m_reader.peekTokenType() == TokenType::Comma) + { + m_reader.advanceToken(); + // If there is a comma parse another arg + continue; + } + break; + } + break; + } + } + } +} + +SlangResult Parser::_maybeConsume(IdentifierStyle style) +{ + while (advanceIfStyle(style)) + ; + return SLANG_OK; +} + +// True if two of these token types of the same type placed immediately after one another +// produce a different token. Can be conservative, as if not strictly required +// it will just mean more spacing in the output +static bool _canRepeatTokenType(TokenType type) +{ + switch (type) + { + case TokenType::OpAdd: + case TokenType::OpSub: + case TokenType::OpAnd: + case TokenType::OpOr: + case TokenType::OpGreater: + case TokenType::OpLess: + case TokenType::Identifier: + case TokenType::OpAssign: + case TokenType::Colon: + { + return false; + } + default: + break; + } + return true; +} + +// Returns true if there needs to be a space between the previous token type, and the current token +// type for correct output. It is assumed that the token stream is appropriate. +// The implementation might need more sophistication, but this at least avoids Blah const * -> +// Blahconst* +static bool _tokenConcatNeedsSpace(TokenType prev, TokenType cur) +{ + if ((cur == TokenType::OpAssign) || (prev == cur && !_canRepeatTokenType(cur))) + { + return true; + } + return false; +} + +void Parser::_getTypeTokens( + TokenReader::ParsingCursor start, + TokenReader::ParsingCursor nameCursor, + List& outToks) +{ + auto endCursor = m_reader.getCursor(); + m_reader.setCursor(start); + + while (!m_reader.isAtCursor(endCursor)) + { + if (m_reader.getCursor() == nameCursor) + { + m_reader.advanceToken(); + } + else + { + outToks.add(m_reader.advanceToken()); + } + } +} + +UnownedStringSlice Parser::_concatType( + TokenReader::ParsingCursor start, + TokenReader::ParsingCursor nameCursor) +{ + List toks; + _getTypeTokens(start, nameCursor, toks); + return _concatTokens(toks.getBuffer(), toks.getCount()); +} + +UnownedStringSlice Parser::_concatTokens(const Token* toks, Index toksCount) +{ + StringBuilder buf; + + TokenType prevTokenType = TokenType::Unknown; + for (Index i = 0; i < toksCount; ++i) + { + const auto token = toks[i]; + + // Check if we need a space between tokens + if (_tokenConcatNeedsSpace(prevTokenType, token.type)) + { + buf << " "; + } + + buf << token.getContent(); + + prevTokenType = token.type; + } + + StringSlicePool* typePool = m_nodeTree->m_typePool; + return typePool->getSlice(typePool->add(buf)); +} + +UnownedStringSlice Parser::_concatTokens(TokenReader::ParsingCursor start) +{ + auto endCursor = m_reader.getCursor(); + + m_reader.setCursor(start); + + TokenType prevTokenType = TokenType::Unknown; + + StringBuilder buf; + while (!m_reader.isAtCursor(endCursor)) + { + const Token token = m_reader.advanceToken(); + // Check if we need a space between tokens + if (_tokenConcatNeedsSpace(prevTokenType, token.type)) + { + buf << " "; + } + buf << token.getContent(); + + prevTokenType = token.type; + } + + StringSlicePool* typePool = m_nodeTree->m_typePool; + return typePool->getSlice(typePool->add(buf)); +} + +SlangResult Parser::_maybeParseType( + Index& ioTemplateDepth, + TokenReader::ParsingCursor& outNameCursor) +{ + outNameCursor = TokenReader::ParsingCursor(); + + while (true) + { + if (m_reader.peekTokenType() == TokenType::Identifier) + { + const IdentifierStyle style = + m_nodeTree->m_identifierLookup->get(m_reader.peekToken().getContent()); + + if (style == IdentifierStyle::TypeModifier || + style == IdentifierStyle::IntegerModifier || style == IdentifierStyle::Class || + style == IdentifierStyle::Struct) + { + // These are ok keywords in this context + } + else if (hasFlag(style, IdentifierFlag::Keyword)) + { + return SLANG_FAIL; + } + } + + _maybeConsume(IdentifierStyle::TypeModifier); + + if (advanceIfStyle(IdentifierStyle::IntegerModifier)) + { + // Consume the integer typename (if there is one) + const Token peekToken = m_reader.peekToken(); + if (peekToken.type == TokenType::Identifier) + { + const IdentifierStyle style = + m_nodeTree->m_identifierLookup->get(peekToken.getContent()); + if (style == IdentifierStyle::IntegerType) + { + m_reader.advanceToken(); + } + } + break; + } + + advanceIfToken(TokenType::Scope); + while (true) + { + // if we have a struct/class prefix in front of a name just consume it. + if (m_reader.peekTokenType() == TokenType::Identifier) + { + const IdentifierStyle style = + m_nodeTree->m_identifierLookup->get(m_reader.peekToken().getContent()); + if (style == IdentifierStyle::Class || style == IdentifierStyle::Struct) + { + m_reader.advanceToken(); + } + } + + Token identifierToken; + if (!advanceIfToken(TokenType::Identifier, &identifierToken)) + { + return SLANG_FAIL; + } + + const IdentifierStyle style = + m_nodeTree->m_identifierLookup->get(identifierToken.getContent()); + if (hasFlag(style, IdentifierFlag::Keyword)) + { + return SLANG_FAIL; + } + + if (advanceIfToken(TokenType::Scope)) + { + continue; + } + break; + } + + if (m_reader.peekTokenType() == TokenType::OpLess) + { + SLANG_RETURN_ON_FAIL(_maybeParseTemplateArgs(ioTemplateDepth)); + } + + if (m_reader.peekTokenType() == TokenType::Scope) + { + // Skip the scope and repeat + m_reader.advanceToken(); + continue; + } + + break; + } + + // Strip all the consts etc modifiers + _maybeConsume(IdentifierStyle::TypeModifier); + + // It's a reference and we are done + if (advanceIfToken(TokenType::OpBitAnd)) + { + return SLANG_OK; + } + + while (true) + { + if (advanceIfToken(TokenType::OpMul)) + { + // Strip all the consts + _maybeConsume(IdentifierStyle::TypeModifier); + continue; + } + break; + } + + if (advanceIfToken(TokenType::LParent)) + { + // TODO(JS): + // Doesn't handle all the modifiers just (*SomeName) + + SLANG_RETURN_ON_FAIL(expect(TokenType::OpMul)); + outNameCursor = m_reader.getCursor(); + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier)); + + SLANG_RETURN_ON_FAIL(expect(TokenType::RParent)); + + // We need to parse and add the params + if (m_reader.peekTokenType() != TokenType::LParent) + { + m_sink->diagnose( + m_reader.peekToken(), + CPPDiagnostics::expectingToken, + TokenType::LParent); + return SLANG_FAIL; + } + + // Consume the params + SLANG_RETURN_ON_FAIL(_consumeBalancedParens()); + } + else if (m_reader.peekTokenType() == TokenType::Identifier) + { + auto potentialNameCursor = m_reader.getCursor(); + m_reader.advanceToken(); + if (m_reader.peekTokenType() == TokenType::LBracket) + { + outNameCursor = potentialNameCursor; + while (advanceIfToken(TokenType::LBracket)) + { + List exprToks; + SLANG_RETURN_ON_FAIL(_parseExpression(exprToks)); + SLANG_RETURN_ON_FAIL(expect(TokenType::RBracket)); + } + } + else + { + // Wasn't an array type..., so rewind + m_reader.setCursor(potentialNameCursor); + } + } + + return SLANG_OK; +} + +SlangResult Parser::_maybeParseType(List& outToks, Token& outName) +{ + // Set to unknown + outName = Token(); + + auto startCursor = m_reader.getCursor(); + + TokenReader::ParsingCursor nameCursor; + + Index templateDepth = 0; + SlangResult res = _maybeParseType(templateDepth, nameCursor); + if (SLANG_FAILED(res) && m_sink->getErrorCount()) + { + return res; + } + + if (templateDepth != 0) + { + m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::unexpectedTemplateClose); + return SLANG_FAIL; + } + + auto endCursor = m_reader.getCursor(); + m_reader.setCursor(startCursor); + + if (nameCursor.isValid()) + { + while (!m_reader.isAtCursor(endCursor)) + { + if (m_reader.getCursor() == nameCursor) + { + outName = m_reader.advanceToken(); + } + else + { + outToks.add(m_reader.advanceToken()); + } + } + } + else + { + while (!m_reader.isAtCursor(endCursor)) + { + outToks.add(m_reader.advanceToken()); + } + } + + return SLANG_OK; +} + +SlangResult Parser::_parseSpecialMacro() +{ + Token name; + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &name)); + + List params; + + if (m_reader.peekTokenType() == TokenType::LParent) + { + // Mark the start + auto startCursor = m_reader.getCursor(); + + // Consume the params + SLANG_RETURN_ON_FAIL(_consumeBalancedParens()); + + auto endCursor = m_reader.getCursor(); + m_reader.setCursor(startCursor); + + while (!m_reader.isAtCursor(endCursor)) + { + params.add(m_reader.advanceToken()); + } + } + + // Can do special handling here + const UnownedStringSlice suffix = name.getContent().tail(m_options->m_markPrefix.getLength()); + + if (suffix == "COM_INTERFACE") + { + // TODO(JS): It's a com interface. Extact the GUID + } + + return SLANG_OK; +} + +SlangResult Parser::_parseMarker() +{ + SLANG_ASSERT( + m_reader.peekTokenType() == TokenType::Identifier && + _isMarker(m_reader.peekToken().getContent()) && m_currentScope->isClassLike()); + + ClassLikeNode* node = as(m_currentScope); + + if (node->m_marker.type != TokenType::Unknown) + { + m_sink->diagnose( + m_reader.peekToken(), + CPPDiagnostics::classMarkerAlreadyFound, + node->m_name.getContent()); + m_sink->diagnose(node->m_marker, CPPDiagnostics::previousLocation); + return SLANG_FAIL; + } + + // Set the marker token. + node->m_marker = m_reader.advanceToken(); + + // Looks like it's a marker + UnownedStringSlice slice(node->m_marker.getContent()); + + // Strip the prefix and suffix + slice = UnownedStringSlice( + slice.begin() + m_options->m_markPrefix.getLength(), + slice.end() - m_options->m_markSuffix.getLength()); + + // Strip ABSTRACT_ if it's there + UnownedStringSlice abstractSlice("ABSTRACT_"); + if (slice.startsWith(abstractSlice)) + { + slice = UnownedStringSlice(slice.begin() + abstractSlice.getLength(), slice.end()); + } + + // TODO: We could strip other stuff or have other heuristics there, but this is + // probably okay for now + + // Set the typeSet + node->m_typeSet = m_nodeTree->getOrAddTypeSet(slice); + + // Okay now looking for ( identifier) + Token typeNameToken; + + SLANG_RETURN_ON_FAIL(expect(TokenType::LParent)); + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &typeNameToken)); + SLANG_RETURN_ON_FAIL(expect(TokenType::RParent)); + + if (typeNameToken.getContent() != node->m_name.getContent()) + { + m_sink->diagnose( + typeNameToken, + CPPDiagnostics::typeNameDoesntMatch, + node->m_name.getContent()); + return SLANG_FAIL; + } + + // If has the marker it is assumed reflected + node->m_reflectionType = ReflectionType::Reflected; + return SLANG_OK; +} + +SlangResult Parser::_maybeParseType(UnownedStringSlice& outType, Token& outName) +{ + auto startCursor = m_reader.getCursor(); + + Index templateDepth = 0; + + TokenReader::ParsingCursor nameCursor; + + SlangResult res = _maybeParseType(templateDepth, nameCursor); + if (SLANG_FAILED(res) && m_sink->getErrorCount()) + { + return res; + } + + if (templateDepth != 0) + { + m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::unexpectedTemplateClose); + return SLANG_FAIL; + } + + if (nameCursor.isValid()) + { + const auto cursor = m_reader.getCursor(); + m_reader.setCursor(nameCursor); + outName = m_reader.peekToken(); + m_reader.setCursor(cursor); + + // Extract the contents + List toks; + _getTypeTokens(startCursor, nameCursor, toks); + outType = _concatTokens(toks.getBuffer(), toks.getCount()); + } + else + { + // We can build up the out type, from the tokens we found + outType = _concatTokens(startCursor); + } + return SLANG_OK; +} + +static bool _isBalancedOpen(TokenType tokenType) +{ + return tokenType == TokenType::LBrace || tokenType == TokenType::LParent || + tokenType == TokenType::LBracket; +} + +static bool _isBalancedClose(TokenType tokenType) +{ + return tokenType == TokenType::RBrace || tokenType == TokenType::RParent || + tokenType == TokenType::RBracket; +} + +static TokenType _getBalancedClose(TokenType tokenType) +{ + SLANG_ASSERT(_isBalancedOpen(tokenType)); + switch (tokenType) + { + case TokenType::LBrace: + return TokenType::RBrace; + case TokenType::LParent: + return TokenType::RParent; + case TokenType::LBracket: + return TokenType::RBracket; + default: + return TokenType::Unknown; + } +} + +SlangResult Parser::_parseBalanced(DiagnosticSink* sink) +{ + const TokenType openTokenType = m_reader.peekTokenType(); + if (!_isBalancedOpen(openTokenType)) + { + return SLANG_FAIL; + } + + // Save the start token + const Token startToken = m_reader.advanceToken(); + // Get the token type that would close the open + const TokenType closeTokenType = _getBalancedClose(openTokenType); + + while (true) + { + const TokenType tokenType = m_reader.peekTokenType(); + + // If we hit the closing token, we are done + if (tokenType == closeTokenType) + { + m_reader.advanceToken(); + return SLANG_OK; + } + + // If we hit a balanced open, recurse + if (_isBalancedOpen(tokenType)) + { + SLANG_RETURN_ON_FAIL(_parseBalanced(sink)); + continue; + } + + // If we hit a close token that doesn't match, then the balancing has gone wrong + if (_isBalancedClose(tokenType)) + { + // Only diagnose if required + if (sink) + { + sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::unexpectedUnbalancedToken); + sink->diagnose(startToken, CPPDiagnostics::seeOpen); + } + return SLANG_FAIL; + } + + // If we hit the end of the file and have not hit the closing token, then + // somethings gone wrong + if (tokenType == TokenType::EndOfFile) + { + if (sink) + { + sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::unexpectedEndOfFile); + sink->diagnose(startToken, CPPDiagnostics::seeOpen); + } + + return SLANG_FAIL; + } + + // Skip the token + m_reader.advanceToken(); + } +} + +SlangResult Parser::_consumeBalancedParens() +{ + SLANG_ASSERT(m_reader.peekTokenType() == TokenType::LParent); + + Index parenCount = 0; + + while (true) + { + const TokenType tokenType = m_reader.peekTokenType(); + + switch (tokenType) + { + case TokenType::LParent: + { + parenCount++; + break; + } + case TokenType::RParent: + { + --parenCount; + // If no more parens then we are done + if (parenCount == 0) + { + m_reader.advanceToken(); + return SLANG_OK; + } + break; + } + case TokenType::EndOfFile: + { + // If we hit the end of the file, then not balanced + return SLANG_FAIL; + } + default: + break; + } + + m_reader.advanceToken(); + } +} + +SlangResult Parser::_parseExpression(List& outExprTokens) +{ + Index parenCount = 0; + Index bracketCount = 0; + + // TODO(JS): NOTE! This doesn't handle an expression that contains a template params in + // Something, because without knowing what Something is, it's not known if < is a + // comparison or or a 'template' bracket + // + // This can be worked around in the originating source by placing in parens + + while (true) + { + TokenType tokenType = m_reader.peekTokenType(); + + switch (tokenType) + { + case TokenType::LParent: + { + parenCount++; + break; + } + case TokenType::RParent: + { + // If no parens, and nothing else is open then we are done + if (parenCount == 0) + { + if (bracketCount) + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::cannotParseExpression); + return SLANG_FAIL; + } + + return SLANG_OK; + } + --parenCount; + break; + } + case TokenType::LBracket: + { + bracketCount++; + break; + } + case TokenType::RBracket: + { + // If no brackets are open we are done + if (bracketCount == 0) + { + if (parenCount) + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::cannotParseExpression); + return SLANG_FAIL; + } + return SLANG_OK; + } + --bracketCount; + break; + } + case TokenType::EndOfFile: + { + if ((bracketCount | parenCount) == 0) + { + return SLANG_OK; + } + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::cannotParseExpression); + return SLANG_FAIL; + } + case TokenType::RBrace: + case TokenType::Semicolon: + case TokenType::Comma: + { + if ((bracketCount | parenCount) == 0) + { + return SLANG_OK; + } + break; + } + + default: + break; + } + + outExprTokens.add(m_reader.advanceToken()); + } +} + +SlangResult Parser::_parseTypeDef() +{ + if (!m_currentScope->canContainTypes()) + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::cannotDeclareTypeInScope); + return SLANG_FAIL; + } + + // Consume the typedef + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier)); + + Token nameToken; + // Parse the type + List toks; + SLANG_RETURN_ON_FAIL(_maybeParseType(toks, nameToken)); + + // Followed by the name + if (nameToken.type != TokenType::Identifier) + { + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &nameToken)); + } + + if (Node::lookupNameInScope(m_currentScope, nameToken.getContent())) + { + m_sink->diagnose( + nameToken.loc, + CPPDiagnostics::identifierAlreadyDefined, + nameToken.getContent()); + return SLANG_FAIL; + } + + SLANG_RETURN_ON_FAIL(expect(TokenType::Semicolon)); + + RefPtr node = new TypeDefNode; + node->m_name = nameToken; + node->m_reflectionType = m_currentScope->getContainedReflectionType(); + + // Set what aliases too + node->m_targetTypeTokens.swapWith(toks); + + m_currentScope->addChild(node); + + return SLANG_OK; +} + + +bool Parser::_isCtor() +{ + bool isCtor = false; + // It's a constructor + if (m_currentScope->isClassLike() && m_reader.peekTokenType() == TokenType::Identifier && + m_reader.peekToken().getContent() == m_currentScope->m_name.getContent()) + { + // We need to check it's followed immediately by ( to be sure it's a ctor + + auto cursor = m_reader.getCursor(); + m_reader.advanceToken(); + isCtor = (m_reader.peekTokenType() == TokenType::LParent); + m_reader.setCursor(cursor); + } + + return isCtor; +} + +bool isAlphaNumeric(char c) +{ + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9'); +} + +SlangResult Parser::_maybeParseContained(Node** outNode) +{ + *outNode = nullptr; + + _maybeConsume(IdentifierStyle::CallableMisc); + + bool isStatic = false; + bool isVirtual = false; + + while (m_reader.peekTokenType() == TokenType::Identifier) + { + const IdentifierStyle style = + m_nodeTree->m_identifierLookup->get(m_reader.peekToken().getContent()); + + // Check for virtualness + if (style == IdentifierStyle::Virtual) + { + isVirtual = true; + m_reader.advanceToken(); + continue; + } + + // Check if static + if (style == IdentifierStyle::Static) + { + isStatic = true; + m_reader.advanceToken(); + continue; + } + + break; + } + + _maybeConsume(IdentifierStyle::CallableMisc); + + UnownedStringSlice typeName; + Token nameToken; + + bool isConstructor = false; + + if (m_currentScope->isClassLike()) + { + // If it's a dtor + if (advanceIfToken(TokenType::OpBitNot, &nameToken)) + { + // Dtor + // For Dtor we don't hold the full name just the ~ + Token tok; + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &tok)); + + if (tok.getContent() != m_currentScope->m_name.getContent()) + { + m_sink->diagnose( + m_reader.peekLoc(), + CPPDiagnostics::destructorNameDoesntMatch, + m_currentScope->m_name.getContent()); + return SLANG_FAIL; + } + } + else if (_isCtor()) + { + nameToken = m_reader.advanceToken(); + isConstructor = true; + } + } + + // If don't have a name it's not a dtor or ctor, so see if it's a type + if (nameToken.type == TokenType::Unknown) + { + if (SLANG_FAILED(_maybeParseType(typeName, nameToken))) + { + if (m_sink->getErrorCount()) + { + return SLANG_FAIL; + } + + _consumeToSync(); + return SLANG_OK; + } + } + + if (nameToken.type == TokenType::Unknown) + { + // Has a calling convention (must be a function/method) + Token callingConventionToken; + advanceIfStyle(IdentifierStyle::CallingConvention, &callingConventionToken); + + // Expecting a name + if (!advanceIfToken(TokenType::Identifier, &nameToken)) + { + _consumeToSync(); + return SLANG_OK; + } + } + + // Handles other scenarios, but here for catching operator overloading + if (nameToken.type == TokenType::Identifier) + { + const auto style = m_nodeTree->m_identifierLookup->get(nameToken.getContent()); + if (style != IdentifierStyle::None) + { + _consumeToSync(); + return SLANG_OK; + } + } + + if (m_reader.peekTokenType() == TokenType::LParent) + { + if (!m_currentScope->canContainCallable()) + { + SLANG_RETURN_ON_FAIL(_consumeBalancedParens()); + // Consume everything up to ; or { + SLANG_RETURN_ON_FAIL(_consumeToSync()); + + return SLANG_OK; + } + + // Looks like it's a callable + m_reader.advanceToken(); + + List params; + + if (m_reader.peekTokenType() != TokenType::RParent) + { + while (true) + { + Token paramName; + UnownedStringSlice type; + SlangResult res = _maybeParseType(type, paramName); + + if (SLANG_FAILED(res)) + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::expectingType); + return res; + } + + if (paramName.type != TokenType::Identifier) + { + if (m_reader.peekTokenType() == TokenType::Identifier) + { + paramName = m_reader.advanceToken(); + } + } + + // If we have a name check for default value + if (paramName.type == TokenType::Identifier && advanceIfToken(TokenType::OpAssign)) + { + // Check if we have a default value + List exprTokens; + SLANG_RETURN_ON_FAIL(_parseExpression(exprTokens)); + } + + CallableNode::Param param; + param.m_name = paramName; + param.m_type = type; + + params.add(param); + + { + const auto peekType = m_reader.peekTokenType(); + if (peekType == TokenType::RParent) + { + break; + } + if (peekType == TokenType::Comma) + { + m_reader.advanceToken(); + continue; + } + } + + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::expectingToken, ", or ) or ="); + return SLANG_FAIL; + } + } + + // Skip ) + m_reader.advanceToken(); + + // Parse suffix + bool isPure = false; + + // const? + _maybeConsume(IdentifierStyle::TypeModifier); + + if (isConstructor) + { + // Initializer list + if (advanceIfToken(TokenType::Colon)) + { + while (true) + { + auto peekType = m_reader.peekTokenType(); + if (peekType == TokenType::Semicolon || peekType == TokenType::LBrace || + peekType == TokenType::EndOfFile) + { + break; + } + // Consume + m_reader.advanceToken(); + } + } + } + + // = 0 ? or = default + if (advanceIfToken(TokenType::OpAssign)) + { + if (m_reader.peekTokenType() == TokenType::IntegerLiteral) + { + Int value = -1; + if (SLANG_SUCCEEDED( + StringUtil::parseInt(m_reader.peekToken().getContent(), value)) && + value == 0) + { + isPure = true; + m_reader.advanceToken(); + } + else + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::expectingToken, "0"); + return SLANG_FAIL; + } + } + else if (advanceIfStyle(IdentifierStyle::Default)) + { + } + else + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::cannotParseCallable); + return SLANG_FAIL; + } + } + + if (m_reader.peekTokenType() == TokenType::Semicolon) + { + m_reader.advanceToken(); + } + else if (m_reader.peekTokenType() == TokenType::LBrace) + { + SLANG_RETURN_ON_FAIL(consumeToClosingBrace()); + } + else + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::expectingToken, "; or {"); + return SLANG_FAIL; + } + + RefPtr callableNode = new CallableNode; + + callableNode->m_returnType = typeName; + callableNode->m_name = nameToken; + callableNode->m_reflectionType = m_currentScope->getContainedReflectionType(); + + callableNode->m_isVirtual = isVirtual; + callableNode->m_isPure = isPure; + callableNode->m_isStatic = isStatic; + + callableNode->m_params.swapWith(params); + + Node* nodeWithName = m_currentScope->findChild(nameToken.getContent()); + + if (nodeWithName) + { + CallableNode* initialOverload = as(nodeWithName); + if (!initialOverload) + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::cannotOverload); + m_sink->diagnose(nodeWithName->getSourceLoc(), CPPDiagnostics::seeDeclarationOf); + return SLANG_FAIL; + } + + callableNode->m_nextOverload = initialOverload->m_nextOverload; + initialOverload->m_nextOverload = initialOverload; + + m_currentScope->addChildIgnoringName(callableNode); + } + else + { + m_currentScope->addChild(callableNode); + } + + *outNode = callableNode; + return SLANG_OK; + } + else + { + // Looks like variable + if (!m_currentScope->canContainFields() || nameToken.type != TokenType::Identifier) + { + _consumeToSync(); + return SLANG_OK; + } + + // Check if has a default value + if (advanceIfToken(TokenType::OpAssign)) + { + List exprTokens; + SLANG_RETURN_ON_FAIL(_parseExpression(exprTokens)); + } + + // Hit end of field/variable + if (m_reader.peekTokenType() == TokenType::Semicolon) + { + RefPtr fieldNode = new FieldNode; + + fieldNode->m_fieldType = typeName; + fieldNode->m_name = nameToken; + fieldNode->m_reflectionType = m_currentScope->getContainedReflectionType(); + fieldNode->m_isStatic = isStatic; + if (fieldNode->m_reflectionType == ReflectionType::Reflected) + { + static const char* illegalTypes[] = { + "size_t", + "Int", + "UInt", + "Index", + "Count", + "UIndex", + "UCount", + "PtrInt", + "intptr_t", + "uintptr_t"}; + for (const auto& illegalType : illegalTypes) + { + int index = typeName.indexOf(UnownedStringSlice(illegalType)); + if (index != -1) + { + index += UnownedStringSlice(illegalType).getLength(); + if (index >= typeName.getLength() || !isAlphaNumeric(typeName[index])) + { + // Cannot use this type in a field (as it's arch dependent + m_sink->diagnose( + nameToken, + CPPDiagnostics::cannoseUseArchDependentType, + illegalType); + return SLANG_FAIL; + } + } + } + } + m_currentScope->addChild(fieldNode); + + *outNode = fieldNode; + return SLANG_OK; + } + } + + _consumeToSync(); + return SLANG_OK; +} + +/* static */ Node::Kind Parser::_toNodeKind(IdentifierStyle style) +{ + switch (style) + { + case IdentifierStyle::Class: + return Node::Kind::ClassType; + case IdentifierStyle::Struct: + return Node::Kind::StructType; + case IdentifierStyle::Namespace: + return Node::Kind::Namespace; + case IdentifierStyle::Enum: + return Node::Kind::Enum; + case IdentifierStyle::TypeDef: + return Node::Kind::TypeDef; + default: + return Node::Kind::Invalid; + } +} + +static UnownedStringSlice _trimUnderscorePrefix(const UnownedStringSlice& slice) +{ + if (slice.getLength() && slice[0] == '_') + { + return UnownedStringSlice(slice.begin() + 1, slice.end()); + } + else + { + return slice; + } +} + +SlangResult Parser::_parsePreDeclare() +{ + // Skip the declare type token + m_reader.advanceToken(); + + SLANG_RETURN_ON_FAIL(expect(TokenType::LParent)); + + // Get the typeSet + Token typeSetToken; + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &typeSetToken)); + TypeSet* typeSet = m_nodeTree->getOrAddTypeSet(typeSetToken.getContent()); + + SLANG_RETURN_ON_FAIL(expect(TokenType::Comma)); + + // Get the type of type + Node::Kind nodeKind; + { + Token typeToken; + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &typeToken)); + + const IdentifierStyle style = m_nodeTree->m_identifierLookup->get(typeToken.getContent()); + + if (style != IdentifierStyle::Struct && style != IdentifierStyle::Class) + { + m_sink->diagnose( + typeToken, + CPPDiagnostics::expectingTypeKeyword, + typeToken.getContent()); + return SLANG_FAIL; + } + nodeKind = _toNodeKind(style); + } + + Token name; + Token super; + + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &name)); + + if (advanceIfToken(TokenType::Colon)) + { + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &super)); + } + + SLANG_RETURN_ON_FAIL(expect(TokenType::RParent)); + + switch (nodeKind) + { + case Node::Kind::ClassType: + case Node::Kind::StructType: + { + RefPtr node(new ClassLikeNode(nodeKind)); + + node->m_name = name; + node->m_super = super; + node->m_typeSet = typeSet; + + // Assume it is reflected + node->m_reflectionType = ReflectionType::Reflected; + + SLANG_RETURN_ON_FAIL(pushScope(node)); + // Pop out of the node + popScope(); + break; + } + default: + { + return SLANG_FAIL; + } + } + + + return SLANG_OK; +} + +SlangResult Parser::_parseTypeSet() +{ + // Skip the declare type token + m_reader.advanceToken(); + + SLANG_RETURN_ON_FAIL(expect(TokenType::LParent)); + + Token typeSetToken; + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &typeSetToken)); + + TypeSet* typeSet = m_nodeTree->getOrAddTypeSet(typeSetToken.getContent()); + + SLANG_RETURN_ON_FAIL(expect(TokenType::Comma)); + + // Get the type of type + Token typeToken; + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &typeToken)); + + SLANG_RETURN_ON_FAIL(expect(TokenType::RParent)); + + // Set the typename + typeSet->m_typeName = typeToken.getContent(); + + return SLANG_OK; +} + +SlangResult Parser::parse(SourceOrigin* sourceOrigin, const Options* options) +{ + SLANG_ASSERT(options); + m_options = options; + + // Set the current origin + m_sourceOrigin = sourceOrigin; + + SourceFile* sourceFile = sourceOrigin->m_sourceFile; + + SourceManager* manager = sourceFile->getSourceManager(); + + SourceView* sourceView = manager->createSourceView(sourceFile, nullptr, SourceLoc::fromRaw(0)); + + Lexer lexer; + + // Set up the scope stack + m_scopeStack.clear(); + + m_currentScope = m_nodeTree->m_rootNode; + m_scopeStack.add(m_currentScope); + + if (!options->m_requireMark) + { + m_currentScope->m_reflectionOverride = ReflectionType::Reflected; + } + + lexer.initialize(sourceView, m_sink, m_nodeTree->m_namePool, manager->getMemoryArena()); + m_tokenList = lexer.lexAllSemanticTokens(); + // See if there were any errors + if (m_sink->getErrorCount()) + { + return SLANG_FAIL; + } + + m_reader = TokenReader(m_tokenList); + + while (true) + { + switch (m_reader.peekTokenType()) + { + case TokenType::OpBitNot: + { + // Handle dtor + if (m_currentScope->isClassLike()) + { + Node* containedNode = nullptr; + SLANG_RETURN_ON_FAIL(_maybeParseContained(&containedNode)); + } + else + { + // consume + m_reader.advanceToken(); + } + break; + } + case TokenType::Identifier: + { + const IdentifierStyle style = + m_nodeTree->m_identifierLookup->get(m_reader.peekToken().getContent()); + + switch (style) + { + case IdentifierStyle::Extern: + { + m_reader.advanceToken(); + + Token externType; + SLANG_RETURN_ON_FAIL(expect(TokenType::StringLiteral, &externType)); + + if (advanceIfToken(TokenType::LBrace)) + { + // Push a 'special' scope (which is basically transparent) + pushScope(nullptr); + } + break; + } + case IdentifierStyle::Template: + { + SLANG_RETURN_ON_FAIL(_consumeTemplate()); + break; + } + case IdentifierStyle::PreDeclare: + { + SLANG_RETURN_ON_FAIL(_parsePreDeclare()); + break; + } + case IdentifierStyle::TypeSet: + { + SLANG_RETURN_ON_FAIL(_parseTypeSet()); + break; + } + case IdentifierStyle::Reflected: + { + m_reader.advanceToken(); + if (m_currentScope) + { + m_currentScope->m_reflectionOverride = ReflectionType::Reflected; + } + break; + } + case IdentifierStyle::Unreflected: + { + m_reader.advanceToken(); + if (m_currentScope) + { + m_currentScope->m_reflectionOverride = ReflectionType::NotReflected; + } + break; + } + case IdentifierStyle::Access: + { + m_reader.advanceToken(); + SLANG_RETURN_ON_FAIL(expect(TokenType::Colon)); + break; + } + case IdentifierStyle::TypeDef: + { + if (isTypeEnabled(Node::Kind::TypeDef)) + { + SLANG_RETURN_ON_FAIL(_parseTypeDef()); + } + else + { + m_reader.advanceToken(); + SLANG_RETURN_ON_FAIL(_consumeToSync()); + } + break; + } + default: + { + IdentifierFlags flags = getFlags(style); + + if (flags & IdentifierFlag::StartScope) + { + Node::Kind kind = _toNodeKind(style); + SLANG_ASSERT(kind != Node::Kind::Invalid); + + if (isTypeEnabled(kind)) + { + SLANG_RETURN_ON_FAIL(_maybeParseNode(kind)); + } + else + { + SLANG_RETURN_ON_FAIL(_maybeConsumeScope()); + } + } + else + { + UnownedStringSlice content = m_reader.peekToken().getContent(); + + // If it's a marker handle it + if (_isMarker(content)) + { + if (!m_currentScope->isClassLike()) + { + m_sink->diagnose( + m_reader.peekLoc(), + CPPDiagnostics::classMarkerOutsideOfClass); + return SLANG_FAIL; + } + + SLANG_RETURN_ON_FAIL(_parseMarker()); + break; + } + + if (m_options->m_markPrefix.getLength() > 0 && + content.startsWith(m_options->m_markPrefix.getUnownedSlice())) + { + SLANG_RETURN_ON_FAIL(_parseSpecialMacro()); + break; + } + + + // Special case the node that's the root of the hierarchy (as far as + // reflection is concerned) This could be a field + if (m_currentScope->canContainFields() || + m_currentScope->canContainCallable()) + { + Node* containedNode = nullptr; + SLANG_RETURN_ON_FAIL(_maybeParseContained(&containedNode)); + } + else + { + m_reader.advanceToken(); + } + } + break; + } + } + break; + } + case TokenType::LBrace: + { + SLANG_RETURN_ON_FAIL(consumeToClosingBrace()); + break; + } + case TokenType::RBrace: + { + SLANG_RETURN_ON_FAIL(popScope()); + m_reader.advanceToken(); + break; + } + case TokenType::EndOfFile: + { + // Okay we need to confirm that we are in the root node, and with no open braces + if (m_currentScope != m_nodeTree->getRootNode()) + { + m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::braceOpenAtEndOfFile); + return SLANG_FAIL; + } + if (m_sink->getErrorCount()) + return SLANG_FAIL; + return SLANG_OK; + } + case TokenType::Pound: + { + Token token = m_reader.peekToken(); + if (token.flags & TokenFlag::AtStartOfLine) + { + // We are just going to ignore all of these for now.... + m_reader.advanceToken(); + for (;;) + { + auto t = m_reader.peekToken(); + if (t.type == TokenType::EndOfFile || (t.flags & TokenFlag::AtStartOfLine)) + { + break; + } + m_reader.advanceToken(); + } + break; + } + // Skip it then + m_reader.advanceToken(); + break; + } + default: + { + // Skip it then + m_reader.advanceToken(); + break; + } + } + } +} + +} // namespace CppParse diff --git a/tools/slang-cpp-parser/parser.h b/tools/slang-cpp-parser/parser.h new file mode 100644 index 000000000..519ad6935 --- /dev/null +++ b/tools/slang-cpp-parser/parser.h @@ -0,0 +1,112 @@ +#pragma once + +#include "compiler-core/slang-lexer.h" +#include "diagnostics.h" +#include "identifier-lookup.h" +#include "node-tree.h" +#include "node.h" + +namespace CppParse +{ +using namespace Slang; + +class Parser +{ +public: + typedef uint32_t NodeTypeBitType; + + SlangResult expect(TokenType type, Token* outToken = nullptr); + + bool advanceIfMarker(Token* outToken = nullptr); + bool advanceIfToken(TokenType type, Token* outToken = nullptr); + bool advanceIfStyle(IdentifierStyle style, Token* outToken = nullptr); + + SlangResult pushAnonymousNamespace(); + SlangResult pushScope(ScopeNode* node); + SlangResult consumeToClosingBrace(const Token* openBraceToken = nullptr); + SlangResult popScope(); + + /// Parse the contents of the source file + SlangResult parse(SourceOrigin* sourceOrigin, const Options* options); + + void setKindEnabled(Node::Kind kind, bool isEnabled = true); + bool isTypeEnabled(Node::Kind kind) + { + return (m_nodeTypeEnabled & (NodeTypeBitType(1) << int(kind))) != 0; + } + + void setKindsEnabled(const Node::Kind* kinds, Index kindsCount, bool isEnabled = true); + + Parser(NodeTree* nodeTree, DiagnosticSink* sink); + +protected: + static Node::Kind _toNodeKind(IdentifierStyle style); + + bool _isMarker(const UnownedStringSlice& name); + + SlangResult _maybeConsumeScope(); + + SlangResult _parsePreDeclare(); + SlangResult _parseTypeSet(); + + SlangResult _maybeParseNode(Node::Kind kind); + SlangResult _maybeParseContained(Node** outNode); + + SlangResult _parseTypeDef(); + SlangResult _parseEnum(); + SlangResult _parseMarker(); + SlangResult _parseSpecialMacro(); + + SlangResult _maybeParseType(List& outToks, Token& outName); + SlangResult _maybeParseType(UnownedStringSlice& outType, Token& outName); + SlangResult _maybeParseType(Index& ioTemplateDepth, TokenReader::ParsingCursor& outCursor); + + SlangResult _parseExpression(List& outExprTokens); + + SlangResult _maybeParseTemplateArgs(Index& ioTemplateDepth); + SlangResult _maybeParseTemplateArg(Index& ioTemplateDepth); + + /// Parse balanced - if a sink is set will report to that sink + SlangResult _parseBalanced(DiagnosticSink* sink); + + bool _isCtor(); + + /// Concatenate all tokens from start to the current position + UnownedStringSlice _concatTokens(TokenReader::ParsingCursor start); + UnownedStringSlice _concatTokens(const Token* toks, Index toksCount); + + UnownedStringSlice _concatType( + TokenReader::ParsingCursor start, + TokenReader::ParsingCursor nameCursor); + + void _getTypeTokens( + TokenReader::ParsingCursor start, + TokenReader::ParsingCursor nameCursor, + List& outToks); + + /// Consume what looks like a template definition + SlangResult _consumeTemplate(); + SlangResult _maybeConsume(IdentifierStyle style); + + SlangResult _consumeToSync(); + /// Consumes balanced parens. Will return an error if not matched. Assumes starts on opening ( + SlangResult _consumeBalancedParens(); + + NodeTypeBitType m_nodeTypeEnabled; + + TokenList m_tokenList; + TokenReader m_reader; + + List m_scopeStack; + + ScopeNode* m_currentScope; ///< The current scope being processed + SourceOrigin* m_sourceOrigin; ///< The source origin that all tokens are in + + DiagnosticSink* m_sink; ///< Diagnostic sink + + NodeTree* m_nodeTree; ///< Shared state between parses. Nodes will be added to this + + const Options* m_options; +}; + +} // namespace CppParse diff --git a/tools/slang-cpp-parser/unit-test.cpp b/tools/slang-cpp-parser/unit-test.cpp new file mode 100644 index 000000000..70851fd7c --- /dev/null +++ b/tools/slang-cpp-parser/unit-test.cpp @@ -0,0 +1,127 @@ +#include "unit-test.h" + +#include "compiler-core/slang-lexer.h" +#include "compiler-core/slang-name-convention-util.h" +#include "compiler-core/slang-source-loc.h" +#include "core/slang-io.h" +#include "identifier-lookup.h" +#include "node-tree.h" +#include "options.h" +#include "parser.h" + +namespace CppParse +{ +using namespace Slang; + + +struct TestState +{ + TestState() + : m_slicePool(StringSlicePool::Style::Default) + { + m_identifierLookup.initDefault(UnownedStringSlice::fromLiteral("SLANG_")); + + m_sourceManager.initialize(nullptr, nullptr); + + m_sink.init(&m_sourceManager, Lexer::sourceLocationLexer); + + m_namePool.setRootNamePool(&m_rootNamePool); + + // We don't require marker + m_options.m_requireMark = false; + } + + RootNamePool m_rootNamePool; + Options m_options; + SourceManager m_sourceManager; + DiagnosticSink m_sink; + NamePool m_namePool; + StringSlicePool m_slicePool; + IdentifierLookup m_identifierLookup; +}; + +static const char someSource[] = "class ISomeInterface\n" + "{\n" + " public:\n" + " virtual int SLANG_MCALL someMethod(int a, int b) const = 0;\n" + " virtual float SLANG_MCALL anotherMethod(float a) = 0;\n" + "};\n" + "\n" + "struct SomeStruct\n" + "{\n" + " SomeStruct() = default;\n" + " SomeStruct(float v = 0.0f):b(v) {}\n" + " ~SomeStruct() {}\n" + " int a = 10; \n" + " float b; \n" + " int another[10];\n" + " const char* yetAnother = nullptr;\n" + "};\n" + "\n" + "enum SomeEnum\n" + "{\n" + " Value,\n" + " Another = 10,\n" + "};\n" + "\n" + "typedef int (*SomeFunc)(int a);\n" + "\n" + "typedef SomeEnum AliasEnum;\n" + "void someFunc(int a, float b) { }\n" + "namespace Blah {\n" + "int add(int a, int b) { return a + b; }\n" + "unsigned add(unsigned a, unsigned b) { return a + b; }\n" + "}\n"; + + +/* static */ SlangResult UnitTestUtil::run() +{ + { + TestState state; + + NodeTree tree(&state.m_slicePool, &state.m_namePool, &state.m_identifierLookup); + + UnownedStringSlice contents = UnownedStringSlice::fromLiteral(someSource); + PathInfo pathInfo = PathInfo::makeFromString("source.h"); + + SourceManager* sourceManager = &state.m_sourceManager; + + SourceFile* sourceFile = sourceManager->createSourceFileWithString(pathInfo, contents); + SourceOrigin* sourceOrigin = tree.addSourceOrigin(sourceFile, state.m_options); + + Parser parser(&tree, &state.m_sink); + + + { + const Node::Kind enableKinds[] = { + Node::Kind::Enum, + Node::Kind::EnumClass, + Node::Kind::EnumCase, + Node::Kind::TypeDef}; + parser.setKindsEnabled(enableKinds, SLANG_COUNT_OF(enableKinds)); + } + + SlangResult res = parser.parse(sourceOrigin, &state.m_options); + + if (state.m_sink.outputBuffer.getLength()) + { + printf("%s\n", state.m_sink.outputBuffer.getBuffer()); + } + + if (SLANG_FAILED(res)) + { + return res; + } + + { + StringBuilder buf; + tree.getRootNode()->dump(0, buf); + + SLANG_UNUSED(buf); + } + } + + return SLANG_OK; +} + +} // namespace CppParse diff --git a/tools/slang-cpp-parser/unit-test.h b/tools/slang-cpp-parser/unit-test.h new file mode 100644 index 000000000..e8d1b53b2 --- /dev/null +++ b/tools/slang-cpp-parser/unit-test.h @@ -0,0 +1,14 @@ +#pragma once + +#include "diagnostics.h" + +namespace CppParse +{ +using namespace Slang; + +struct UnitTestUtil +{ + static SlangResult run(); +}; + +} // namespace CppParse -- cgit v1.2.3