diff options
| author | Lauro Oyen <15063951+laurooyen@users.noreply.github.com> | 2024-12-02 20:46:43 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-12-02 11:46:43 -0800 |
| commit | eaa8dcfcc9deabb906cc09bf31fc17ab6f343ff4 (patch) | |
| tree | 8e0f4658de3efb5e7696e8588c55471f9d65ba18 /tools/slang-cpp-parser | |
| parent | 7aaf7009e2c6055a714ba4a93ab3dd73d2d2cdb7 (diff) | |
Move c++ parsing code from slang-cpp-extractor to static library (#5675)
* Move c++ parsing code from slang-cpp-extractor to static library
* Format code
* Remove relative includes
---------
Co-authored-by: slangbot <ellieh+slangbot@nvidia.com>
Co-authored-by: Yong He <yonghe@outlook.com>
Diffstat (limited to 'tools/slang-cpp-parser')
| -rw-r--r-- | tools/slang-cpp-parser/diagnostic-defs.h | 92 | ||||
| -rw-r--r-- | tools/slang-cpp-parser/diagnostics.cpp | 15 | ||||
| -rw-r--r-- | tools/slang-cpp-parser/diagnostics.h | 16 | ||||
| -rw-r--r-- | tools/slang-cpp-parser/file-util.cpp | 101 | ||||
| -rw-r--r-- | tools/slang-cpp-parser/file-util.h | 31 | ||||
| -rw-r--r-- | tools/slang-cpp-parser/identifier-lookup.cpp | 177 | ||||
| -rw-r--r-- | tools/slang-cpp-parser/identifier-lookup.h | 121 | ||||
| -rw-r--r-- | tools/slang-cpp-parser/node-tree.cpp | 176 | ||||
| -rw-r--r-- | tools/slang-cpp-parser/node-tree.h | 101 | ||||
| -rw-r--r-- | tools/slang-cpp-parser/node.cpp | 700 | ||||
| -rw-r--r-- | tools/slang-cpp-parser/node.h | 422 | ||||
| -rw-r--r-- | tools/slang-cpp-parser/options.cpp | 153 | ||||
| -rw-r--r-- | tools/slang-cpp-parser/options.h | 63 | ||||
| -rw-r--r-- | tools/slang-cpp-parser/parser.cpp | 2261 | ||||
| -rw-r--r-- | tools/slang-cpp-parser/parser.h | 112 | ||||
| -rw-r--r-- | tools/slang-cpp-parser/unit-test.cpp | 127 | ||||
| -rw-r--r-- | tools/slang-cpp-parser/unit-test.h | 14 |
17 files changed, 4682 insertions, 0 deletions
diff --git a/tools/slang-cpp-parser/diagnostic-defs.h b/tools/slang-cpp-parser/diagnostic-defs.h new file mode 100644 index 000000000..cba5a8e6f --- /dev/null +++ b/tools/slang-cpp-parser/diagnostic-defs.h @@ -0,0 +1,92 @@ +// + +// The file is meant to be included multiple times, to produce different +// pieces of declaration/definition code related to diagnostic messages +// +// Each diagnostic is declared here with: +// +// DIAGNOSTIC(id, severity, name, messageFormat) +// +// Where `id` is the unique diagnostic ID, `severity` is the default +// severity (from the `Severity` enum), `name` is a name used to refer +// to this diagnostic from code, and `messageFormat` is the default +// (non-localized) message for the diagnostic, with placeholders +// for any arguments. + +#ifndef DIAGNOSTIC +#error Need to #define DIAGNOSTIC(...) before including "slang-cpp-parser/diagnostics-defs.h" +#define DIAGNOSTIC(id, severity, name, messageFormat) /* */ +#endif + +DIAGNOSTIC(-1, Note, seeDeclarationOf, "see declaration of '$0'") +DIAGNOSTIC(-1, Note, seeOpen, "see open $0") +DIAGNOSTIC(-1, Note, commandLine, "Command line: $0") +DIAGNOSTIC(-1, Note, previousLocation, "previous location") + +DIAGNOSTIC(1, Error, cannotOpenFile, "cannot open file '$0'.") +DIAGNOSTIC(1, Error, errorAccessingFile, "error accessing file '$0'.") + +DIAGNOSTIC(1, Error, extractorFailed, "C++ Extractor failed") +DIAGNOSTIC(1, Error, internalError, "Unknown internal error in C++ Extractor, aborted!") + +// Parsing errors +DIAGNOSTIC(100000, Error, expectingToken, "Expecting token $0") +DIAGNOSTIC(100001, Error, typeAlreadyDeclared, "Type '$0' already declared") +DIAGNOSTIC(100002, Error, scopeNotClosed, "Scope not closed") +DIAGNOSTIC(100003, Error, typeNameDoesntMatch, "Type name doesn't match $0") +DIAGNOSTIC(100004, Error, didntFindMatchingBrace, "Didn't find brace matching $0") +DIAGNOSTIC(100005, Error, braceOpenAtEndOfFile, "Brace open at file end") +DIAGNOSTIC(100006, Error, unexpectedTemplateClose, "Unexpected template close") +DIAGNOSTIC(100007, Error, superTypeNotFound, "Super type not found for $0") +DIAGNOSTIC(100008, Error, superTypeNotAType, "Named super type is not a type $0") +DIAGNOSTIC(100009, Error, unexpectedUnbalancedToken, "Unexpected unbalanced token") +DIAGNOSTIC(100010, Error, unexpectedEndOfFile, "Unexpected end of file") +DIAGNOSTIC( + 100011, + Error, + expectingTypeKeyword, + "Expecting type keyword - struct or class, found $0") + +DIAGNOSTIC( + 100012, + Error, + typeInDifferentTypeSet, + "Type $0 in different type set $1 from super class $2") +DIAGNOSTIC(100013, Error, expectingIdentifier, "Expecting an identifier, found $0") +DIAGNOSTIC(100014, Error, cannotDeclareTypeInScope, "Cannot declare types in this scope") +DIAGNOSTIC(100015, Error, identifierAlreadyDefined, "Identifier already defined '$0'") +DIAGNOSTIC(100016, Error, expectingType, "Expecting a type") +DIAGNOSTIC(100017, Error, cannotParseExpression, "Cannot parse expression") +DIAGNOSTIC(100018, Error, cannotOverload, "Cannot overload this declaration"); +DIAGNOSTIC( + 100019, + Error, + classMarkerOutsideOfClass, + "A class/struct marker is found outside of class or struct"); +DIAGNOSTIC( + 100020, + Error, + classMarkerAlreadyFound, + "A class/struct marker already found in type '$0'"); +DIAGNOSTIC(100021, Error, cannotParseType, "Cannot parse type"); +DIAGNOSTIC( + 100022, + Error, + destructorNameDoesntMatch, + "Destructor name doesn't match class name '$0'"); +DIAGNOSTIC(100023, Error, cannotParseCallable, "Cannot parse callable"); +DIAGNOSTIC( + 100024, + Error, + cannoseUseArchDependentType, + "Cannot use architecture dependent type '$0' for serializable data.") + +// Command line errors 100100 + +DIAGNOSTIC(100101, Error, optionAlreadyDefined, "Option '$0' is already defined '$1'") +DIAGNOSTIC(100102, Error, requireValueAfterOption, "Require a value after $0 option") +DIAGNOSTIC(100103, Error, unknownOption, "Unknown option '$0'") +DIAGNOSTIC(100104, Error, noInputPathsSpecified, "No input paths specified") +DIAGNOSTIC(100105, Error, noOutputPathSpecified, "No -o output path specified") + +#undef DIAGNOSTIC diff --git a/tools/slang-cpp-parser/diagnostics.cpp b/tools/slang-cpp-parser/diagnostics.cpp new file mode 100644 index 000000000..10455a9ff --- /dev/null +++ b/tools/slang-cpp-parser/diagnostics.cpp @@ -0,0 +1,15 @@ +#include "diagnostics.h" + +namespace CppParse +{ + +namespace CPPDiagnostics +{ +using namespace Slang; + +#define DIAGNOSTIC(id, severity, name, messageFormat) \ + const DiagnosticInfo name = {id, Severity::severity, #name, messageFormat}; +#include "diagnostic-defs.h" +} // namespace CPPDiagnostics + +} // namespace CppParse diff --git a/tools/slang-cpp-parser/diagnostics.h b/tools/slang-cpp-parser/diagnostics.h new file mode 100644 index 000000000..4db70fbca --- /dev/null +++ b/tools/slang-cpp-parser/diagnostics.h @@ -0,0 +1,16 @@ +#pragma once + +#include "slang/slang-diagnostics.h" + +namespace CppParse +{ +using namespace Slang; + +namespace CPPDiagnostics +{ + +#define DIAGNOSTIC(id, severity, name, messageFormat) extern const DiagnosticInfo name; +#include "diagnostic-defs.h" + +} // namespace CPPDiagnostics +} // namespace CppParse diff --git a/tools/slang-cpp-parser/file-util.cpp b/tools/slang-cpp-parser/file-util.cpp new file mode 100644 index 000000000..9e7c60755 --- /dev/null +++ b/tools/slang-cpp-parser/file-util.cpp @@ -0,0 +1,101 @@ +#include "file-util.h" + +#include "core/slang-io.h" + +namespace CppParse +{ +using namespace Slang; + +namespace +{ // anonymous +struct DiagnosticReporter +{ + SlangResult report(SlangResult res) + { + if (SLANG_FAILED(res)) + { + if (m_sink) + { + if (res == SLANG_E_CANNOT_OPEN) + { + m_sink->diagnose(SourceLoc(), CPPDiagnostics::cannotOpenFile, m_filename); + } + else + { + m_sink->diagnose(SourceLoc(), CPPDiagnostics::errorAccessingFile, m_filename); + } + } + } + return res; + } + + DiagnosticReporter(const String& filename, DiagnosticSink* sink) + : m_filename(filename), m_sink(sink) + { + } + + DiagnosticSink* m_sink; + String m_filename; +}; + +} // namespace + +/* static */ SlangResult FileUtil::readAllText( + const Slang::String& fileName, + DiagnosticSink* sink, + String& outRead) +{ + DiagnosticReporter reporter(fileName, sink); + + RefPtr<FileStream> stream = new FileStream; + SLANG_RETURN_ON_FAIL(reporter.report( + stream->init(fileName, FileMode::Open, FileAccess::Read, FileShare::ReadWrite))); + + StreamReader reader; + SLANG_RETURN_ON_FAIL(reporter.report(reader.init(stream))); + SLANG_RETURN_ON_FAIL(reporter.report(reader.readToEnd(outRead))); + + return SLANG_OK; +} + +/* static */ SlangResult FileUtil::writeAllText( + const Slang::String& fileName, + DiagnosticSink* sink, + const UnownedStringSlice& text) +{ + // TODO(JS): + // There is an optimization/behavior here,that checks if the contents has changed. It only + // writes if it hasn't That might not be what you want (both because of extra work of read, the + // file modified stamp or other reasons, file is write only etc) NOTE! That this also does the + // work of the comparison after it is decoded, but the *bytes* might actually be different. + + if (File::exists(fileName)) + { + String existingText; + if (SLANG_SUCCEEDED(readAllText(fileName, nullptr, existingText))) + { + if (existingText == text) + return SLANG_OK; + } + } + + DiagnosticReporter reporter(fileName, sink); + + RefPtr<FileStream> stream = new FileStream; + SLANG_RETURN_ON_FAIL(reporter.report(stream->init(fileName, FileMode::Create))); + + StreamWriter writer; + SLANG_RETURN_ON_FAIL(reporter.report(writer.init(stream))); + SLANG_RETURN_ON_FAIL(reporter.report(writer.write(text))) + return SLANG_OK; +} + +/* static */ void FileUtil::indent(Index indentCount, StringBuilder& out) +{ + for (Index i = 0; i < indentCount; ++i) + { + out << CPP_EXTRACT_INDENT_STRING; + } +} + +} // namespace CppParse diff --git a/tools/slang-cpp-parser/file-util.h b/tools/slang-cpp-parser/file-util.h new file mode 100644 index 000000000..735758384 --- /dev/null +++ b/tools/slang-cpp-parser/file-util.h @@ -0,0 +1,31 @@ +#pragma once + +#include "diagnostics.h" + +namespace CppParse +{ +using namespace Slang; + +// A macro to define a single indent as a string +#define CPP_EXTRACT_INDENT_STRING " " + +struct FileUtil +{ + /// Read text into outRead. Any failures written to sink (can be passed as nullptr, for no + /// output) + static SlangResult readAllText( + const Slang::String& fileName, + DiagnosticSink* sink, + String& outRead); + /// Write text to filename. Any failures written to sink. (can be passed as nullptr, for no + /// output) + static SlangResult writeAllText( + const Slang::String& fileName, + DiagnosticSink* sink, + const UnownedStringSlice& text); + + /// Appends CPP_EXTRACT_INDENT_STRING indentCount number of times to out + static void indent(Index indentCount, StringBuilder& out); +}; + +} // namespace CppParse diff --git a/tools/slang-cpp-parser/identifier-lookup.cpp b/tools/slang-cpp-parser/identifier-lookup.cpp new file mode 100644 index 000000000..3a704a454 --- /dev/null +++ b/tools/slang-cpp-parser/identifier-lookup.cpp @@ -0,0 +1,177 @@ +#include "identifier-lookup.h" + +namespace CppParse +{ +using namespace Slang; + +/* static */ const IdentifierFlags + IdentifierLookup::kIdentifierFlags[Index(IdentifierStyle::CountOf)] = { + 0, /// None + 0, /// Identifier + 0, /// Declare type + 0, /// Type set + IdentifierFlag::Keyword, /// TypeModifier + IdentifierFlag::Keyword, /// Keyword + + IdentifierFlag::Keyword | IdentifierFlag::StartScope | IdentifierFlag::ClassLike, /// Class + IdentifierFlag::Keyword | IdentifierFlag::StartScope | IdentifierFlag::ClassLike, /// Struct + IdentifierFlag::Keyword | IdentifierFlag::StartScope, /// Namespace + IdentifierFlag::Keyword | IdentifierFlag::StartScope, /// Enum + + IdentifierFlag::Keyword, /// Typedef + + IdentifierFlag::Keyword, /// Access + IdentifierFlag::Reflection, /// Reflected + IdentifierFlag::Reflection, /// Unreflected + + IdentifierFlag::Keyword, /// virtual + 0, /// Calling convention + IdentifierFlag::Keyword, /// template + IdentifierFlag::Keyword, /// static + + IdentifierFlag::Keyword, /// unsigned/signed + + IdentifierFlag::Keyword, /// extern + + 0, /// Callable misc + 0, /// IntegerType int, short, char, long + + IdentifierFlag::Keyword, /// default +}; + +void IdentifierLookup::set(const UnownedStringSlice& name, IdentifierStyle style) +{ + StringSlicePool::Handle handle; + if (m_pool.findOrAdd(name, handle)) + { + // Add the extra flags + m_styles[Index(handle)] = style; + } + else + { + Index index = Index(handle); + SLANG_ASSERT(index == m_styles.getCount()); + m_styles.add(style); + } +} + +void IdentifierLookup::set(const char* const* names, size_t namesCount, IdentifierStyle style) +{ + for (size_t i = 0; i < namesCount; ++i) + { + set(UnownedStringSlice(names[i]), style); + } +} + +void IdentifierLookup::set(const Pair* pairs, Index pairsCount) +{ + for (Index i = 0; i < pairsCount; ++i) + { + const auto& pair = pairs[i]; + set(UnownedStringSlice(pair.name), pair.style); + } +} + +void IdentifierLookup::initDefault(const UnownedStringSlice& markPrefix) +{ + reset(); + + // Some keywords + { + const char* names[] = { + "continue", + "if", + "case", + "break", + "catch", + "delete", + "do", + "else", + "for", + "new", + "goto", + "return", + "switch", + "throw", + "using", + "while", + "operator", + "explicit"}; + set(names, SLANG_COUNT_OF(names), IdentifierStyle::Keyword); + } + + // Type modifier keywords + { + const char* names[] = {"const", "volatile"}; + set(names, SLANG_COUNT_OF(names), IdentifierStyle::TypeModifier); + } + + // Special markers + { + const char* names[] = {"PRE_DECLARE", "TYPE_SET", "REFLECTED", "UNREFLECTED"}; + const IdentifierStyle styles[] = { + IdentifierStyle::PreDeclare, + IdentifierStyle::TypeSet, + IdentifierStyle::Reflected, + IdentifierStyle::Unreflected}; + SLANG_COMPILE_TIME_ASSERT(SLANG_COUNT_OF(names) == SLANG_COUNT_OF(styles)); + + StringBuilder buf; + for (Index i = 0; i < SLANG_COUNT_OF(names); ++i) + { + buf.clear(); + buf << markPrefix << names[i]; + set(buf.getUnownedSlice(), styles[i]); + } + } + + { + set("virtual", IdentifierStyle::Virtual); + + set("template", IdentifierStyle::Template); + set("static", IdentifierStyle::Static); + set("extern", IdentifierStyle::Extern); + set("default", IdentifierStyle::Default); + } + + { + const char* names[] = {"char", "short", "int", "long"}; + set(names, SLANG_COUNT_OF(names), IdentifierStyle::IntegerType); + } + + { + const char* names[] = {"SLANG_MCALL"}; + set(names, SLANG_COUNT_OF(names), IdentifierStyle::CallingConvention); + } + + { + const char* names[] = {"SLANG_NO_THROW", "inline"}; + set(names, SLANG_COUNT_OF(names), IdentifierStyle::CallableMisc); + } + + // Keywords which introduce types/scopes + { + const Pair pairs[] = { + {"struct", IdentifierStyle::Struct}, + {"class", IdentifierStyle::Class}, + {"namespace", IdentifierStyle::Namespace}, + {"enum", IdentifierStyle::Enum}, + {"typedef", IdentifierStyle::TypeDef}, + }; + + set(pairs, SLANG_COUNT_OF(pairs)); + } + + // Keywords that control access + { + const char* names[] = {"private", "protected", "public"}; + set(names, SLANG_COUNT_OF(names), IdentifierStyle::Access); + } + { + const char* names[] = {"signed", "unsigned"}; + + set(names, SLANG_COUNT_OF(names), IdentifierStyle::IntegerModifier); + } +} + +} // namespace CppParse diff --git a/tools/slang-cpp-parser/identifier-lookup.h b/tools/slang-cpp-parser/identifier-lookup.h new file mode 100644 index 000000000..226d00667 --- /dev/null +++ b/tools/slang-cpp-parser/identifier-lookup.h @@ -0,0 +1,121 @@ +#pragma once + +#include "diagnostics.h" + +namespace CppParse +{ +using namespace Slang; + +enum class IdentifierStyle +{ + None, ///< It's not an identifier + + Identifier, ///< Just an identifier + + PreDeclare, ///< Declare a type (not visible in C++ code) + TypeSet, ///< TypeSet + + TypeModifier, ///< const, volatile etc + Keyword, ///< A keyword C/C++ keyword that is not another type + + Class, ///< class + Struct, ///< struct + Namespace, ///< namespace + Enum, ///< enum + + TypeDef, ///< typedef + + Access, ///< public, protected, private + + Reflected, + Unreflected, + + CallingConvention, ///< Used on a method + Virtual, ///< + + Template, + + Static, + + IntegerModifier, + + Extern, + + CallableMisc, ///< For SLANG_NO_THROW etc + + IntegerType, ///< Built in integer type + + Default, /// default + + CountOf, +}; + +typedef uint32_t IdentifierFlags; +struct IdentifierFlag +{ + enum Enum : IdentifierFlags + { + StartScope = 0x1, ///< namespace, struct or class + ClassLike = 0x2, ///< Struct or class + Keyword = 0x4, + Reflection = 0x8, + }; +}; + + +class IdentifierLookup +{ +public: + struct Pair + { + const char* name; + IdentifierStyle style; + }; + + IdentifierStyle get(const UnownedStringSlice& slice) const + { + Index index = m_pool.findIndex(slice); + return (index >= 0) ? m_styles[index] : IdentifierStyle::None; + } + + void set(const char* name, IdentifierStyle style) { set(UnownedStringSlice(name), style); } + + void set(const UnownedStringSlice& name, IdentifierStyle style); + + void set(const char* const* names, size_t namesCount, IdentifierStyle style); + + void set(const Pair* pairs, Index pairsCount); + + void reset() + { + m_styles.clear(); + m_pool.clear(); + } + + void initDefault(const UnownedStringSlice& markPrefix); + + IdentifierLookup() + : m_pool(StringSlicePool::Style::Empty) + { + SLANG_ASSERT(m_pool.getSlicesCount() == 0); + } + + static const IdentifierFlags kIdentifierFlags[Index(IdentifierStyle::CountOf)]; + +protected: + List<IdentifierStyle> m_styles; + StringSlicePool m_pool; +}; + + +SLANG_FORCE_INLINE IdentifierFlags getFlags(IdentifierStyle style) +{ + return IdentifierLookup::kIdentifierFlags[Index(style)]; +} + +SLANG_FORCE_INLINE bool hasFlag(IdentifierStyle style, IdentifierFlag::Enum flag) +{ + return (getFlags(style) & flag) != 0; +} + +} // namespace CppParse diff --git a/tools/slang-cpp-parser/node-tree.cpp b/tools/slang-cpp-parser/node-tree.cpp new file mode 100644 index 000000000..e48ce9779 --- /dev/null +++ b/tools/slang-cpp-parser/node-tree.cpp @@ -0,0 +1,176 @@ +#include "node-tree.h" + +#include "compiler-core/slang-name-convention-util.h" +#include "core/slang-io.h" +#include "identifier-lookup.h" +#include "options.h" + +namespace CppParse +{ +using namespace Slang; + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!! NodeTree !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +NodeTree::NodeTree( + StringSlicePool* typePool, + NamePool* namePool, + IdentifierLookup* identifierLookup) + : m_typePool(typePool) + , m_namePool(namePool) + , m_identifierLookup(identifierLookup) + , m_typeSetPool(StringSlicePool::Style::Empty) +{ + m_rootNode = new ScopeNode(Node::Kind::Namespace); + m_rootNode->m_reflectionType = ReflectionType::Reflected; +} + +TypeSet* NodeTree::getTypeSet(const UnownedStringSlice& slice) +{ + Index index = m_typeSetPool.findIndex(slice); + if (index < 0) + { + return nullptr; + } + return m_typeSets[index]; +} + +TypeSet* NodeTree::getOrAddTypeSet(const UnownedStringSlice& slice) +{ + const Index index = Index(m_typeSetPool.add(slice)); + if (index >= m_typeSets.getCount()) + { + SLANG_ASSERT(m_typeSets.getCount() == index); + TypeSet* typeSet = new TypeSet; + + m_typeSets.add(typeSet); + typeSet->m_macroName = m_typeSetPool.getSlice(StringSlicePool::Handle(index)); + return typeSet; + } + else + { + return m_typeSets[index]; + } +} + +SourceOrigin* NodeTree::addSourceOrigin(SourceFile* sourceFile, const Options& options) +{ + // Calculate from the path, a 'macro origin' name. + const String macroOrigin = calcMacroOrigin(sourceFile->getPathInfo().foundPath, options); + + SourceOrigin* origin = new SourceOrigin(sourceFile, macroOrigin); + m_sourceOrigins.add(origin); + return origin; +} + +/* static */ String NodeTree::calcMacroOrigin(const String& filePath, const Options& options) +{ + // Get the filename without extension + String fileName = Path::getFileNameWithoutExt(filePath); + + // We can work on just the slice + UnownedStringSlice slice = fileName.getUnownedSlice(); + + // Filename prefix + if (options.m_stripFilePrefix.getLength() && + slice.startsWith(options.m_stripFilePrefix.getUnownedSlice())) + { + const Index len = options.m_stripFilePrefix.getLength(); + slice = UnownedStringSlice(slice.begin() + len, slice.end()); + } + + // Trim - + slice = slice.trim('-'); + + StringBuilder out; + NameConventionUtil::convert(slice, NameConvention::UpperSnake, out); + return out; +} + +SlangResult NodeTree::_calcDerivedTypesRec(ScopeNode* inScopeNode, DiagnosticSink* sink) +{ + if (inScopeNode->isClassLike()) + { + ClassLikeNode* classLikeNode = static_cast<ClassLikeNode*>(inScopeNode); + + if (classLikeNode->m_super.hasContent()) + { + ScopeNode* parentScope = classLikeNode->m_parentScope; + if (parentScope == nullptr) + { + sink->diagnoseRaw( + Severity::Error, + UnownedStringSlice::fromLiteral("Can't lookup in scope if there is none!")); + return SLANG_FAIL; + } + + Node* superNode = Node::lookup(parentScope, classLikeNode->m_super.getContent()); + + if (!superNode) + { + if (classLikeNode->isReflected()) + { + sink->diagnose( + classLikeNode->m_name, + CPPDiagnostics::superTypeNotFound, + classLikeNode->getAbsoluteName()); + return SLANG_FAIL; + } + } + else + { + ClassLikeNode* superType = as<ClassLikeNode>(superNode); + + if (!superType) + { + sink->diagnose( + classLikeNode->m_name, + CPPDiagnostics::superTypeNotAType, + classLikeNode->getAbsoluteName()); + return SLANG_FAIL; + } + + if (superType->m_typeSet != classLikeNode->m_typeSet) + { + sink->diagnose( + classLikeNode->m_name, + CPPDiagnostics::typeInDifferentTypeSet, + classLikeNode->m_name.getContent(), + classLikeNode->m_typeSet->m_macroName, + superType->m_typeSet->m_macroName); + return SLANG_FAIL; + } + + // The base class must be defined in same scope (as we didn't allow different scopes + // for base classes) + superType->addDerived(classLikeNode); + } + } + else + { + // Add to it's own typeset + if (classLikeNode->isReflected() && classLikeNode->m_typeSet) + { + classLikeNode->m_typeSet->m_baseTypes.add(classLikeNode); + } + } + } + + for (Node* child : inScopeNode->m_children) + { + ScopeNode* childScope = as<ScopeNode>(child); + if (childScope) + { + SLANG_RETURN_ON_FAIL(_calcDerivedTypesRec(childScope, sink)); + } + } + + return SLANG_OK; +} + +SlangResult NodeTree::calcDerivedTypes(DiagnosticSink* sink) +{ + return _calcDerivedTypesRec(m_rootNode, sink); +} + + +} // namespace CppParse diff --git a/tools/slang-cpp-parser/node-tree.h b/tools/slang-cpp-parser/node-tree.h new file mode 100644 index 000000000..8911d0d71 --- /dev/null +++ b/tools/slang-cpp-parser/node-tree.h @@ -0,0 +1,101 @@ +#pragma once + +#include "compiler-core/slang-lexer.h" +#include "diagnostics.h" +#include "identifier-lookup.h" +#include "node.h" + +namespace CppParse +{ +using namespace Slang; + +class TypeSet : public RefObject +{ +public: + /// This is the looked up name. + UnownedStringSlice + m_macroName; ///< The name extracted from the macro SLANG_ABSTRACT_AST_CLASS -> AST + + String m_typeName; ///< The enum type name associated with this type for AST it is ASTNode + String m_fileMark; ///< This 'mark' becomes of the output filename + + List<ClassLikeNode*> m_baseTypes; ///< The base types for this type set +}; + +class SourceOrigin : public RefObject +{ +public: + void addNode(Node* node) + { + if (auto classLike = as<ClassLikeNode>(node)) + { + SLANG_ASSERT(classLike->m_origin == nullptr); + classLike->m_origin = this; + } + + m_nodes.add(node); + } + + SourceOrigin(SourceFile* sourceFile, const String& macroOrigin) + : m_sourceFile(sourceFile), m_macroOrigin(macroOrigin) + { + } + + String m_macroOrigin; ///< The macro text is inserted into the macro to identify the origin. It + ///< is based on the filename + SourceFile* m_sourceFile; ///< The source file - also holds the path information + + /// All of the nodes defined in this file in the order they were defined + /// Note that the same namespace may be listed multiple times. + List<RefPtr<Node>> m_nodes; +}; + +struct Options; +class IdentifierLookup; + +/* NodeTree holds nodes that have been parsed into a tree rooted on the 'rootNode'. +Also contains other state associated with or useful to a node tree */ +class NodeTree +{ +public: + friend class Parser; + /// Get all of the parsed source origins + const List<RefPtr<SourceOrigin>>& getSourceOrigins() const { return m_sourceOrigins; } + + TypeSet* getTypeSet(const UnownedStringSlice& slice); + TypeSet* getOrAddTypeSet(const UnownedStringSlice& slice); + + SourceOrigin* addSourceOrigin(SourceFile* sourceFile, const Options& options); + + /// Get all of the type sets + const List<RefPtr<TypeSet>>& getTypeSets() const { return m_typeSets; } + + /// Get the root node + Node* getRootNode() const { return m_rootNode; } + + /// When parsing we don't lookup all up super types/add derived types. This is because + /// we allow files to be processed in any order, so we have to do the type lookup as a separate + /// operation + SlangResult calcDerivedTypes(DiagnosticSink* sink); + + NodeTree(StringSlicePool* typePool, NamePool* namePool, IdentifierLookup* identifierLookup); + + static String calcMacroOrigin(const String& filePath, const Options& options); + +protected: + SlangResult _calcDerivedTypesRec(ScopeNode* node, DiagnosticSink* sink); + + StringSlicePool m_typeSetPool; ///< Pool for type set names + List<RefPtr<TypeSet>> m_typeSets; ///< The type sets + + IdentifierLookup* m_identifierLookup; + StringSlicePool* m_typePool; ///< Pool for just types + + NamePool* m_namePool; + + RefPtr<ScopeNode> m_rootNode; ///< The root scope + + List<RefPtr<SourceOrigin>> m_sourceOrigins; +}; + +} // namespace CppParse diff --git a/tools/slang-cpp-parser/node.cpp b/tools/slang-cpp-parser/node.cpp new file mode 100644 index 000000000..16484ead3 --- /dev/null +++ b/tools/slang-cpp-parser/node.cpp @@ -0,0 +1,700 @@ +#include "node.h" + +#include "core/slang-string-escape-util.h" +#include "core/slang-string-util.h" +#include "file-util.h" + +namespace CppParse +{ + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Node Impl +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +SLANG_FORCE_INLINE static void _indent(Index indentCount, StringBuilder& out) +{ + FileUtil::indent(indentCount, out); +} + +void Node::dumpMarkup(int indentCount, StringBuilder& out) +{ + if (m_markup.getLength() <= 0) + { + return; + } + + List<UnownedStringSlice> lines; + StringUtil::calcLines(m_markup.getUnownedSlice(), lines); + + // Remove empty lines from the end + while (lines.getCount()) + { + auto lastLine = lines.getLast(); + if (lastLine.trim().getLength() == 0) + { + lines.removeLast(); + continue; + } + break; + } + + if (lines.getCount() == 0) + { + return; + } + + for (auto line : lines) + { + _indent(indentCount, out); + out << "// " << line << "\n"; + } +} + +ScopeNode* Node::getRootScope() +{ + if (m_parentScope) + { + ScopeNode* scope = m_parentScope; + while (scope->m_parentScope) + { + scope = scope->m_parentScope; + } + return scope; + } + else + { + return as<ScopeNode>(this); + } +} + +void Node::calcScopeDepthFirst(List<Node*>& outNodes) +{ + outNodes.add(this); +} + +void Node::calcAbsoluteName(StringBuilder& outName) const +{ + List<Node*> path; + calcScopePath(const_cast<Node*>(this), path); + + // 1 so we skip the global scope + for (Index i = 1; i < path.getCount(); ++i) + { + Node* node = path[i]; + + if (i > 1) + { + outName << "::"; + } + + if (node->m_kind == Kind::AnonymousNamespace) + { + outName << "{Anonymous}"; + } + else + { + outName << node->m_name.getContent(); + } + } +} + +/* static */ void Node::calcScopePath(Node* node, List<Node*>& outPath) +{ + outPath.clear(); + + while (node) + { + outPath.add(node); + node = node->m_parentScope; + } + + // reverse the order, so we go from root to the node + outPath.reverse(); +} + +/* static */ void Node::filterImpl(Filter inFilter, List<Node*>& ioNodes) +{ + // Filter out all the unreflected nodes + Index count = ioNodes.getCount(); + for (Index j = 0; j < count;) + { + Node* node = ioNodes[j]; + + if (!inFilter(node)) + { + ioNodes.removeAt(j); + count--; + } + else + { + j++; + } + } +} + +/* static */ Node* Node::lookupNameInScope(ScopeNode* scope, const UnownedStringSlice& name) +{ + // TODO(JS): Doesn't handle 'using namespace'. + + // Must be unqualified name + SLANG_ASSERT(name.indexOf(UnownedStringSlice::fromLiteral("::")) < 0); + + Node* childNode = scope->findChild(name); + if (childNode) + { + return childNode; + } + + // If we have an anonymous namespace in this scope, try looking up in there.. + if (scope->m_anonymousNamespace) + { + Node* childNode = scope->m_anonymousNamespace->findChild(name); + if (childNode) + { + return childNode; + } + } + + // I could have an enum (that's not an enum class) + for (Node* node : scope->m_children) + { + EnumNode* enumNode = as<EnumNode>(node); + if (enumNode && enumNode->m_kind == Node::Kind::Enum) + { + Node** nodePtr = enumNode->m_childMap.tryGetValue(name); + if (nodePtr) + { + return *nodePtr; + } + } + } + + return nullptr; +} + +/* static */ Node* Node::lookupFromScope( + ScopeNode* scope, + const UnownedStringSlice* parts, + Index partsCount) +{ + SLANG_ASSERT(partsCount > 0); + if (partsCount == 1) + { + return lookupNameInScope(scope, parts[0]); + } + + for (Index i = 0; i < partsCount; ++i) + { + const UnownedStringSlice& part = parts[i]; + + Node* node = lookupNameInScope(scope, part); + if (node == nullptr) + { + return node; + } + // If at end, then we are done + if (i == partsCount - 1) + { + return node; + } + + // If there are more elements, then node must be some kind of scope, + // if we are going to find it + scope = as<ScopeNode>(node); + if (scope == nullptr) + { + break; + } + } + + return nullptr; +} + +/* static */ void Node::splitPath( + const UnownedStringSlice& inPath, + List<UnownedStringSlice>& outParts) +{ + if (inPath.indexOf(UnownedStringSlice::fromLiteral("::")) >= 0) + { + StringUtil::split(inPath, UnownedStringSlice::fromLiteral("::"), outParts); + // Remove any whitespace + for (auto& part : outParts) + { + part = part.trim(); + } + } + else + { + outParts.clear(); + outParts.add(inPath.trim()); + } +} + +/* static */ Node* Node::lookupFromScope(ScopeNode* scope, const UnownedStringSlice& inPath) +{ + if (inPath.indexOf(UnownedStringSlice::fromLiteral("::")) >= 0) + { + List<UnownedStringSlice> parts; + splitPath(inPath, parts); + + return lookupFromScope(scope, parts.getBuffer(), parts.getCount()); + } + else + { + return lookupNameInScope(scope, inPath); + } +} + +/* static */ Node* Node::lookup(ScopeNode* scope, const UnownedStringSlice& inPath) +{ + if (inPath.indexOf(UnownedStringSlice::fromLiteral("::")) >= 0) + { + List<UnownedStringSlice> parts; + splitPath(inPath, parts); + + if (parts[0].getLength() == 0) + { + // It's a lookup from global scope + ScopeNode* rootScope = scope->getRootScope(); + return lookupFromScope(rootScope, parts.getBuffer() + 1, parts.getCount() + 1); + } + + // Okay lets try a lookup from each scope up to the global scope + while (scope) + { + Node* node = lookupFromScope(scope, parts.getBuffer(), parts.getCount()); + if (node) + { + return node; + } + + scope = scope->m_parentScope; + } + } + else + { + while (scope) + { + // Lookup in this scope + Node* node = lookupNameInScope(scope, inPath); + if (node) + { + return node; + } + + // Try parent scope + scope = scope->m_parentScope; + } + } + + return nullptr; +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ScopeNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +ScopeNode* ScopeNode::getAnonymousNamespace() +{ + if (!m_anonymousNamespace) + { + m_anonymousNamespace = new ScopeNode(Kind::AnonymousNamespace); + m_anonymousNamespace->m_parentScope = this; + m_children.add(m_anonymousNamespace); + } + + return m_anonymousNamespace; +} + +void ScopeNode::addChildIgnoringName(Node* child) +{ + SLANG_ASSERT(child->m_parentScope == nullptr); + // Can't add anonymous namespace this way - should be added via getAnonymousNamespace + SLANG_ASSERT(child->m_kind != Kind::AnonymousNamespace); + + child->m_parentScope = this; + m_children.add(child); +} + +void ScopeNode::addChild(Node* child) +{ + addChildIgnoringName(child); + + if (child->m_name.hasContent()) + { + m_childMap.add(child->m_name.getContent(), child); + } +} + +Node* ScopeNode::findChild(const UnownedStringSlice& name) const +{ + Node* const* nodePtr = m_childMap.tryGetValue(name); + if (nodePtr) + { + return *nodePtr; + } + return nullptr; +} + +void ScopeNode::calcScopeDepthFirst(List<Node*>& outNodes) +{ + outNodes.add(this); + for (Node* child : m_children) + { + child->calcScopeDepthFirst(outNodes); + } +} + +void ScopeNode::dump(int indentCount, StringBuilder& out) +{ + dumpMarkup(indentCount, out); + + _indent(indentCount, out); + + switch (m_kind) + { + case Kind::AnonymousNamespace: + { + out << "namespace {\n"; + } + case Kind::Namespace: + { + if (m_name.hasContent()) + { + out << "namespace " << m_name.getContent() << " {\n"; + } + else + { + out << "{\n"; + } + break; + } + } + + for (Node* child : m_children) + { + child->dump(indentCount + 1, out); + } + + _indent(indentCount, out); + out << "}\n"; +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! EnumCaseNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +/* Returns true if needs space between the tokens. +It determines this based on the locs, and if they contain something between them. +*/ +static bool _needsSpace(const Token& prevTok, const Token& tok) +{ + auto prevLoc = prevTok.getLoc(); + auto loc = tok.getLoc(); + + auto prevContent = prevTok.getContent(); + + if (prevLoc + prevContent.getLength() == loc) + { + return false; + } + + return true; +} + + +static void _dumpTokens(const Token* toks, Index count, StringBuilder& out) +{ + if (count > 0) + { + out << toks[0].getContent(); + + for (Index i = 1; i < count; ++i) + { + const auto& prevToken = toks[i - 1]; + const auto& token = toks[i]; + + if (_needsSpace(prevToken, token)) + { + out << " "; + } + + out << token.getContent(); + } + } +} + +static void _dumpTokens(const List<Token>& toks, StringBuilder& out) +{ + _dumpTokens(toks.getBuffer(), toks.getCount(), out); +} + + +void EnumCaseNode::dump(int indent, StringBuilder& out) +{ + if (isReflected()) + { + dumpMarkup(indent, out); + + _indent(indent, out); + out << m_name.getContent(); + + if (m_valueTokens.getCount()) + { + out << " = "; + _dumpTokens(m_valueTokens, out); + } + + out << ",\n"; + } +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! EnumNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +void TypeDefNode::dump(int indent, StringBuilder& out) +{ + if (isReflected()) + { + dumpMarkup(indent, out); + + _indent(indent, out); + + out << "typedef "; + _dumpTokens(m_targetTypeTokens, out); + out << " " << m_name.getContent() << ";\n"; + } +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! EnumNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +void EnumNode::dump(int indent, StringBuilder& out) +{ + if (!isReflected()) + { + return; + } + + dumpMarkup(indent, out); + + _indent(indent, out); + + out << "enum "; + + if (m_kind == Kind::EnumClass) + { + out << "class "; + } + + if (m_name.type != TokenType::Invalid) + { + out << m_name.getContent(); + } + + if (m_backingTokens.getCount() > 0) + { + out << " : "; + _dumpTokens(m_backingTokens, out); + } + + out << "\n"; + _indent(indent, out); + out << "{\n"; + + for (Node* child : m_children) + { + child->dump(indent + 1, out); + } + + _indent(indent, out); + out << "}\n"; +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!! CallableNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +void CallableNode::dump(int indent, StringBuilder& out) +{ + if (!isReflected()) + { + return; + } + + dumpMarkup(indent, out); + + _indent(indent, out); + + if (m_isStatic) + { + out << "static "; + } + if (m_isVirtual) + { + out << "virtual "; + } + + out << m_returnType << " "; + out << m_name.getContent() << "("; + + const Index count = m_params.getCount(); + for (Index i = 0; i < count; ++i) + { + if (i > 0) + { + out << ", "; + } + + const auto& param = m_params[i]; + out << param.m_type; + if (param.m_name.type == TokenType::Identifier) + { + out << " " << param.m_name.getContent(); + } + } + + out << ")"; + + if (m_isPure) + { + out << " = 0"; + } + + out << "\n"; +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! FieldNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +void FieldNode::dump(int indent, StringBuilder& out) +{ + if (!isReflected()) + { + return; + } + + dumpMarkup(indent, out); + + _indent(indent, out); + + if (m_isStatic) + { + out << "static "; + } + + out << m_fieldType << " " << m_name.getContent() << "\n"; +} + +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ClassLikeNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +/// Add a node that is derived from this +void ClassLikeNode::addDerived(ClassLikeNode* derived) +{ + SLANG_ASSERT(derived->m_superNode == nullptr); + derived->m_superNode = this; + m_derivedTypes.add(derived); +} + +void ClassLikeNode::calcDerivedDepthFirst(List<ClassLikeNode*>& outNodes) +{ + outNodes.add(this); + for (ClassLikeNode* derivedType : m_derivedTypes) + { + derivedType->calcDerivedDepthFirst(outNodes); + } +} + +void ClassLikeNode::dumpDerived(int indentCount, StringBuilder& out) +{ + if (isClassLike() && isReflected() && m_name.hasContent()) + { + _indent(indentCount, out); + out << m_name.getContent() << "\n"; + } + + for (ClassLikeNode* derivedType : m_derivedTypes) + { + derivedType->dumpDerived(indentCount + 1, out); + } +} + +Index ClassLikeNode::calcDerivedDepth() const +{ + const ClassLikeNode* node = this; + Index count = 0; + + while (node) + { + count++; + node = node->m_superNode; + } + + return count; +} + +ClassLikeNode* ClassLikeNode::findLastDerived() +{ + for (Index i = m_derivedTypes.getCount() - 1; i >= 0; --i) + { + ClassLikeNode* derivedType = m_derivedTypes[i]; + ClassLikeNode* found = derivedType->findLastDerived(); + if (found) + { + return found; + } + } + return this; +} + +bool ClassLikeNode::hasReflectedDerivedType() const +{ + for (ClassLikeNode* type : m_derivedTypes) + { + if (type->isReflected()) + { + return true; + } + } + return false; +} + +void ClassLikeNode::getReflectedDerivedTypes(List<ClassLikeNode*>& out) const +{ + out.clear(); + for (ClassLikeNode* type : m_derivedTypes) + { + if (type->isReflected()) + { + out.add(type); + } + } +} + +void ClassLikeNode::dump(int indentCount, StringBuilder& out) +{ + dumpMarkup(indentCount, out); + + _indent(indentCount, out); + + const char* typeName = (m_kind == Kind::StructType) ? "struct" : "class"; + + out << typeName << " "; + + if (!isReflected()) + { + out << " ("; + } + out << m_name.getContent(); + if (!isReflected()) + { + out << ") "; + } + + if (m_super.hasContent()) + { + out << " : " << m_super.getContent(); + } + + out << " {\n"; + + for (Node* child : m_children) + { + child->dump(indentCount + 1, out); + } + + _indent(indentCount, out); + out << "}\n"; +} + +} // namespace CppParse diff --git a/tools/slang-cpp-parser/node.h b/tools/slang-cpp-parser/node.h new file mode 100644 index 000000000..4dad9473e --- /dev/null +++ b/tools/slang-cpp-parser/node.h @@ -0,0 +1,422 @@ +#pragma once + +#include "compiler-core/slang-doc-extractor.h" +#include "diagnostics.h" + +namespace CppParse +{ +using namespace Slang; + +enum class ReflectionType : uint8_t +{ + NotReflected, + Reflected, +}; + +// Pre-declare +class TypeSet; +class SourceOrigin; + +struct ScopeNode; + +class Node : public RefObject +{ +public: + enum class Kind : uint8_t + { + Invalid, + + StructType, + ClassType, + + Enum, + EnumClass, + + Namespace, + AnonymousNamespace, + + Field, + EnumCase, + + TypeDef, + + Callable, ///< Functions/methods + + Other, ///< Used 'other' parsing like for TYPE + Unknown, ///< Used for marking tokens consumed but usage is not known + + CountOf, + }; + + enum class KindRange + { + ScopeStart = int(Kind::StructType), + ScopeEnd = int(Kind::AnonymousNamespace), + + ClassLikeStart = int(Kind::StructType), + ClassLikeEnd = int(Kind::ClassType), + + ScopeTypeStart = int(Kind::StructType), + ScopeTypeEnd = int(Kind::EnumClass), + + OtherTypeStart = int(Kind::TypeDef), + OtherTypeEnd = int(Kind::TypeDef), + + EnumStart = int(Kind::Enum), + EnumEnd = int(Kind::EnumClass), + }; + + /// Returns true if kind can cast to this type + /// Used for implementing as<T> casting + static bool isOfKind(Kind kind) + { + SLANG_UNUSED(kind); + return true; + } + + static bool isKindScope(Kind kind) + { + return int(kind) >= int(KindRange::ScopeStart) && int(kind) <= int(KindRange::ScopeEnd); + } + static bool isKindClassLike(Kind kind) + { + return int(kind) >= int(KindRange::ClassLikeStart) && + int(kind) <= int(KindRange::ClassLikeEnd); + } + static bool isKindEnumLike(Kind kind) + { + return int(kind) >= int(KindRange::EnumStart) && int(kind) <= int(KindRange::EnumEnd); + } + + /// It a type, but doesn't have a scope + static bool isKindOtherType(Kind kind) + { + return int(kind) >= int(KindRange::OtherTypeStart) && + int(kind) <= int(KindRange::OtherTypeEnd); + } + /// Is a type and has a scope + static bool isKindScopeType(Kind kind) + { + return int(kind) >= int(KindRange::ScopeTypeStart) && + int(kind) <= int(KindRange::ScopeTypeEnd); + } + + /// True if the kind is any type + static bool isKindType(Kind kind) { return isKindOtherType(kind) || isKindScopeType(kind); } + + /// True if the kind can accept contained types + static bool canKindContainTypes(Kind type) + { + switch (type) + { + case Kind::StructType: + case Kind::ClassType: + case Kind::Namespace: + case Kind::AnonymousNamespace: + { + return true; + } + default: + break; + } + return false; + } + + bool isNamespace() const { return m_kind == Kind::Namespace; } + bool isClassLike() const { return isKindClassLike(m_kind); } + bool isScope() const { return isKindScope(m_kind); } + bool isEnumLike() const { return isKindEnumLike(m_kind); } + + /// These are useful for the filter + static bool isClassLikeAndReflected(Node* node) + { + return node->isClassLike() && node->isReflected(); + } + static bool isClassLike(Node* node) { return isKindClassLike(node->m_kind); } + + virtual void dump(int indent, StringBuilder& out) = 0; + + /// Do depth first traversal of nodes in scopes + virtual void calcScopeDepthFirst(List<Node*>& outNodes); + + /// Calculate the absolute name for this namespace/type + void calcAbsoluteName(StringBuilder& outName) const; + + /// Get the absolute name + String getAbsoluteName() const + { + StringBuilder buf; + calcAbsoluteName(buf); + return buf.produceString(); + } + + /// Calculate the scope path to this node, from the root + void calcScopePath(List<Node*>& outPath) { calcScopePath(this, outPath); } + + /// True if reflected + bool isReflected() const { return m_reflectionType == ReflectionType::Reflected; } + + SourceLoc getSourceLoc() const { return m_name.getLoc(); } + + ScopeNode* getRootScope(); + + typedef bool (*Filter)(Node* node); + + template<typename T> + static void filter(Filter filter, List<T*>& io) + { + const Node* _isNodeDerived = (T*)nullptr; + SLANG_UNUSED(_isNodeDerived); + filterImpl(filter, reinterpret_cast<List<Node*>&>(io)); + } + + static void filterImpl(Filter filter, List<Node*>& io); + + static void calcScopePath(Node* node, List<Node*>& outPath); + + /// Lookup a name in just the specified scope + /// Handles anonymous namespaces, or name lookups that are in the parents space + static Node* lookupNameInScope(ScopeNode* scope, const UnownedStringSlice& name); + + /// Lookup from a path + static Node* lookupFromScope(ScopeNode* scope, const UnownedStringSlice* path, Index pathCount); + /// Looks up *just* from the specified scope. + static Node* lookupFromScope(ScopeNode* scope, const UnownedStringSlice& slice); + + /// Look up name (which can contain ::) + static Node* lookup(ScopeNode* scope, const UnownedStringSlice& name); + + static void splitPath(const UnownedStringSlice& slice, List<UnownedStringSlice>& outSplitPath); + + /// If markup is specified dump it + void dumpMarkup(int indent, StringBuilder& out); + + Node(Kind type) + : m_kind(type), m_parentScope(nullptr), m_reflectionType(ReflectionType::NotReflected) + { + } + + Kind m_kind; ///< The kind of node this is + ReflectionType m_reflectionType; ///< Classes can be traversed, but not reflected. To be + ///< reflected they have to contain the marker + + MarkupVisibility m_markupVisibility = + MarkupVisibility::Public; ///< The visibility of the markup + String m_markup; ///< Documentation associated with this node + + Token m_name; ///< The name of this scope/type + + ScopeNode* m_parentScope; ///< The scope this type/scope is defined in +}; + +struct ScopeNode : public Node +{ + typedef Node Super; + + static bool isOfKind(Kind kind) { return isKindScope(kind); } + + virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; + virtual void calcScopeDepthFirst(List<Node*>& outNodes) SLANG_OVERRIDE; + + /// True if can contain callable entries + bool canContainCallable() const { return isClassLike() || isNamespace(); } + + /// True if can accept fields (class like types can) + bool canContainFields() const { return isClassLike(); } + + /// True if the scope can accept types + bool canContainTypes() const { return canKindContainTypes(m_kind); } + + /// Gets the reflection for any contained types + ReflectionType getContainedReflectionType() const + { + return m_reflectionType == ReflectionType::NotReflected ? ReflectionType::NotReflected + : m_reflectionOverride; + } + + /// Add a child node to this nodes scope + void addChild(Node* child); + /// Adds the child but does not add the name to the map + void addChildIgnoringName(Node* child); + + /// Find a child node in this scope with the specified name. Return nullptr if not found + Node* findChild(const UnownedStringSlice& name) const; + + /// Gets the anonymous namespace associated with this scope + ScopeNode* getAnonymousNamespace(); + + ScopeNode(Kind kind) + : Super(kind) + , m_reflectionOverride(ReflectionType::Reflected) + , m_anonymousNamespace(nullptr) + { + } + + /// For child types, fields, how reflection is handled. If this type is not reflected + ReflectionType m_reflectionOverride; + + /// All of the types and namespaces in this *scope* + List<RefPtr<Node>> m_children; + + /// Map from a name (in this scope) to the Node + Dictionary<UnownedStringSlice, Node*> m_childMap; + + /// There can only be one anonymousNamespace for a scope. If there is one it's held here + ScopeNode* m_anonymousNamespace; +}; + +struct FieldNode : public Node +{ + typedef Node Super; + + static bool isOfKind(Kind kind) { return kind == Kind::Field; } + + virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; + + FieldNode() + : Super(Kind::Field) + { + } + + UnownedStringSlice m_fieldType; + + bool m_isStatic = false; + + /// TODO(JS): We may want to add initializer tokens +}; + +struct ClassLikeNode : public ScopeNode +{ + typedef ScopeNode Super; + + static bool isOfKind(Kind kind) { return isKindClassLike(kind); } + + /// Add a node that is derived from this + void addDerived(ClassLikeNode* derived); + + /// Dump all of the derived types + void dumpDerived(int indentCount, StringBuilder& out); + + /// Calculates the derived depth + Index calcDerivedDepth() const; + + /// Find the last (reflected) derived type + ClassLikeNode* findLastDerived(); + + /// Traverse the hierarchy of derived nodes, in depth first order + void calcDerivedDepthFirst(List<ClassLikeNode*>& outNodes); + + /// True if has a derived type that is reflected + bool hasReflectedDerivedType() const; + + /// Stores in out any reflected derived types + void getReflectedDerivedTypes(List<ClassLikeNode*>& out) const; + + // Node Impl + virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; + + ClassLikeNode(Kind kind) + : Super(kind), m_origin(nullptr), m_typeSet(nullptr), m_superNode(nullptr) + { + SLANG_ASSERT(kind == Kind::ClassType || kind == Kind::StructType); + } + + SourceOrigin* m_origin; ///< Defines where this was uniquely defined. + + Token m_marker; ///< The marker associated with this scope (typically the marker is SLANG_CLASS + ///< etc, that is used to identify reflectedType) + + List<RefPtr<ClassLikeNode>> m_derivedTypes; ///< All of the types derived from this type + + TypeSet* m_typeSet; ///< The typeset this type belongs to. + + Token m_super; ///< Super class name + ClassLikeNode* m_superNode; ///< If this is a class/struct, the type it is derived from (or + ///< nullptr if base) +}; + +struct CallableNode : public Node +{ + typedef Node Super; + + static bool isOfKind(Kind kind) { return kind == Kind::Callable; } + + virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; + + CallableNode() + : Super(Kind::Callable) + { + } + + struct Param + { + UnownedStringSlice m_type; + Token m_name; + }; + + UnownedStringSlice m_returnType; + + CallableNode* m_nextOverload = nullptr; + + List<Param> m_params; + + bool m_isStatic = false; + bool m_isVirtual = false; + bool m_isPure = false; +}; + +struct EnumCaseNode : public Node +{ + typedef Node Super; + + static bool isOfKind(Kind kind) { return kind == Kind::EnumCase; } + + virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; + + EnumCaseNode() + : Super(Kind::EnumCase) + { + } + + // Tokens that make up the value. If not defined will be empty + List<Token> m_valueTokens; +}; + +struct EnumNode : public ScopeNode +{ + typedef ScopeNode Super; + static bool isOfKind(Kind kind) { return isKindEnumLike(kind); } + + virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; + + EnumNode(Kind kind) + : Super(kind) + { + SLANG_ASSERT(isKindEnumLike(kind)); + } + + List<Token> m_backingTokens; +}; + +struct TypeDefNode : public Node +{ + typedef Node Super; + static bool isOfKind(Kind kind) { return kind == Kind::TypeDef; } + + virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; + + TypeDefNode() + : Super(Kind::TypeDef) + { + } + + List<Token> m_targetTypeTokens; +}; + +template<typename T> +T* as(Node* node) +{ + return (node && T::isOfKind(node->m_kind)) ? static_cast<T*>(node) : nullptr; +} + +} // namespace CppParse diff --git a/tools/slang-cpp-parser/options.cpp b/tools/slang-cpp-parser/options.cpp new file mode 100644 index 000000000..8be319e7b --- /dev/null +++ b/tools/slang-cpp-parser/options.cpp @@ -0,0 +1,153 @@ +#include "options.h" + +#include "diagnostics.h" + +namespace CppParse +{ + +SlangResult OptionsParser::_parseArgFlag(const char* option, bool& outFlag) +{ + SLANG_ASSERT(UnownedStringSlice(m_args[m_index]) == option); + SLANG_ASSERT(m_index < m_argCount); + + m_index++; + outFlag = true; + return SLANG_OK; +} + +SlangResult OptionsParser::_parseArgWithValue(const char* option, String& ioValue) +{ + SLANG_ASSERT(UnownedStringSlice(m_args[m_index]) == option); + if (m_index + 1 < m_argCount) + { + // Next parameter is the output path, there can only be one + if (ioValue.getLength()) + { + // There already is output + m_sink->diagnose(SourceLoc(), CPPDiagnostics::optionAlreadyDefined, option, ioValue); + return SLANG_FAIL; + } + } + else + { + m_sink->diagnose(SourceLoc(), CPPDiagnostics::requireValueAfterOption, option); + return SLANG_FAIL; + } + + ioValue = m_args[m_index + 1]; + m_index += 2; + return SLANG_OK; +} + +SlangResult OptionsParser::_parseArgReplaceValue(const char* option, String& ioValue) +{ + SLANG_ASSERT(UnownedStringSlice(m_args[m_index]) == option); + if (m_index + 1 >= m_argCount) + { + m_sink->diagnose(SourceLoc(), CPPDiagnostics::requireValueAfterOption, option); + return SLANG_FAIL; + } + + ioValue = m_args[m_index + 1]; + m_index += 2; + return SLANG_OK; +} + +SlangResult OptionsParser::parse( + int argc, + const char* const* argv, + DiagnosticSink* sink, + Options& outOptions) +{ + outOptions.reset(); + + m_index = 0; + m_argCount = argc; + m_args = argv; + m_sink = sink; + + outOptions.reset(); + + while (m_index < m_argCount) + { + const UnownedStringSlice arg = UnownedStringSlice(argv[m_index]); + + if (arg.getLength() > 0 && arg[0] == '-') + { + if (arg == "-d") + { + SLANG_RETURN_ON_FAIL(_parseArgWithValue("-d", outOptions.m_inputDirectory)); + continue; + } + else if (arg == "-o") + { + SLANG_RETURN_ON_FAIL(_parseArgWithValue("-o", outOptions.m_outputPath)); + continue; + } + else if (arg == "-dump") + { + SLANG_RETURN_ON_FAIL(_parseArgFlag("-dump", outOptions.m_dump)); + continue; + } + else if (arg == "-mark-prefix") + { + SLANG_RETURN_ON_FAIL( + _parseArgReplaceValue("-mark-prefix", outOptions.m_markPrefix)); + continue; + } + else if (arg == "-mark-suffix") + { + SLANG_RETURN_ON_FAIL( + _parseArgReplaceValue("-mark-suffix", outOptions.m_markSuffix)); + continue; + } + else if (arg == "-defs") + { + SLANG_RETURN_ON_FAIL(_parseArgFlag("-defs", outOptions.m_defs)); + continue; + } + else if (arg == "-output-fields") + { + SLANG_RETURN_ON_FAIL(_parseArgFlag("-output-fields", outOptions.m_outputFields)); + continue; + } + else if (arg == "-strip-prefix") + { + SLANG_RETURN_ON_FAIL( + _parseArgWithValue("-strip-prefix", outOptions.m_stripFilePrefix)); + continue; + } + else if (arg == "-unit-test") + { + SLANG_RETURN_ON_FAIL(_parseArgFlag("-unit-test", outOptions.m_runUnitTests)); + continue; + } + else if (arg == "-unmarked") + { + bool unmarked; + SLANG_RETURN_ON_FAIL(_parseArgFlag("-unmarked", unmarked)); + outOptions.m_requireMark = !unmarked; + continue; + } + + m_sink->diagnose(SourceLoc(), CPPDiagnostics::unknownOption, arg); + return SLANG_FAIL; + } + else + { + // If it doesn't start with - then it's assumed to be an input path + outOptions.m_inputPaths.add(arg); + m_index++; + } + } + + if (outOptions.m_inputPaths.getCount() < 0) + { + m_sink->diagnose(SourceLoc(), CPPDiagnostics::noInputPathsSpecified); + return SLANG_FAIL; + } + + return SLANG_OK; +} + +} // namespace CppParse diff --git a/tools/slang-cpp-parser/options.h b/tools/slang-cpp-parser/options.h new file mode 100644 index 000000000..c1cbb1a27 --- /dev/null +++ b/tools/slang-cpp-parser/options.h @@ -0,0 +1,63 @@ +#pragma once + +#include "slang/slang-diagnostics.h" + +namespace CppParse +{ +using namespace Slang; + + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Options !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +struct Options +{ + void reset() { *this = Options(); } + + Options() + { + m_markPrefix = "SLANG_"; + m_markSuffix = "_CLASS"; + } + + bool m_defs = false; ///< If set will output a '-defs.h' file for each of the input files, that + ///< corresponds to previous defs files (although doesn't have fields/RAW) + bool m_dump = + false; ///< If true will dump to stderr the types/fields and hierarchy it extracted + bool m_runUnitTests = false; ///< If true will run internal unit tests + bool m_extractDoc = true; ///< If set will try to extract documentation associated with nodes + + bool m_outputFields = false; ///< When dumping macros also dump field definitions + bool m_requireMark = true; + + List<String> m_inputPaths; ///< The input paths to the files to be processed + + String m_outputPath; ///< The output path. Note that the extractor can generate multiple output + ///< files, and this will actually be the 'stem' of several files + + String m_inputDirectory; ///< The input directory that is by default used for reading + ///< m_inputPaths from. + String m_markPrefix; ///< The prefix of the 'marker' used to identify a reflected type + String m_markSuffix; ///< The postfix of the 'marker' used to identify a reflected type + String m_stripFilePrefix; ///< Used for the 'origin' information, this is stripped from the + ///< source filename, and the remainder of the filename (without + ///< extension) is 'macroized' +}; + +struct OptionsParser +{ + /// Parse the parameters. NOTE! Must have the program path removed + SlangResult parse(int argc, const char* const* argv, DiagnosticSink* sink, Options& outOptions); + + SlangResult _parseArgWithValue(const char* option, String& outValue); + SlangResult _parseArgReplaceValue(const char* option, String& outValue); + SlangResult _parseArgFlag(const char* option, bool& outFlag); + + String m_reflectType; + + Index m_index; + Int m_argCount; + const char* const* m_args; + DiagnosticSink* m_sink; +}; + +} // namespace CppParse diff --git a/tools/slang-cpp-parser/parser.cpp b/tools/slang-cpp-parser/parser.cpp new file mode 100644 index 000000000..1edc38ac7 --- /dev/null +++ b/tools/slang-cpp-parser/parser.cpp @@ -0,0 +1,2261 @@ +#include "parser.h" + +#include "compiler-core/slang-name-convention-util.h" +#include "core/slang-io.h" +#include "core/slang-string-util.h" +#include "identifier-lookup.h" +#include "options.h" + +namespace CppParse +{ +using namespace Slang; + +// If fails then we need more bits to identify types +SLANG_COMPILE_TIME_ASSERT(int(Node::Kind::CountOf) <= 8 * sizeof(uint32_t)); + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Parser !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +Parser::Parser(NodeTree* nodeTree, DiagnosticSink* sink) + : m_sink(sink), m_nodeTree(nodeTree), m_nodeTypeEnabled(0) +{ + // Enable types by default + const Node::Kind defaultEnabled[] = { + Node::Kind::ClassType, + Node::Kind::StructType, + Node::Kind::Namespace, + Node::Kind::AnonymousNamespace, + Node::Kind::Field, + + // These are disabled by default because AST uses macro magic to build up the types + // Node::Type::TypeDef, + // Node::Type::Enum, + // Node::Type::EnumClass, + + Node::Kind::Callable, + }; + setKindsEnabled(defaultEnabled, SLANG_COUNT_OF(defaultEnabled)); +} + +void Parser::setKindEnabled(Node::Kind kind, bool isEnabled) +{ + if (isEnabled) + { + m_nodeTypeEnabled |= (NodeTypeBitType(1) << int(kind)); + } + else + { + m_nodeTypeEnabled &= ~(NodeTypeBitType(1) << int(kind)); + } +} + +void Parser::setKindsEnabled(const Node::Kind* kinds, Index kindsCount, bool isEnabled) +{ + for (Index i = 0; i < kindsCount; ++i) + { + setKindEnabled(kinds[i], isEnabled); + } +} + +bool Parser::_isMarker(const UnownedStringSlice& name) +{ + return name.startsWith(m_options->m_markPrefix.getUnownedSlice()) && + name.endsWith(m_options->m_markSuffix.getUnownedSlice()); +} + +SlangResult Parser::expect(TokenType type, Token* outToken) +{ + if (m_reader.peekTokenType() != type) + { + m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::expectingToken, type); + return SLANG_FAIL; + } + + if (outToken) + { + *outToken = m_reader.advanceToken(); + } + else + { + m_reader.advanceToken(); + } + return SLANG_OK; +} + +bool Parser::advanceIfToken(TokenType type, Token* outToken) +{ + if (m_reader.peekTokenType() == type) + { + Token token = m_reader.advanceToken(); + if (outToken) + { + *outToken = token; + } + return true; + } + return false; +} + +bool Parser::advanceIfMarker(Token* outToken) +{ + const Token peekToken = m_reader.peekToken(); + if (peekToken.type == TokenType::Identifier && _isMarker(peekToken.getContent())) + { + m_reader.advanceToken(); + if (outToken) + { + *outToken = peekToken; + } + return true; + } + return false; +} + +bool Parser::advanceIfStyle(IdentifierStyle style, Token* outToken) +{ + if (m_reader.peekTokenType() == TokenType::Identifier) + { + IdentifierStyle readStyle = + m_nodeTree->m_identifierLookup->get(m_reader.peekToken().getContent()); + if (readStyle == style) + { + Token token = m_reader.advanceToken(); + if (outToken) + { + *outToken = token; + } + return true; + } + } + return false; +} + + +SlangResult Parser::pushAnonymousNamespace() +{ + m_currentScope = m_currentScope->getAnonymousNamespace(); + + if (m_sourceOrigin) + { + m_sourceOrigin->addNode(m_currentScope); + } + + // Add the to the scope stack so can pop. + m_scopeStack.add(m_currentScope); + + return SLANG_OK; +} + +SlangResult Parser::pushScope(ScopeNode* scopeNode) +{ + // We can only have one 'special' scope. + SLANG_ASSERT(scopeNode || m_scopeStack.getLast()); + + // We keep to track. + m_scopeStack.add(scopeNode); + + // If we pass nullptr, we don't update the current scope. + if (scopeNode == nullptr) + { + return SLANG_OK; + } + + if (m_sourceOrigin) + { + m_sourceOrigin->addNode(scopeNode); + } + + if (scopeNode->m_name.hasContent()) + { + // For anonymous namespace, we should look if we already have one and just reopen that. + // Doing so will mean will find anonymous namespace clashes + + if (Node* foundNode = m_currentScope->findChild(scopeNode->m_name.getContent())) + { + if (scopeNode->isClassLike()) + { + m_sink->diagnose( + m_reader.peekToken(), + CPPDiagnostics::typeAlreadyDeclared, + scopeNode->m_name.getContent()); + m_sink->diagnose( + foundNode->m_name, + CPPDiagnostics::seeDeclarationOf, + scopeNode->m_name.getContent()); + return SLANG_FAIL; + } + + if (foundNode->m_kind == Node::Kind::Namespace) + { + if (foundNode->m_kind != scopeNode->m_kind) + { + // Different types can't work + m_sink->diagnose( + m_reader.peekToken(), + CPPDiagnostics::typeAlreadyDeclared, + scopeNode->m_name.getContent()); + return SLANG_FAIL; + } + + ScopeNode* foundScopeNode = as<ScopeNode>(foundNode); + SLANG_ASSERT(foundScopeNode); + + // Make sure the node is empty, as we are *not* going to add it, we are just going + // to use the pre-existing namespace + SLANG_ASSERT(scopeNode->m_children.getCount() == 0); + + // We can just use the pre-existing namespace + m_currentScope = foundScopeNode; + return SLANG_OK; + } + } + } + + m_currentScope->addChild(scopeNode); + m_currentScope = scopeNode; + return SLANG_OK; +} + +SlangResult Parser::popScope() +{ + if (m_scopeStack.getCount() <= 0) + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::scopeNotClosed); + return SLANG_FAIL; + } + + ScopeNode* topScope = m_scopeStack.getLast(); + m_scopeStack.removeLast(); + + // If the top is nullptr, we don't change the current scope + if (topScope == nullptr) + { + return SLANG_OK; + } + + m_currentScope = m_currentScope->m_parentScope; + return SLANG_OK; +} + +SlangResult Parser::_maybeConsumeScope() +{ + // Look for either ; or { to open scope + while (true) + { + const TokenType type = m_reader.peekTokenType(); + if (type == TokenType::Semicolon) + { + m_reader.advanceToken(); + return SLANG_OK; + } + else if (type == TokenType::LBrace) + { + // m_reader.advanceToken(); + return consumeToClosingBrace(); + } + else if (type == TokenType::EndOfFile) + { + return SLANG_OK; + } + + m_reader.advanceToken(); + } +} + +SlangResult Parser::consumeToClosingBrace(const Token* inOpenBraceToken) +{ + Token openToken; + if (inOpenBraceToken) + { + openToken = *inOpenBraceToken; + } + else + { + openToken = m_reader.advanceToken(); + } + SLANG_ASSERT(openToken.type == TokenType::LBrace); + + while (true) + { + switch (m_reader.peekTokenType()) + { + case TokenType::EndOfFile: + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::didntFindMatchingBrace); + m_sink->diagnose(openToken, CPPDiagnostics::seeOpen); + return SLANG_FAIL; + } + case TokenType::LBrace: + { + SLANG_RETURN_ON_FAIL(consumeToClosingBrace()); + break; + } + case TokenType::RBrace: + { + m_reader.advanceToken(); + return SLANG_OK; + } + default: + { + m_reader.advanceToken(); + break; + } + } + } +} + + +SlangResult Parser::_parseEnum() +{ + // We are looking for + // enum ([class name] | [name]) [: base] ( { | ; ) + + Token enumToken; + + // consume enum + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &enumToken)); + + if (!m_currentScope->canContainTypes()) + { + m_sink->diagnose(enumToken.loc, CPPDiagnostics::cannotDeclareTypeInScope); + return SLANG_FAIL; + } + + Node::Kind kind = Node::Kind::Enum; + + Token nameToken; + if (advanceIfToken(TokenType::Identifier, &nameToken)) + { + const IdentifierStyle style = m_nodeTree->m_identifierLookup->get(nameToken.getContent()); + + if (style == IdentifierStyle::Class) + { + kind = Node::Kind::EnumClass; + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &nameToken)); + } + else if (style == IdentifierStyle::None) + { + // It holds the name then + } + else + { + m_sink->diagnose( + nameToken.loc, + CPPDiagnostics::expectingIdentifier, + nameToken.getContent()); + return SLANG_FAIL; + } + } + + RefPtr<EnumNode> node = new EnumNode(kind); + node->m_name = nameToken; + node->m_reflectionType = m_currentScope->getContainedReflectionType(); + + if (advanceIfToken(TokenType::Colon)) + { + // We may have tokens up to { or ; + List<Token> backingTokens; + + while (true) + { + TokenType tokenType = m_reader.peekTokenType(); + if (tokenType == TokenType::Semicolon || tokenType == TokenType::LBrace || + tokenType == TokenType::EndOfFile) + { + break; + } + + backingTokens.add(m_reader.advanceToken()); + } + + // TODO - Look up the backing type. It can only be an integral. We can assume it must be + // defined before lookup for our uses here. If we can't find the type, we could assume it's + // size is undefined + + if (backingTokens.getCount() > 0) + { + node->m_backingTokens.swapWith(backingTokens); + } + } + + pushScope(node); + + if (advanceIfToken(TokenType::Semicolon)) + { + if (nameToken.type != TokenType::Invalid) + { + Node* node = m_currentScope->findChild(nameToken.getContent()); + if (node) + { + // Strictly speaking we should check the backing type etc, match, but for now ignore + // and assume it's ok + + if (node->m_kind == kind) + { + return SLANG_OK; + } + m_sink->diagnose( + nameToken.loc, + CPPDiagnostics::typeAlreadyDeclared, + nameToken.getContent()); + return SLANG_FAIL; + } + return popScope(); + } + } + + SLANG_RETURN_ON_FAIL(expect(TokenType::LBrace)); + + while (true) + { + TokenType tokenType = m_reader.peekTokenType(); + if (tokenType == TokenType::RBrace) + { + break; + } + + RefPtr<EnumCaseNode> caseNode(new EnumCaseNode); + + // We could also check if the name is a valid identifier for name, for now just assume. + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &caseNode->m_name)); + + if (node->findChild(caseNode->m_name.getContent())) + { + m_sink->diagnose( + caseNode->m_name.loc, + CPPDiagnostics::identifierAlreadyDefined, + caseNode->m_name.getContent()); + return SLANG_FAIL; + } + + caseNode->m_reflectionType = m_currentScope->getContainedReflectionType(); + + // Add the value + node->addChild(caseNode); + + if (advanceIfToken(TokenType::OpAssign)) + { + List<Token> valueTokens; + SLANG_RETURN_ON_FAIL(_parseExpression(valueTokens)); + + if (valueTokens.getCount() > 0) + { + caseNode->m_valueTokens.swapWith(valueTokens); + } + } + + tokenType = m_reader.peekTokenType(); + if (tokenType == TokenType::Comma) + { + m_reader.advanceToken(); + continue; + } + + break; + } + + SLANG_RETURN_ON_FAIL(expect(TokenType::RBrace)); + SLANG_RETURN_ON_FAIL(expect(TokenType::Semicolon)); + + return popScope(); +} + +SlangResult Parser::_consumeTemplate() +{ + // Skip the current 'template' token. + m_reader.advanceToken(); + + // Consume everything in <> + SLANG_RETURN_ON_FAIL(expect(TokenType::OpLess)); + + { + Index arrowCount = 1; + while (true) + { + auto tokenType = m_reader.peekTokenType(); + + if (tokenType == TokenType::OpLess) + { + m_reader.advanceToken(); + arrowCount++; + } + else if (tokenType == TokenType::OpGreater) + { + m_reader.advanceToken(); + if (arrowCount == 1) + { + break; + } + --arrowCount; + } + else if (tokenType == TokenType::OpRsh) + { + if (arrowCount < 2) + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::unexpectedTemplateClose); + return SLANG_FAIL; + } + m_reader.advanceToken(); + if (arrowCount == 2) + { + break; + } + arrowCount -= 2; + } + else if (tokenType == TokenType::EndOfFile) + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::unexpectedEndOfFile); + return SLANG_FAIL; + } + else + { + m_reader.advanceToken(); + } + } + } + + // Search for { or ; to consume remaining + while (true) + { + auto tokenType = m_reader.peekTokenType(); + + switch (tokenType) + { + case TokenType::EndOfFile: + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::unexpectedEndOfFile); + return SLANG_FAIL; + } + case TokenType::Semicolon: + { + // Ends with semicolon if it's a template pre-declaration + m_reader.advanceToken(); + return SLANG_OK; + } + case TokenType::LBrace: + { + // If ends with {, means could be body of a struct/class or a body of a + // function/method. Consume it + SLANG_RETURN_ON_FAIL(consumeToClosingBrace()); + // If we hit a ; just consume and ignore + advanceIfToken(TokenType::Semicolon); + return SLANG_OK; + } + default: + { + // Consume + m_reader.advanceToken(); + break; + } + } + } +} + +SlangResult Parser::_maybeParseNode(Node::Kind kind) +{ + // We are looking for + // struct/class identifier [: [public|private|protected] Identifier ] { + // [public|private|proctected:]* marker ( identifier ); + + if (kind == Node::Kind::Namespace) + { + // consume namespace + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier)); + + Token name; + if (advanceIfToken(TokenType::LBrace)) + { + return pushAnonymousNamespace(); + } + else if (advanceIfToken(TokenType::Identifier, &name)) + { + if (advanceIfToken(TokenType::LBrace)) + { + // Okay looks like we are opening a namespace + RefPtr<ScopeNode> node(new ScopeNode(Node::Kind::Namespace)); + node->m_name = name; + + node->m_reflectionType = m_currentScope->getContainedReflectionType(); + // Push the node + return pushScope(node); + } + } + + // Just ignore it then + return SLANG_OK; + } + else if (Node::isKindEnumLike(kind)) + { + return _parseEnum(); + } + + // Must be class | struct + + SLANG_ASSERT(kind == Node::Kind::ClassType || kind == Node::Kind::StructType); + + Token name; + + // consume class | struct + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier)); + // Next is the class name + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &name)); + + + if (m_reader.peekTokenType() == TokenType::Semicolon) + { + // pre declaration; + return SLANG_OK; + } + + RefPtr<ClassLikeNode> node(new ClassLikeNode(kind)); + node->m_name = name; + + // We default to the containing scope for reflection type. + if (!m_options->m_requireMark) + { + node->m_reflectionType = m_currentScope->getContainedReflectionType(); + } + else + { + // Defaults to not reflected + SLANG_ASSERT(!node->isReflected()); + } + + if (advanceIfToken(TokenType::Colon)) + { + // Could have public + advanceIfStyle(IdentifierStyle::Access); + + if (!advanceIfToken(TokenType::Identifier, &node->m_super)) + { + return SLANG_OK; + } + } + + // We only accept a single super class. Consume everything afterwards until we hit the { brace + + if (m_reader.peekTokenType() != TokenType::LBrace) + { + // Consume up until we see a brace else it's an error + while (true) + { + const TokenType peekTokenType = m_reader.peekTokenType(); + if (peekTokenType == TokenType::EndOfFile) + { + // Expecting brace + m_sink->diagnose( + m_reader.peekToken(), + CPPDiagnostics::expectingToken, + TokenType::LBrace); + return SLANG_FAIL; + } + else if (peekTokenType == TokenType::LBrace) + { + break; + } + m_reader.advanceToken(); + } + + return pushScope(node); + } + + const Token braceToken = m_reader.advanceToken(); + + // Push the class scope + return pushScope(node); +} + +SlangResult Parser::_consumeToSync() +{ + while (true) + { + TokenType type = m_reader.peekTokenType(); + + switch (type) + { + case TokenType::Semicolon: + { + m_reader.advanceToken(); + return SLANG_OK; + } + case TokenType::Pound: + case TokenType::EndOfFile: + case TokenType::LBrace: + case TokenType::RBrace: + { + return SLANG_OK; + } + } + + m_reader.advanceToken(); + } +} + +SlangResult Parser::_maybeParseTemplateArg(Index& ioTemplateDepth) +{ + switch (m_reader.peekTokenType()) + { + case TokenType::Identifier: + { + TokenReader::ParsingCursor nameCursor; + SLANG_RETURN_ON_FAIL(_maybeParseType(ioTemplateDepth, nameCursor)); + return SLANG_OK; + } + case TokenType::IntegerLiteral: + { + m_reader.advanceToken(); + return SLANG_OK; + } + default: + break; + } + return SLANG_FAIL; +} + +SlangResult Parser::_maybeParseTemplateArgs(Index& ioTemplateDepth) +{ + if (!advanceIfToken(TokenType::OpLess)) + { + return SLANG_FAIL; + } + + ioTemplateDepth++; + + while (true) + { + if (ioTemplateDepth == 0) + { + return SLANG_OK; + } + + switch (m_reader.peekTokenType()) + { + case TokenType::OpGreater: + { + if (ioTemplateDepth <= 0) + { + m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::unexpectedTemplateClose); + return SLANG_FAIL; + } + ioTemplateDepth--; + m_reader.advanceToken(); + return SLANG_OK; + } + case TokenType::OpRsh: + { + if (ioTemplateDepth <= 1) + { + m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::unexpectedTemplateClose); + return SLANG_FAIL; + } + ioTemplateDepth -= 2; + m_reader.advanceToken(); + return SLANG_OK; + } + default: + { + while (true) + { + SLANG_RETURN_ON_FAIL(_maybeParseTemplateArg(ioTemplateDepth)); + + if (m_reader.peekTokenType() == TokenType::Comma) + { + m_reader.advanceToken(); + // If there is a comma parse another arg + continue; + } + break; + } + break; + } + } + } +} + +SlangResult Parser::_maybeConsume(IdentifierStyle style) +{ + while (advanceIfStyle(style)) + ; + return SLANG_OK; +} + +// True if two of these token types of the same type placed immediately after one another +// produce a different token. Can be conservative, as if not strictly required +// it will just mean more spacing in the output +static bool _canRepeatTokenType(TokenType type) +{ + switch (type) + { + case TokenType::OpAdd: + case TokenType::OpSub: + case TokenType::OpAnd: + case TokenType::OpOr: + case TokenType::OpGreater: + case TokenType::OpLess: + case TokenType::Identifier: + case TokenType::OpAssign: + case TokenType::Colon: + { + return false; + } + default: + break; + } + return true; +} + +// Returns true if there needs to be a space between the previous token type, and the current token +// type for correct output. It is assumed that the token stream is appropriate. +// The implementation might need more sophistication, but this at least avoids Blah const * -> +// Blahconst* +static bool _tokenConcatNeedsSpace(TokenType prev, TokenType cur) +{ + if ((cur == TokenType::OpAssign) || (prev == cur && !_canRepeatTokenType(cur))) + { + return true; + } + return false; +} + +void Parser::_getTypeTokens( + TokenReader::ParsingCursor start, + TokenReader::ParsingCursor nameCursor, + List<Token>& outToks) +{ + auto endCursor = m_reader.getCursor(); + m_reader.setCursor(start); + + while (!m_reader.isAtCursor(endCursor)) + { + if (m_reader.getCursor() == nameCursor) + { + m_reader.advanceToken(); + } + else + { + outToks.add(m_reader.advanceToken()); + } + } +} + +UnownedStringSlice Parser::_concatType( + TokenReader::ParsingCursor start, + TokenReader::ParsingCursor nameCursor) +{ + List<Token> toks; + _getTypeTokens(start, nameCursor, toks); + return _concatTokens(toks.getBuffer(), toks.getCount()); +} + +UnownedStringSlice Parser::_concatTokens(const Token* toks, Index toksCount) +{ + StringBuilder buf; + + TokenType prevTokenType = TokenType::Unknown; + for (Index i = 0; i < toksCount; ++i) + { + const auto token = toks[i]; + + // Check if we need a space between tokens + if (_tokenConcatNeedsSpace(prevTokenType, token.type)) + { + buf << " "; + } + + buf << token.getContent(); + + prevTokenType = token.type; + } + + StringSlicePool* typePool = m_nodeTree->m_typePool; + return typePool->getSlice(typePool->add(buf)); +} + +UnownedStringSlice Parser::_concatTokens(TokenReader::ParsingCursor start) +{ + auto endCursor = m_reader.getCursor(); + + m_reader.setCursor(start); + + TokenType prevTokenType = TokenType::Unknown; + + StringBuilder buf; + while (!m_reader.isAtCursor(endCursor)) + { + const Token token = m_reader.advanceToken(); + // Check if we need a space between tokens + if (_tokenConcatNeedsSpace(prevTokenType, token.type)) + { + buf << " "; + } + buf << token.getContent(); + + prevTokenType = token.type; + } + + StringSlicePool* typePool = m_nodeTree->m_typePool; + return typePool->getSlice(typePool->add(buf)); +} + +SlangResult Parser::_maybeParseType( + Index& ioTemplateDepth, + TokenReader::ParsingCursor& outNameCursor) +{ + outNameCursor = TokenReader::ParsingCursor(); + + while (true) + { + if (m_reader.peekTokenType() == TokenType::Identifier) + { + const IdentifierStyle style = + m_nodeTree->m_identifierLookup->get(m_reader.peekToken().getContent()); + + if (style == IdentifierStyle::TypeModifier || + style == IdentifierStyle::IntegerModifier || style == IdentifierStyle::Class || + style == IdentifierStyle::Struct) + { + // These are ok keywords in this context + } + else if (hasFlag(style, IdentifierFlag::Keyword)) + { + return SLANG_FAIL; + } + } + + _maybeConsume(IdentifierStyle::TypeModifier); + + if (advanceIfStyle(IdentifierStyle::IntegerModifier)) + { + // Consume the integer typename (if there is one) + const Token peekToken = m_reader.peekToken(); + if (peekToken.type == TokenType::Identifier) + { + const IdentifierStyle style = + m_nodeTree->m_identifierLookup->get(peekToken.getContent()); + if (style == IdentifierStyle::IntegerType) + { + m_reader.advanceToken(); + } + } + break; + } + + advanceIfToken(TokenType::Scope); + while (true) + { + // if we have a struct/class prefix in front of a name just consume it. + if (m_reader.peekTokenType() == TokenType::Identifier) + { + const IdentifierStyle style = + m_nodeTree->m_identifierLookup->get(m_reader.peekToken().getContent()); + if (style == IdentifierStyle::Class || style == IdentifierStyle::Struct) + { + m_reader.advanceToken(); + } + } + + Token identifierToken; + if (!advanceIfToken(TokenType::Identifier, &identifierToken)) + { + return SLANG_FAIL; + } + + const IdentifierStyle style = + m_nodeTree->m_identifierLookup->get(identifierToken.getContent()); + if (hasFlag(style, IdentifierFlag::Keyword)) + { + return SLANG_FAIL; + } + + if (advanceIfToken(TokenType::Scope)) + { + continue; + } + break; + } + + if (m_reader.peekTokenType() == TokenType::OpLess) + { + SLANG_RETURN_ON_FAIL(_maybeParseTemplateArgs(ioTemplateDepth)); + } + + if (m_reader.peekTokenType() == TokenType::Scope) + { + // Skip the scope and repeat + m_reader.advanceToken(); + continue; + } + + break; + } + + // Strip all the consts etc modifiers + _maybeConsume(IdentifierStyle::TypeModifier); + + // It's a reference and we are done + if (advanceIfToken(TokenType::OpBitAnd)) + { + return SLANG_OK; + } + + while (true) + { + if (advanceIfToken(TokenType::OpMul)) + { + // Strip all the consts + _maybeConsume(IdentifierStyle::TypeModifier); + continue; + } + break; + } + + if (advanceIfToken(TokenType::LParent)) + { + // TODO(JS): + // Doesn't handle all the modifiers just (*SomeName) + + SLANG_RETURN_ON_FAIL(expect(TokenType::OpMul)); + outNameCursor = m_reader.getCursor(); + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier)); + + SLANG_RETURN_ON_FAIL(expect(TokenType::RParent)); + + // We need to parse and add the params + if (m_reader.peekTokenType() != TokenType::LParent) + { + m_sink->diagnose( + m_reader.peekToken(), + CPPDiagnostics::expectingToken, + TokenType::LParent); + return SLANG_FAIL; + } + + // Consume the params + SLANG_RETURN_ON_FAIL(_consumeBalancedParens()); + } + else if (m_reader.peekTokenType() == TokenType::Identifier) + { + auto potentialNameCursor = m_reader.getCursor(); + m_reader.advanceToken(); + if (m_reader.peekTokenType() == TokenType::LBracket) + { + outNameCursor = potentialNameCursor; + while (advanceIfToken(TokenType::LBracket)) + { + List<Token> exprToks; + SLANG_RETURN_ON_FAIL(_parseExpression(exprToks)); + SLANG_RETURN_ON_FAIL(expect(TokenType::RBracket)); + } + } + else + { + // Wasn't an array type..., so rewind + m_reader.setCursor(potentialNameCursor); + } + } + + return SLANG_OK; +} + +SlangResult Parser::_maybeParseType(List<Token>& outToks, Token& outName) +{ + // Set to unknown + outName = Token(); + + auto startCursor = m_reader.getCursor(); + + TokenReader::ParsingCursor nameCursor; + + Index templateDepth = 0; + SlangResult res = _maybeParseType(templateDepth, nameCursor); + if (SLANG_FAILED(res) && m_sink->getErrorCount()) + { + return res; + } + + if (templateDepth != 0) + { + m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::unexpectedTemplateClose); + return SLANG_FAIL; + } + + auto endCursor = m_reader.getCursor(); + m_reader.setCursor(startCursor); + + if (nameCursor.isValid()) + { + while (!m_reader.isAtCursor(endCursor)) + { + if (m_reader.getCursor() == nameCursor) + { + outName = m_reader.advanceToken(); + } + else + { + outToks.add(m_reader.advanceToken()); + } + } + } + else + { + while (!m_reader.isAtCursor(endCursor)) + { + outToks.add(m_reader.advanceToken()); + } + } + + return SLANG_OK; +} + +SlangResult Parser::_parseSpecialMacro() +{ + Token name; + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &name)); + + List<Token> params; + + if (m_reader.peekTokenType() == TokenType::LParent) + { + // Mark the start + auto startCursor = m_reader.getCursor(); + + // Consume the params + SLANG_RETURN_ON_FAIL(_consumeBalancedParens()); + + auto endCursor = m_reader.getCursor(); + m_reader.setCursor(startCursor); + + while (!m_reader.isAtCursor(endCursor)) + { + params.add(m_reader.advanceToken()); + } + } + + // Can do special handling here + const UnownedStringSlice suffix = name.getContent().tail(m_options->m_markPrefix.getLength()); + + if (suffix == "COM_INTERFACE") + { + // TODO(JS): It's a com interface. Extact the GUID + } + + return SLANG_OK; +} + +SlangResult Parser::_parseMarker() +{ + SLANG_ASSERT( + m_reader.peekTokenType() == TokenType::Identifier && + _isMarker(m_reader.peekToken().getContent()) && m_currentScope->isClassLike()); + + ClassLikeNode* node = as<ClassLikeNode>(m_currentScope); + + if (node->m_marker.type != TokenType::Unknown) + { + m_sink->diagnose( + m_reader.peekToken(), + CPPDiagnostics::classMarkerAlreadyFound, + node->m_name.getContent()); + m_sink->diagnose(node->m_marker, CPPDiagnostics::previousLocation); + return SLANG_FAIL; + } + + // Set the marker token. + node->m_marker = m_reader.advanceToken(); + + // Looks like it's a marker + UnownedStringSlice slice(node->m_marker.getContent()); + + // Strip the prefix and suffix + slice = UnownedStringSlice( + slice.begin() + m_options->m_markPrefix.getLength(), + slice.end() - m_options->m_markSuffix.getLength()); + + // Strip ABSTRACT_ if it's there + UnownedStringSlice abstractSlice("ABSTRACT_"); + if (slice.startsWith(abstractSlice)) + { + slice = UnownedStringSlice(slice.begin() + abstractSlice.getLength(), slice.end()); + } + + // TODO: We could strip other stuff or have other heuristics there, but this is + // probably okay for now + + // Set the typeSet + node->m_typeSet = m_nodeTree->getOrAddTypeSet(slice); + + // Okay now looking for ( identifier) + Token typeNameToken; + + SLANG_RETURN_ON_FAIL(expect(TokenType::LParent)); + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &typeNameToken)); + SLANG_RETURN_ON_FAIL(expect(TokenType::RParent)); + + if (typeNameToken.getContent() != node->m_name.getContent()) + { + m_sink->diagnose( + typeNameToken, + CPPDiagnostics::typeNameDoesntMatch, + node->m_name.getContent()); + return SLANG_FAIL; + } + + // If has the marker it is assumed reflected + node->m_reflectionType = ReflectionType::Reflected; + return SLANG_OK; +} + +SlangResult Parser::_maybeParseType(UnownedStringSlice& outType, Token& outName) +{ + auto startCursor = m_reader.getCursor(); + + Index templateDepth = 0; + + TokenReader::ParsingCursor nameCursor; + + SlangResult res = _maybeParseType(templateDepth, nameCursor); + if (SLANG_FAILED(res) && m_sink->getErrorCount()) + { + return res; + } + + if (templateDepth != 0) + { + m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::unexpectedTemplateClose); + return SLANG_FAIL; + } + + if (nameCursor.isValid()) + { + const auto cursor = m_reader.getCursor(); + m_reader.setCursor(nameCursor); + outName = m_reader.peekToken(); + m_reader.setCursor(cursor); + + // Extract the contents + List<Token> toks; + _getTypeTokens(startCursor, nameCursor, toks); + outType = _concatTokens(toks.getBuffer(), toks.getCount()); + } + else + { + // We can build up the out type, from the tokens we found + outType = _concatTokens(startCursor); + } + return SLANG_OK; +} + +static bool _isBalancedOpen(TokenType tokenType) +{ + return tokenType == TokenType::LBrace || tokenType == TokenType::LParent || + tokenType == TokenType::LBracket; +} + +static bool _isBalancedClose(TokenType tokenType) +{ + return tokenType == TokenType::RBrace || tokenType == TokenType::RParent || + tokenType == TokenType::RBracket; +} + +static TokenType _getBalancedClose(TokenType tokenType) +{ + SLANG_ASSERT(_isBalancedOpen(tokenType)); + switch (tokenType) + { + case TokenType::LBrace: + return TokenType::RBrace; + case TokenType::LParent: + return TokenType::RParent; + case TokenType::LBracket: + return TokenType::RBracket; + default: + return TokenType::Unknown; + } +} + +SlangResult Parser::_parseBalanced(DiagnosticSink* sink) +{ + const TokenType openTokenType = m_reader.peekTokenType(); + if (!_isBalancedOpen(openTokenType)) + { + return SLANG_FAIL; + } + + // Save the start token + const Token startToken = m_reader.advanceToken(); + // Get the token type that would close the open + const TokenType closeTokenType = _getBalancedClose(openTokenType); + + while (true) + { + const TokenType tokenType = m_reader.peekTokenType(); + + // If we hit the closing token, we are done + if (tokenType == closeTokenType) + { + m_reader.advanceToken(); + return SLANG_OK; + } + + // If we hit a balanced open, recurse + if (_isBalancedOpen(tokenType)) + { + SLANG_RETURN_ON_FAIL(_parseBalanced(sink)); + continue; + } + + // If we hit a close token that doesn't match, then the balancing has gone wrong + if (_isBalancedClose(tokenType)) + { + // Only diagnose if required + if (sink) + { + sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::unexpectedUnbalancedToken); + sink->diagnose(startToken, CPPDiagnostics::seeOpen); + } + return SLANG_FAIL; + } + + // If we hit the end of the file and have not hit the closing token, then + // somethings gone wrong + if (tokenType == TokenType::EndOfFile) + { + if (sink) + { + sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::unexpectedEndOfFile); + sink->diagnose(startToken, CPPDiagnostics::seeOpen); + } + + return SLANG_FAIL; + } + + // Skip the token + m_reader.advanceToken(); + } +} + +SlangResult Parser::_consumeBalancedParens() +{ + SLANG_ASSERT(m_reader.peekTokenType() == TokenType::LParent); + + Index parenCount = 0; + + while (true) + { + const TokenType tokenType = m_reader.peekTokenType(); + + switch (tokenType) + { + case TokenType::LParent: + { + parenCount++; + break; + } + case TokenType::RParent: + { + --parenCount; + // If no more parens then we are done + if (parenCount == 0) + { + m_reader.advanceToken(); + return SLANG_OK; + } + break; + } + case TokenType::EndOfFile: + { + // If we hit the end of the file, then not balanced + return SLANG_FAIL; + } + default: + break; + } + + m_reader.advanceToken(); + } +} + +SlangResult Parser::_parseExpression(List<Token>& outExprTokens) +{ + Index parenCount = 0; + Index bracketCount = 0; + + // TODO(JS): NOTE! This doesn't handle an expression that contains a template params in + // Something<Arg1, 3>, because without knowing what Something is, it's not known if < is a + // comparison or or a 'template' bracket + // + // This can be worked around in the originating source by placing in parens + + while (true) + { + TokenType tokenType = m_reader.peekTokenType(); + + switch (tokenType) + { + case TokenType::LParent: + { + parenCount++; + break; + } + case TokenType::RParent: + { + // If no parens, and nothing else is open then we are done + if (parenCount == 0) + { + if (bracketCount) + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::cannotParseExpression); + return SLANG_FAIL; + } + + return SLANG_OK; + } + --parenCount; + break; + } + case TokenType::LBracket: + { + bracketCount++; + break; + } + case TokenType::RBracket: + { + // If no brackets are open we are done + if (bracketCount == 0) + { + if (parenCount) + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::cannotParseExpression); + return SLANG_FAIL; + } + return SLANG_OK; + } + --bracketCount; + break; + } + case TokenType::EndOfFile: + { + if ((bracketCount | parenCount) == 0) + { + return SLANG_OK; + } + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::cannotParseExpression); + return SLANG_FAIL; + } + case TokenType::RBrace: + case TokenType::Semicolon: + case TokenType::Comma: + { + if ((bracketCount | parenCount) == 0) + { + return SLANG_OK; + } + break; + } + + default: + break; + } + + outExprTokens.add(m_reader.advanceToken()); + } +} + +SlangResult Parser::_parseTypeDef() +{ + if (!m_currentScope->canContainTypes()) + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::cannotDeclareTypeInScope); + return SLANG_FAIL; + } + + // Consume the typedef + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier)); + + Token nameToken; + // Parse the type + List<Token> toks; + SLANG_RETURN_ON_FAIL(_maybeParseType(toks, nameToken)); + + // Followed by the name + if (nameToken.type != TokenType::Identifier) + { + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &nameToken)); + } + + if (Node::lookupNameInScope(m_currentScope, nameToken.getContent())) + { + m_sink->diagnose( + nameToken.loc, + CPPDiagnostics::identifierAlreadyDefined, + nameToken.getContent()); + return SLANG_FAIL; + } + + SLANG_RETURN_ON_FAIL(expect(TokenType::Semicolon)); + + RefPtr<TypeDefNode> node = new TypeDefNode; + node->m_name = nameToken; + node->m_reflectionType = m_currentScope->getContainedReflectionType(); + + // Set what aliases too + node->m_targetTypeTokens.swapWith(toks); + + m_currentScope->addChild(node); + + return SLANG_OK; +} + + +bool Parser::_isCtor() +{ + bool isCtor = false; + // It's a constructor + if (m_currentScope->isClassLike() && m_reader.peekTokenType() == TokenType::Identifier && + m_reader.peekToken().getContent() == m_currentScope->m_name.getContent()) + { + // We need to check it's followed immediately by ( to be sure it's a ctor + + auto cursor = m_reader.getCursor(); + m_reader.advanceToken(); + isCtor = (m_reader.peekTokenType() == TokenType::LParent); + m_reader.setCursor(cursor); + } + + return isCtor; +} + +bool isAlphaNumeric(char c) +{ + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9'); +} + +SlangResult Parser::_maybeParseContained(Node** outNode) +{ + *outNode = nullptr; + + _maybeConsume(IdentifierStyle::CallableMisc); + + bool isStatic = false; + bool isVirtual = false; + + while (m_reader.peekTokenType() == TokenType::Identifier) + { + const IdentifierStyle style = + m_nodeTree->m_identifierLookup->get(m_reader.peekToken().getContent()); + + // Check for virtualness + if (style == IdentifierStyle::Virtual) + { + isVirtual = true; + m_reader.advanceToken(); + continue; + } + + // Check if static + if (style == IdentifierStyle::Static) + { + isStatic = true; + m_reader.advanceToken(); + continue; + } + + break; + } + + _maybeConsume(IdentifierStyle::CallableMisc); + + UnownedStringSlice typeName; + Token nameToken; + + bool isConstructor = false; + + if (m_currentScope->isClassLike()) + { + // If it's a dtor + if (advanceIfToken(TokenType::OpBitNot, &nameToken)) + { + // Dtor + // For Dtor we don't hold the full name just the ~ + Token tok; + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &tok)); + + if (tok.getContent() != m_currentScope->m_name.getContent()) + { + m_sink->diagnose( + m_reader.peekLoc(), + CPPDiagnostics::destructorNameDoesntMatch, + m_currentScope->m_name.getContent()); + return SLANG_FAIL; + } + } + else if (_isCtor()) + { + nameToken = m_reader.advanceToken(); + isConstructor = true; + } + } + + // If don't have a name it's not a dtor or ctor, so see if it's a type + if (nameToken.type == TokenType::Unknown) + { + if (SLANG_FAILED(_maybeParseType(typeName, nameToken))) + { + if (m_sink->getErrorCount()) + { + return SLANG_FAIL; + } + + _consumeToSync(); + return SLANG_OK; + } + } + + if (nameToken.type == TokenType::Unknown) + { + // Has a calling convention (must be a function/method) + Token callingConventionToken; + advanceIfStyle(IdentifierStyle::CallingConvention, &callingConventionToken); + + // Expecting a name + if (!advanceIfToken(TokenType::Identifier, &nameToken)) + { + _consumeToSync(); + return SLANG_OK; + } + } + + // Handles other scenarios, but here for catching operator overloading + if (nameToken.type == TokenType::Identifier) + { + const auto style = m_nodeTree->m_identifierLookup->get(nameToken.getContent()); + if (style != IdentifierStyle::None) + { + _consumeToSync(); + return SLANG_OK; + } + } + + if (m_reader.peekTokenType() == TokenType::LParent) + { + if (!m_currentScope->canContainCallable()) + { + SLANG_RETURN_ON_FAIL(_consumeBalancedParens()); + // Consume everything up to ; or { + SLANG_RETURN_ON_FAIL(_consumeToSync()); + + return SLANG_OK; + } + + // Looks like it's a callable + m_reader.advanceToken(); + + List<CallableNode::Param> params; + + if (m_reader.peekTokenType() != TokenType::RParent) + { + while (true) + { + Token paramName; + UnownedStringSlice type; + SlangResult res = _maybeParseType(type, paramName); + + if (SLANG_FAILED(res)) + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::expectingType); + return res; + } + + if (paramName.type != TokenType::Identifier) + { + if (m_reader.peekTokenType() == TokenType::Identifier) + { + paramName = m_reader.advanceToken(); + } + } + + // If we have a name check for default value + if (paramName.type == TokenType::Identifier && advanceIfToken(TokenType::OpAssign)) + { + // Check if we have a default value + List<Token> exprTokens; + SLANG_RETURN_ON_FAIL(_parseExpression(exprTokens)); + } + + CallableNode::Param param; + param.m_name = paramName; + param.m_type = type; + + params.add(param); + + { + const auto peekType = m_reader.peekTokenType(); + if (peekType == TokenType::RParent) + { + break; + } + if (peekType == TokenType::Comma) + { + m_reader.advanceToken(); + continue; + } + } + + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::expectingToken, ", or ) or ="); + return SLANG_FAIL; + } + } + + // Skip ) + m_reader.advanceToken(); + + // Parse suffix + bool isPure = false; + + // const? + _maybeConsume(IdentifierStyle::TypeModifier); + + if (isConstructor) + { + // Initializer list + if (advanceIfToken(TokenType::Colon)) + { + while (true) + { + auto peekType = m_reader.peekTokenType(); + if (peekType == TokenType::Semicolon || peekType == TokenType::LBrace || + peekType == TokenType::EndOfFile) + { + break; + } + // Consume + m_reader.advanceToken(); + } + } + } + + // = 0 ? or = default + if (advanceIfToken(TokenType::OpAssign)) + { + if (m_reader.peekTokenType() == TokenType::IntegerLiteral) + { + Int value = -1; + if (SLANG_SUCCEEDED( + StringUtil::parseInt(m_reader.peekToken().getContent(), value)) && + value == 0) + { + isPure = true; + m_reader.advanceToken(); + } + else + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::expectingToken, "0"); + return SLANG_FAIL; + } + } + else if (advanceIfStyle(IdentifierStyle::Default)) + { + } + else + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::cannotParseCallable); + return SLANG_FAIL; + } + } + + if (m_reader.peekTokenType() == TokenType::Semicolon) + { + m_reader.advanceToken(); + } + else if (m_reader.peekTokenType() == TokenType::LBrace) + { + SLANG_RETURN_ON_FAIL(consumeToClosingBrace()); + } + else + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::expectingToken, "; or {"); + return SLANG_FAIL; + } + + RefPtr<CallableNode> callableNode = new CallableNode; + + callableNode->m_returnType = typeName; + callableNode->m_name = nameToken; + callableNode->m_reflectionType = m_currentScope->getContainedReflectionType(); + + callableNode->m_isVirtual = isVirtual; + callableNode->m_isPure = isPure; + callableNode->m_isStatic = isStatic; + + callableNode->m_params.swapWith(params); + + Node* nodeWithName = m_currentScope->findChild(nameToken.getContent()); + + if (nodeWithName) + { + CallableNode* initialOverload = as<CallableNode>(nodeWithName); + if (!initialOverload) + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::cannotOverload); + m_sink->diagnose(nodeWithName->getSourceLoc(), CPPDiagnostics::seeDeclarationOf); + return SLANG_FAIL; + } + + callableNode->m_nextOverload = initialOverload->m_nextOverload; + initialOverload->m_nextOverload = initialOverload; + + m_currentScope->addChildIgnoringName(callableNode); + } + else + { + m_currentScope->addChild(callableNode); + } + + *outNode = callableNode; + return SLANG_OK; + } + else + { + // Looks like variable + if (!m_currentScope->canContainFields() || nameToken.type != TokenType::Identifier) + { + _consumeToSync(); + return SLANG_OK; + } + + // Check if has a default value + if (advanceIfToken(TokenType::OpAssign)) + { + List<Token> exprTokens; + SLANG_RETURN_ON_FAIL(_parseExpression(exprTokens)); + } + + // Hit end of field/variable + if (m_reader.peekTokenType() == TokenType::Semicolon) + { + RefPtr<FieldNode> fieldNode = new FieldNode; + + fieldNode->m_fieldType = typeName; + fieldNode->m_name = nameToken; + fieldNode->m_reflectionType = m_currentScope->getContainedReflectionType(); + fieldNode->m_isStatic = isStatic; + if (fieldNode->m_reflectionType == ReflectionType::Reflected) + { + static const char* illegalTypes[] = { + "size_t", + "Int", + "UInt", + "Index", + "Count", + "UIndex", + "UCount", + "PtrInt", + "intptr_t", + "uintptr_t"}; + for (const auto& illegalType : illegalTypes) + { + int index = typeName.indexOf(UnownedStringSlice(illegalType)); + if (index != -1) + { + index += UnownedStringSlice(illegalType).getLength(); + if (index >= typeName.getLength() || !isAlphaNumeric(typeName[index])) + { + // Cannot use this type in a field (as it's arch dependent + m_sink->diagnose( + nameToken, + CPPDiagnostics::cannoseUseArchDependentType, + illegalType); + return SLANG_FAIL; + } + } + } + } + m_currentScope->addChild(fieldNode); + + *outNode = fieldNode; + return SLANG_OK; + } + } + + _consumeToSync(); + return SLANG_OK; +} + +/* static */ Node::Kind Parser::_toNodeKind(IdentifierStyle style) +{ + switch (style) + { + case IdentifierStyle::Class: + return Node::Kind::ClassType; + case IdentifierStyle::Struct: + return Node::Kind::StructType; + case IdentifierStyle::Namespace: + return Node::Kind::Namespace; + case IdentifierStyle::Enum: + return Node::Kind::Enum; + case IdentifierStyle::TypeDef: + return Node::Kind::TypeDef; + default: + return Node::Kind::Invalid; + } +} + +static UnownedStringSlice _trimUnderscorePrefix(const UnownedStringSlice& slice) +{ + if (slice.getLength() && slice[0] == '_') + { + return UnownedStringSlice(slice.begin() + 1, slice.end()); + } + else + { + return slice; + } +} + +SlangResult Parser::_parsePreDeclare() +{ + // Skip the declare type token + m_reader.advanceToken(); + + SLANG_RETURN_ON_FAIL(expect(TokenType::LParent)); + + // Get the typeSet + Token typeSetToken; + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &typeSetToken)); + TypeSet* typeSet = m_nodeTree->getOrAddTypeSet(typeSetToken.getContent()); + + SLANG_RETURN_ON_FAIL(expect(TokenType::Comma)); + + // Get the type of type + Node::Kind nodeKind; + { + Token typeToken; + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &typeToken)); + + const IdentifierStyle style = m_nodeTree->m_identifierLookup->get(typeToken.getContent()); + + if (style != IdentifierStyle::Struct && style != IdentifierStyle::Class) + { + m_sink->diagnose( + typeToken, + CPPDiagnostics::expectingTypeKeyword, + typeToken.getContent()); + return SLANG_FAIL; + } + nodeKind = _toNodeKind(style); + } + + Token name; + Token super; + + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &name)); + + if (advanceIfToken(TokenType::Colon)) + { + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &super)); + } + + SLANG_RETURN_ON_FAIL(expect(TokenType::RParent)); + + switch (nodeKind) + { + case Node::Kind::ClassType: + case Node::Kind::StructType: + { + RefPtr<ClassLikeNode> node(new ClassLikeNode(nodeKind)); + + node->m_name = name; + node->m_super = super; + node->m_typeSet = typeSet; + + // Assume it is reflected + node->m_reflectionType = ReflectionType::Reflected; + + SLANG_RETURN_ON_FAIL(pushScope(node)); + // Pop out of the node + popScope(); + break; + } + default: + { + return SLANG_FAIL; + } + } + + + return SLANG_OK; +} + +SlangResult Parser::_parseTypeSet() +{ + // Skip the declare type token + m_reader.advanceToken(); + + SLANG_RETURN_ON_FAIL(expect(TokenType::LParent)); + + Token typeSetToken; + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &typeSetToken)); + + TypeSet* typeSet = m_nodeTree->getOrAddTypeSet(typeSetToken.getContent()); + + SLANG_RETURN_ON_FAIL(expect(TokenType::Comma)); + + // Get the type of type + Token typeToken; + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &typeToken)); + + SLANG_RETURN_ON_FAIL(expect(TokenType::RParent)); + + // Set the typename + typeSet->m_typeName = typeToken.getContent(); + + return SLANG_OK; +} + +SlangResult Parser::parse(SourceOrigin* sourceOrigin, const Options* options) +{ + SLANG_ASSERT(options); + m_options = options; + + // Set the current origin + m_sourceOrigin = sourceOrigin; + + SourceFile* sourceFile = sourceOrigin->m_sourceFile; + + SourceManager* manager = sourceFile->getSourceManager(); + + SourceView* sourceView = manager->createSourceView(sourceFile, nullptr, SourceLoc::fromRaw(0)); + + Lexer lexer; + + // Set up the scope stack + m_scopeStack.clear(); + + m_currentScope = m_nodeTree->m_rootNode; + m_scopeStack.add(m_currentScope); + + if (!options->m_requireMark) + { + m_currentScope->m_reflectionOverride = ReflectionType::Reflected; + } + + lexer.initialize(sourceView, m_sink, m_nodeTree->m_namePool, manager->getMemoryArena()); + m_tokenList = lexer.lexAllSemanticTokens(); + // See if there were any errors + if (m_sink->getErrorCount()) + { + return SLANG_FAIL; + } + + m_reader = TokenReader(m_tokenList); + + while (true) + { + switch (m_reader.peekTokenType()) + { + case TokenType::OpBitNot: + { + // Handle dtor + if (m_currentScope->isClassLike()) + { + Node* containedNode = nullptr; + SLANG_RETURN_ON_FAIL(_maybeParseContained(&containedNode)); + } + else + { + // consume + m_reader.advanceToken(); + } + break; + } + case TokenType::Identifier: + { + const IdentifierStyle style = + m_nodeTree->m_identifierLookup->get(m_reader.peekToken().getContent()); + + switch (style) + { + case IdentifierStyle::Extern: + { + m_reader.advanceToken(); + + Token externType; + SLANG_RETURN_ON_FAIL(expect(TokenType::StringLiteral, &externType)); + + if (advanceIfToken(TokenType::LBrace)) + { + // Push a 'special' scope (which is basically transparent) + pushScope(nullptr); + } + break; + } + case IdentifierStyle::Template: + { + SLANG_RETURN_ON_FAIL(_consumeTemplate()); + break; + } + case IdentifierStyle::PreDeclare: + { + SLANG_RETURN_ON_FAIL(_parsePreDeclare()); + break; + } + case IdentifierStyle::TypeSet: + { + SLANG_RETURN_ON_FAIL(_parseTypeSet()); + break; + } + case IdentifierStyle::Reflected: + { + m_reader.advanceToken(); + if (m_currentScope) + { + m_currentScope->m_reflectionOverride = ReflectionType::Reflected; + } + break; + } + case IdentifierStyle::Unreflected: + { + m_reader.advanceToken(); + if (m_currentScope) + { + m_currentScope->m_reflectionOverride = ReflectionType::NotReflected; + } + break; + } + case IdentifierStyle::Access: + { + m_reader.advanceToken(); + SLANG_RETURN_ON_FAIL(expect(TokenType::Colon)); + break; + } + case IdentifierStyle::TypeDef: + { + if (isTypeEnabled(Node::Kind::TypeDef)) + { + SLANG_RETURN_ON_FAIL(_parseTypeDef()); + } + else + { + m_reader.advanceToken(); + SLANG_RETURN_ON_FAIL(_consumeToSync()); + } + break; + } + default: + { + IdentifierFlags flags = getFlags(style); + + if (flags & IdentifierFlag::StartScope) + { + Node::Kind kind = _toNodeKind(style); + SLANG_ASSERT(kind != Node::Kind::Invalid); + + if (isTypeEnabled(kind)) + { + SLANG_RETURN_ON_FAIL(_maybeParseNode(kind)); + } + else + { + SLANG_RETURN_ON_FAIL(_maybeConsumeScope()); + } + } + else + { + UnownedStringSlice content = m_reader.peekToken().getContent(); + + // If it's a marker handle it + if (_isMarker(content)) + { + if (!m_currentScope->isClassLike()) + { + m_sink->diagnose( + m_reader.peekLoc(), + CPPDiagnostics::classMarkerOutsideOfClass); + return SLANG_FAIL; + } + + SLANG_RETURN_ON_FAIL(_parseMarker()); + break; + } + + if (m_options->m_markPrefix.getLength() > 0 && + content.startsWith(m_options->m_markPrefix.getUnownedSlice())) + { + SLANG_RETURN_ON_FAIL(_parseSpecialMacro()); + break; + } + + + // Special case the node that's the root of the hierarchy (as far as + // reflection is concerned) This could be a field + if (m_currentScope->canContainFields() || + m_currentScope->canContainCallable()) + { + Node* containedNode = nullptr; + SLANG_RETURN_ON_FAIL(_maybeParseContained(&containedNode)); + } + else + { + m_reader.advanceToken(); + } + } + break; + } + } + break; + } + case TokenType::LBrace: + { + SLANG_RETURN_ON_FAIL(consumeToClosingBrace()); + break; + } + case TokenType::RBrace: + { + SLANG_RETURN_ON_FAIL(popScope()); + m_reader.advanceToken(); + break; + } + case TokenType::EndOfFile: + { + // Okay we need to confirm that we are in the root node, and with no open braces + if (m_currentScope != m_nodeTree->getRootNode()) + { + m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::braceOpenAtEndOfFile); + return SLANG_FAIL; + } + if (m_sink->getErrorCount()) + return SLANG_FAIL; + return SLANG_OK; + } + case TokenType::Pound: + { + Token token = m_reader.peekToken(); + if (token.flags & TokenFlag::AtStartOfLine) + { + // We are just going to ignore all of these for now.... + m_reader.advanceToken(); + for (;;) + { + auto t = m_reader.peekToken(); + if (t.type == TokenType::EndOfFile || (t.flags & TokenFlag::AtStartOfLine)) + { + break; + } + m_reader.advanceToken(); + } + break; + } + // Skip it then + m_reader.advanceToken(); + break; + } + default: + { + // Skip it then + m_reader.advanceToken(); + break; + } + } + } +} + +} // namespace CppParse diff --git a/tools/slang-cpp-parser/parser.h b/tools/slang-cpp-parser/parser.h new file mode 100644 index 000000000..519ad6935 --- /dev/null +++ b/tools/slang-cpp-parser/parser.h @@ -0,0 +1,112 @@ +#pragma once + +#include "compiler-core/slang-lexer.h" +#include "diagnostics.h" +#include "identifier-lookup.h" +#include "node-tree.h" +#include "node.h" + +namespace CppParse +{ +using namespace Slang; + +class Parser +{ +public: + typedef uint32_t NodeTypeBitType; + + SlangResult expect(TokenType type, Token* outToken = nullptr); + + bool advanceIfMarker(Token* outToken = nullptr); + bool advanceIfToken(TokenType type, Token* outToken = nullptr); + bool advanceIfStyle(IdentifierStyle style, Token* outToken = nullptr); + + SlangResult pushAnonymousNamespace(); + SlangResult pushScope(ScopeNode* node); + SlangResult consumeToClosingBrace(const Token* openBraceToken = nullptr); + SlangResult popScope(); + + /// Parse the contents of the source file + SlangResult parse(SourceOrigin* sourceOrigin, const Options* options); + + void setKindEnabled(Node::Kind kind, bool isEnabled = true); + bool isTypeEnabled(Node::Kind kind) + { + return (m_nodeTypeEnabled & (NodeTypeBitType(1) << int(kind))) != 0; + } + + void setKindsEnabled(const Node::Kind* kinds, Index kindsCount, bool isEnabled = true); + + Parser(NodeTree* nodeTree, DiagnosticSink* sink); + +protected: + static Node::Kind _toNodeKind(IdentifierStyle style); + + bool _isMarker(const UnownedStringSlice& name); + + SlangResult _maybeConsumeScope(); + + SlangResult _parsePreDeclare(); + SlangResult _parseTypeSet(); + + SlangResult _maybeParseNode(Node::Kind kind); + SlangResult _maybeParseContained(Node** outNode); + + SlangResult _parseTypeDef(); + SlangResult _parseEnum(); + SlangResult _parseMarker(); + SlangResult _parseSpecialMacro(); + + SlangResult _maybeParseType(List<Token>& outToks, Token& outName); + SlangResult _maybeParseType(UnownedStringSlice& outType, Token& outName); + SlangResult _maybeParseType(Index& ioTemplateDepth, TokenReader::ParsingCursor& outCursor); + + SlangResult _parseExpression(List<Token>& outExprTokens); + + SlangResult _maybeParseTemplateArgs(Index& ioTemplateDepth); + SlangResult _maybeParseTemplateArg(Index& ioTemplateDepth); + + /// Parse balanced - if a sink is set will report to that sink + SlangResult _parseBalanced(DiagnosticSink* sink); + + bool _isCtor(); + + /// Concatenate all tokens from start to the current position + UnownedStringSlice _concatTokens(TokenReader::ParsingCursor start); + UnownedStringSlice _concatTokens(const Token* toks, Index toksCount); + + UnownedStringSlice _concatType( + TokenReader::ParsingCursor start, + TokenReader::ParsingCursor nameCursor); + + void _getTypeTokens( + TokenReader::ParsingCursor start, + TokenReader::ParsingCursor nameCursor, + List<Token>& outToks); + + /// Consume what looks like a template definition + SlangResult _consumeTemplate(); + SlangResult _maybeConsume(IdentifierStyle style); + + SlangResult _consumeToSync(); + /// Consumes balanced parens. Will return an error if not matched. Assumes starts on opening ( + SlangResult _consumeBalancedParens(); + + NodeTypeBitType m_nodeTypeEnabled; + + TokenList m_tokenList; + TokenReader m_reader; + + List<ScopeNode*> m_scopeStack; + + ScopeNode* m_currentScope; ///< The current scope being processed + SourceOrigin* m_sourceOrigin; ///< The source origin that all tokens are in + + DiagnosticSink* m_sink; ///< Diagnostic sink + + NodeTree* m_nodeTree; ///< Shared state between parses. Nodes will be added to this + + const Options* m_options; +}; + +} // namespace CppParse diff --git a/tools/slang-cpp-parser/unit-test.cpp b/tools/slang-cpp-parser/unit-test.cpp new file mode 100644 index 000000000..70851fd7c --- /dev/null +++ b/tools/slang-cpp-parser/unit-test.cpp @@ -0,0 +1,127 @@ +#include "unit-test.h" + +#include "compiler-core/slang-lexer.h" +#include "compiler-core/slang-name-convention-util.h" +#include "compiler-core/slang-source-loc.h" +#include "core/slang-io.h" +#include "identifier-lookup.h" +#include "node-tree.h" +#include "options.h" +#include "parser.h" + +namespace CppParse +{ +using namespace Slang; + + +struct TestState +{ + TestState() + : m_slicePool(StringSlicePool::Style::Default) + { + m_identifierLookup.initDefault(UnownedStringSlice::fromLiteral("SLANG_")); + + m_sourceManager.initialize(nullptr, nullptr); + + m_sink.init(&m_sourceManager, Lexer::sourceLocationLexer); + + m_namePool.setRootNamePool(&m_rootNamePool); + + // We don't require marker + m_options.m_requireMark = false; + } + + RootNamePool m_rootNamePool; + Options m_options; + SourceManager m_sourceManager; + DiagnosticSink m_sink; + NamePool m_namePool; + StringSlicePool m_slicePool; + IdentifierLookup m_identifierLookup; +}; + +static const char someSource[] = "class ISomeInterface\n" + "{\n" + " public:\n" + " virtual int SLANG_MCALL someMethod(int a, int b) const = 0;\n" + " virtual float SLANG_MCALL anotherMethod(float a) = 0;\n" + "};\n" + "\n" + "struct SomeStruct\n" + "{\n" + " SomeStruct() = default;\n" + " SomeStruct(float v = 0.0f):b(v) {}\n" + " ~SomeStruct() {}\n" + " int a = 10; \n" + " float b; \n" + " int another[10];\n" + " const char* yetAnother = nullptr;\n" + "};\n" + "\n" + "enum SomeEnum\n" + "{\n" + " Value,\n" + " Another = 10,\n" + "};\n" + "\n" + "typedef int (*SomeFunc)(int a);\n" + "\n" + "typedef SomeEnum AliasEnum;\n" + "void someFunc(int a, float b) { }\n" + "namespace Blah {\n" + "int add(int a, int b) { return a + b; }\n" + "unsigned add(unsigned a, unsigned b) { return a + b; }\n" + "}\n"; + + +/* static */ SlangResult UnitTestUtil::run() +{ + { + TestState state; + + NodeTree tree(&state.m_slicePool, &state.m_namePool, &state.m_identifierLookup); + + UnownedStringSlice contents = UnownedStringSlice::fromLiteral(someSource); + PathInfo pathInfo = PathInfo::makeFromString("source.h"); + + SourceManager* sourceManager = &state.m_sourceManager; + + SourceFile* sourceFile = sourceManager->createSourceFileWithString(pathInfo, contents); + SourceOrigin* sourceOrigin = tree.addSourceOrigin(sourceFile, state.m_options); + + Parser parser(&tree, &state.m_sink); + + + { + const Node::Kind enableKinds[] = { + Node::Kind::Enum, + Node::Kind::EnumClass, + Node::Kind::EnumCase, + Node::Kind::TypeDef}; + parser.setKindsEnabled(enableKinds, SLANG_COUNT_OF(enableKinds)); + } + + SlangResult res = parser.parse(sourceOrigin, &state.m_options); + + if (state.m_sink.outputBuffer.getLength()) + { + printf("%s\n", state.m_sink.outputBuffer.getBuffer()); + } + + if (SLANG_FAILED(res)) + { + return res; + } + + { + StringBuilder buf; + tree.getRootNode()->dump(0, buf); + + SLANG_UNUSED(buf); + } + } + + return SLANG_OK; +} + +} // namespace CppParse diff --git a/tools/slang-cpp-parser/unit-test.h b/tools/slang-cpp-parser/unit-test.h new file mode 100644 index 000000000..e8d1b53b2 --- /dev/null +++ b/tools/slang-cpp-parser/unit-test.h @@ -0,0 +1,14 @@ +#pragma once + +#include "diagnostics.h" + +namespace CppParse +{ +using namespace Slang; + +struct UnitTestUtil +{ + static SlangResult run(); +}; + +} // namespace CppParse |
