diff options
Diffstat (limited to 'tools/slang-cpp-extractor')
| -rw-r--r-- | tools/slang-cpp-extractor/cpp-extractor-main.cpp | 4 | ||||
| -rw-r--r-- | tools/slang-cpp-extractor/diagnostic-defs.h | 6 | ||||
| -rw-r--r-- | tools/slang-cpp-extractor/identifier-lookup.cpp | 14 | ||||
| -rw-r--r-- | tools/slang-cpp-extractor/identifier-lookup.h | 7 | ||||
| -rw-r--r-- | tools/slang-cpp-extractor/macro-writer.cpp | 7 | ||||
| -rw-r--r-- | tools/slang-cpp-extractor/node.cpp | 80 | ||||
| -rw-r--r-- | tools/slang-cpp-extractor/node.h | 49 | ||||
| -rw-r--r-- | tools/slang-cpp-extractor/parser.cpp | 730 | ||||
| -rw-r--r-- | tools/slang-cpp-extractor/parser.h | 17 | ||||
| -rw-r--r-- | tools/slang-cpp-extractor/unit-test.cpp | 26 |
10 files changed, 774 insertions, 166 deletions
diff --git a/tools/slang-cpp-extractor/cpp-extractor-main.cpp b/tools/slang-cpp-extractor/cpp-extractor-main.cpp index 9db61f8e0..371392851 100644 --- a/tools/slang-cpp-extractor/cpp-extractor-main.cpp +++ b/tools/slang-cpp-extractor/cpp-extractor-main.cpp @@ -107,6 +107,10 @@ static DocMarkupExtractor::SearchStyle _getSearchStyle(Node* node) { return SearchStyle::Variable; } + case Node::Kind::Callable: + { + return SearchStyle::Before; + } default: break; } diff --git a/tools/slang-cpp-extractor/diagnostic-defs.h b/tools/slang-cpp-extractor/diagnostic-defs.h index ba43f1844..c7e364291 100644 --- a/tools/slang-cpp-extractor/diagnostic-defs.h +++ b/tools/slang-cpp-extractor/diagnostic-defs.h @@ -21,6 +21,7 @@ DIAGNOSTIC(-1, Note, seeDeclarationOf, "see declaration of '$0'") DIAGNOSTIC(-1, Note, seeOpen, "see open $0") DIAGNOSTIC(-1, Note, commandLine, "Command line: $0") +DIAGNOSTIC(-1, Note, previousLocation, "previous location") DIAGNOSTIC(1, Error, cannotOpenFile, "cannot open file '$0'.") DIAGNOSTIC(1, Error, errorAccessingFile, "error accessing file '$0'.") @@ -46,6 +47,11 @@ DIAGNOSTIC(100012, Error, typeInDifferentTypeSet, "Type $0 in different type set DIAGNOSTIC(100013, Error, expectingIdentifier, "Expecting an identifier, found $0") DIAGNOSTIC(100014, Error, cannotDeclareTypeInScope, "Cannot declare types in this scope") DIAGNOSTIC(100015, Error, identifierAlreadyDefined, "Identifier already defined '$0'") +DIAGNOSTIC(100016, Error, expectingType, "Expecting a type") +DIAGNOSTIC(100017, Error, cannotParseExpression, "Cannot parse expression") +DIAGNOSTIC(100018, Error, cannotOverload, "Cannot overload this declaration"); +DIAGNOSTIC(100019, Error, classMarkerOutsideOfClass, "A class/struct marker is found outside of class or struct"); +DIAGNOSTIC(100020, Error, classMarkerAlreadyFound, "A class/struct marker already found in type '$0'"); // Command line errors 100100 diff --git a/tools/slang-cpp-extractor/identifier-lookup.cpp b/tools/slang-cpp-extractor/identifier-lookup.cpp index c7be75d82..db8ea2846 100644 --- a/tools/slang-cpp-extractor/identifier-lookup.cpp +++ b/tools/slang-cpp-extractor/identifier-lookup.cpp @@ -22,6 +22,11 @@ using namespace Slang; IdentifierFlag::Keyword, /// Access IdentifierFlag::Reflection, /// Reflected IdentifierFlag::Reflection, /// Unreflected + + IdentifierFlag::Keyword, /// virtual + 0, /// Calling convention + IdentifierFlag::Keyword, /// template + IdentifierFlag::Keyword, /// static }; void IdentifierLookup::set(const UnownedStringSlice& name, IdentifierStyle style) @@ -63,7 +68,7 @@ void IdentifierLookup::initDefault(const UnownedStringSlice& markPrefix) // Some keywords { - const char* names[] = { "virtual", "continue", "if", "case", "break", "catch", "default", "delete", "do", "else", "for", "new", "goto", "return", "switch", "throw", "using", "while", "operator" }; + const char* names[] = { "continue", "if", "case", "break", "catch", "default", "delete", "do", "else", "for", "new", "goto", "return", "switch", "throw", "using", "while", "operator", "explicit"}; set(names, SLANG_COUNT_OF(names), IdentifierStyle::Keyword); } @@ -88,6 +93,13 @@ void IdentifierLookup::initDefault(const UnownedStringSlice& markPrefix) } } + { + set("virtual", IdentifierStyle::Virtual); + set("SLANG_MCALL", IdentifierStyle::CallingConvention); + set("template", IdentifierStyle::Template); + set("static", IdentifierStyle::Static); + } + // Keywords which introduce types/scopes { const Pair pairs[] = diff --git a/tools/slang-cpp-extractor/identifier-lookup.h b/tools/slang-cpp-extractor/identifier-lookup.h index 3cee909ef..40644a1ba 100644 --- a/tools/slang-cpp-extractor/identifier-lookup.h +++ b/tools/slang-cpp-extractor/identifier-lookup.h @@ -30,6 +30,13 @@ enum class IdentifierStyle Reflected, Unreflected, + CallingConvention, ///< Used on a method + Virtual, ///< + + Template, + + Static, + CountOf, }; diff --git a/tools/slang-cpp-extractor/macro-writer.cpp b/tools/slang-cpp-extractor/macro-writer.cpp index c4245a646..0f217cd4e 100644 --- a/tools/slang-cpp-extractor/macro-writer.cpp +++ b/tools/slang-cpp-extractor/macro-writer.cpp @@ -120,13 +120,16 @@ SlangResult MacroWriter::calcChildrenHeader(NodeTree* tree, TypeSet* typeSet, St // Define the derived types out << "#define " << m_options->m_markPrefix << "FIELDS_" << reflectTypeName << "_" << classNode->m_name.getContent() << "(_x_, _param_)"; - // Find all of the fields + // Find all of the instance fields fields List<FieldNode*> fields; for (Node* child : classNode->m_children) { if (auto field = as<FieldNode>(child)) { - fields.add(field); + if (!field->m_isStatic) + { + fields.add(field); + } } } diff --git a/tools/slang-cpp-extractor/node.cpp b/tools/slang-cpp-extractor/node.cpp index 2074b7c41..3fbbf9c56 100644 --- a/tools/slang-cpp-extractor/node.cpp +++ b/tools/slang-cpp-extractor/node.cpp @@ -294,7 +294,7 @@ ScopeNode* ScopeNode::getAnonymousNamespace() return m_anonymousNamespace; } -void ScopeNode::addChild(Node* child) +void ScopeNode::addChildIgnoringName(Node* child) { SLANG_ASSERT(child->m_parentScope == nullptr); // Can't add anonymous namespace this way - should be added via getAnonymousNamespace @@ -302,6 +302,11 @@ void ScopeNode::addChild(Node* child) child->m_parentScope = this; m_children.add(child); +} + +void ScopeNode::addChild(Node* child) +{ + addChildIgnoringName(child); if (child->m_name.hasContent()) { @@ -316,9 +321,6 @@ Node* ScopeNode::findChild(const UnownedStringSlice& name) const { return *nodePtr; } - - - return nullptr; } @@ -377,8 +379,7 @@ static bool _needsSpace(const Token& prevTok, const Token& tok) auto loc = tok.getLoc(); auto prevContent = prevTok.getContent(); - auto content = tok.getContent(); - + if (prevLoc + prevContent.getLength() == loc) { return false; @@ -494,17 +495,76 @@ void EnumNode::dump(int indent, StringBuilder& out) out << "}\n"; } +/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!! CallableNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ + +void CallableNode::dump(int indent, StringBuilder& out) +{ + if (!isReflected()) + { + return; + } + + dumpMarkup(indent, out); + + _indent(indent, out); + + if (m_isStatic) + { + out << "static "; + } + if (m_isVirtual) + { + out << "virtual "; + } + + out << m_returnType << " "; + out << m_name.getContent() << "("; + + const Index count = m_params.getCount(); + for (Index i = 0; i < count; ++i) + { + if (i > 0) + { + out << ", "; + } + + const auto& param = m_params[i]; + out << param.m_type; + if (param.m_name.type == TokenType::Identifier) + { + out << " " << param.m_name.getContent(); + } + } + + out << ")"; + + if (m_isPure) + { + out << " = 0"; + } + + out << "\n"; +} + /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! FieldNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ void FieldNode::dump(int indent, StringBuilder& out) { - if (isReflected()) + if (!isReflected()) { - dumpMarkup(indent, out); + return; + } - _indent(indent, out); - out << m_fieldType << " " << m_name.getContent() << "\n"; + dumpMarkup(indent, out); + + _indent(indent, out); + + if (m_isStatic) + { + out << "static "; } + + out << m_fieldType << " " << m_name.getContent() << "\n"; } /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ClassLikeNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ diff --git a/tools/slang-cpp-extractor/node.h b/tools/slang-cpp-extractor/node.h index f649f1adb..db9a10e36 100644 --- a/tools/slang-cpp-extractor/node.h +++ b/tools/slang-cpp-extractor/node.h @@ -41,6 +41,11 @@ public: TypeDef, + Callable, ///< Functions/methods + + Other, ///< Used 'other' parsing like for TYPE + Unknown, ///< Used for marking tokens consumed but usage is not known + CountOf, }; @@ -64,7 +69,7 @@ public: /// Returns true if kind can cast to this type /// Used for implementing as<T> casting - static bool isOfKind(Kind type) { return true; } + static bool isOfKind(Kind kind) { SLANG_UNUSED(kind); return true; } static bool isKindScope(Kind kind) { return int(kind) >= int(KindRange::ScopeStart) && int(kind) <= int(KindRange::ScopeEnd); } static bool isKindClassLike(Kind kind) { return int(kind) >= int(KindRange::ClassLikeStart) && int(kind) <= int(KindRange::ClassLikeEnd); } @@ -95,7 +100,10 @@ public: return false; } + bool isNamespace() const { return m_kind == Kind::Namespace; } bool isClassLike() const { return isKindClassLike(m_kind); } + bool isScope() const { return isKindScope(m_kind); } + bool isEnumLike() const { return isKindEnumLike(m_kind); } /// These are useful for the filter static bool isClassLikeAndReflected(Node* node) { return node->isClassLike() && node->isReflected(); } @@ -175,8 +183,12 @@ struct ScopeNode : public Node virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; virtual void calcScopeDepthFirst(List<Node*>& outNodes) SLANG_OVERRIDE; + /// True if can contain callable entries + bool canContainCallable() const { return isClassLike() || isNamespace(); } + /// True if can accept fields (class like types can) bool canContainFields() const { return isClassLike(); } + /// True if the scope can accept types bool canContainTypes() const { return canKindContainTypes(m_kind); } @@ -185,6 +197,8 @@ struct ScopeNode : public Node /// Add a child node to this nodes scope void addChild(Node* child); + /// Adds the child but does not add the name to the map + void addChildIgnoringName(Node* child); /// Find a child node in this scope with the specified name. Return nullptr if not found Node* findChild(const UnownedStringSlice& name) const; @@ -216,7 +230,7 @@ struct FieldNode : public Node { typedef Node Super; - static bool isOfKind(Kind type) { return type == Kind::Field; } + static bool isOfKind(Kind kind) { return kind == Kind::Field; } virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; @@ -227,7 +241,9 @@ struct FieldNode : public Node UnownedStringSlice m_fieldType; - // We may want to add initializer tokens + bool m_isStatic = false; + + /// TODO(JS): We may want to add initializer tokens }; struct ClassLikeNode : public ScopeNode @@ -281,6 +297,33 @@ struct ClassLikeNode : public ScopeNode ClassLikeNode* m_superNode; ///< If this is a class/struct, the type it is derived from (or nullptr if base) }; +struct CallableNode : public Node +{ + typedef Node Super; + + static bool isOfKind(Kind kind) { return kind == Kind::Callable; } + + virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE; + + CallableNode() :Super(Kind::Callable) {} + + struct Param + { + UnownedStringSlice m_type; + Token m_name; + }; + + UnownedStringSlice m_returnType; + + CallableNode* m_nextOverload = nullptr; + + List<Param> m_params; + + bool m_isStatic = false; + bool m_isVirtual = false; + bool m_isPure = false; +}; + struct EnumCaseNode : public Node { typedef Node Super; diff --git a/tools/slang-cpp-extractor/parser.cpp b/tools/slang-cpp-extractor/parser.cpp index 29bc517f5..3f87344ed 100644 --- a/tools/slang-cpp-extractor/parser.cpp +++ b/tools/slang-cpp-extractor/parser.cpp @@ -5,6 +5,7 @@ #include "../../source/compiler-core/slang-name-convention-util.h" +#include "../../source/core/slang-string-util.h" #include "../../source/core/slang-io.h" namespace CppExtract { @@ -33,6 +34,8 @@ Parser::Parser(NodeTree* nodeTree, DiagnosticSink* sink) : // Node::Type::TypeDef, // Node::Type::Enum, // Node::Type::EnumClass, + + Node::Kind::Callable, }; setKindsEnabled(defaultEnabled, SLANG_COUNT_OF(defaultEnabled)); } @@ -215,7 +218,7 @@ SlangResult Parser::_maybeConsumeScope() } else if (type == TokenType::LBrace) { - m_reader.advanceToken(); + //m_reader.advanceToken(); return consumeToClosingBrace(); } else if (type == TokenType::EndOfFile) @@ -238,6 +241,7 @@ SlangResult Parser::consumeToClosingBrace(const Token* inOpenBraceToken) { openToken = m_reader.advanceToken(); } + SLANG_ASSERT(openToken.type == TokenType::LBrace); while (true) { @@ -310,6 +314,7 @@ SlangResult Parser::_parseEnum() RefPtr<EnumNode> node = new EnumNode(kind); node->m_name = nameToken; + node->m_reflectionType = m_currentScope->getContainedReflectionType(); if (advanceIfToken(TokenType::Colon)) { @@ -382,28 +387,15 @@ SlangResult Parser::_parseEnum() return SLANG_FAIL; } + caseNode->m_reflectionType = m_currentScope->getContainedReflectionType(); + // Add the value node->addChild(caseNode); - // TODO(JS): - // This could be better. We could lookup the value etc. For now just assume only one token is valid. - if (advanceIfToken(TokenType::OpAssign)) { List<Token> valueTokens; - // Consume up to } or , - while (true) - { - TokenType assignType = m_reader.peekTokenType(); - - if (assignType == TokenType::Comma || - assignType == TokenType::RBrace || - assignType == TokenType::EndOfFile) - { - break; - } - valueTokens.add(m_reader.advanceToken()); - } + SLANG_RETURN_ON_FAIL(_parseExpression(valueTokens)); if (valueTokens.getCount() > 0) { @@ -427,6 +419,97 @@ SlangResult Parser::_parseEnum() return popScope(); } +SlangResult Parser::_consumeTemplate() +{ + // Skip the current 'template' token. + m_reader.advanceToken(); + + // Consume everything in <> + SLANG_RETURN_ON_FAIL(expect(TokenType::OpLess)); + + { + Index arrowCount = 1; + while (true) + { + auto tokenType = m_reader.peekTokenType(); + + if (tokenType == TokenType::OpLess) + { + m_reader.advanceToken(); + arrowCount++; + } + else if (tokenType == TokenType::OpGreater) + { + m_reader.advanceToken(); + if (arrowCount == 1) + { + break; + } + --arrowCount; + } + else if (tokenType == TokenType::OpRsh) + { + if (arrowCount < 2) + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::unexpectedTemplateClose); + return SLANG_FAIL; + } + m_reader.advanceToken(); + if (arrowCount == 2) + { + break; + } + arrowCount -= 2; + } + else if (tokenType == TokenType::EndOfFile) + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::unexpectedEndOfFile); + return SLANG_FAIL; + } + else + { + m_reader.advanceToken(); + } + } + } + + // Search for { or ; to consume remaining + while (true) + { + auto tokenType = m_reader.peekTokenType(); + + switch (tokenType) + { + case TokenType::EndOfFile: + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::unexpectedEndOfFile); + return SLANG_FAIL; + } + case TokenType::Semicolon: + { + // Ends with semicolon if it's a template pre-declaration + m_reader.advanceToken(); + return SLANG_OK; + } + case TokenType::LBrace: + { + // If ends with {, means could be body of a struct/class or a body of a function/method. + // Consume it + SLANG_RETURN_ON_FAIL(consumeToClosingBrace()); + // If we hit a ; just consume and ignore + advanceIfToken(TokenType::Semicolon); + return SLANG_OK; + } + default: + { + // Consume + m_reader.advanceToken(); + break; + } + } + } +} + SlangResult Parser::_maybeParseNode(Node::Kind kind) { // We are looking for @@ -449,6 +532,8 @@ SlangResult Parser::_maybeParseNode(Node::Kind kind) // Okay looks like we are opening a namespace RefPtr<ScopeNode> node(new ScopeNode(Node::Kind::Namespace)); node->m_name = name; + + node->m_reflectionType = m_currentScope->getContainedReflectionType(); // Push the node return pushScope(node); } @@ -482,8 +567,16 @@ SlangResult Parser::_maybeParseNode(Node::Kind kind) RefPtr<ClassLikeNode> node(new ClassLikeNode(kind)); node->m_name = name; - // Defaults to not reflected - SLANG_ASSERT(!node->isReflected()); + // We default to the containing scope for reflection type. + if (!m_requireMarker) + { + node->m_reflectionType = m_currentScope->getContainedReflectionType(); + } + else + { + // Defaults to not reflected + SLANG_ASSERT(!node->isReflected()); + } if (advanceIfToken(TokenType::Colon)) { @@ -496,6 +589,8 @@ SlangResult Parser::_maybeParseNode(Node::Kind kind) } } + // We only accept a single super class. Consume everything afterwards until we hit the { brace + if (m_reader.peekTokenType() != TokenType::LBrace) { // Consume up until we see a brace else it's an error @@ -518,89 +613,12 @@ SlangResult Parser::_maybeParseNode(Node::Kind kind) return pushScope(node); } - Token braceToken = m_reader.advanceToken(); - - while (true) - { - // Okay now we are looking for the markers, or visibility qualifiers - if (advanceIfStyle(IdentifierStyle::Access)) - { - // Consume it and a colon - if (SLANG_FAILED(expect(TokenType::Colon))) - { - consumeToClosingBrace(&braceToken); - return SLANG_OK; - } - continue; - } - - switch (m_reader.peekTokenType()) - { - case TokenType::Identifier: break; - case TokenType::RBrace: - { - SLANG_RETURN_ON_FAIL(pushScope(node)); - SLANG_RETURN_ON_FAIL(popScope()); - m_reader.advanceToken(); - return SLANG_OK; - } - default: - { - SLANG_RETURN_ON_FAIL(pushScope(node)); - return SLANG_OK; - } - } - - // If it's one of the markers, then we continue to extract parameter - if (advanceIfMarker(&node->m_marker)) - { - break; - } - - // We still need to add the node, - SLANG_RETURN_ON_FAIL(pushScope(node)); - return SLANG_OK; - } - - // Let's extract the type set - { - UnownedStringSlice slice(node->m_marker.getContent()); - - SLANG_ASSERT(_isMarker(slice)); - - // Strip the prefix and suffix - slice = UnownedStringSlice(slice.begin() + m_options->m_markPrefix.getLength(), slice.end() - m_options->m_markSuffix.getLength()); - - // Strip ABSTRACT_ if it's there - UnownedStringSlice abstractSlice("ABSTRACT_"); - if (slice.startsWith(abstractSlice)) - { - slice = UnownedStringSlice(slice.begin() + abstractSlice.getLength(), slice.end()); - } - - // TODO: We could strip other stuff or have other heuristics there, but this is - // probably okay for now - - // Set the typeSet - node->m_typeSet = m_nodeTree->getOrAddTypeSet(slice); - } - - // Okay now looking for ( identifier) - Token typeNameToken; - - SLANG_RETURN_ON_FAIL(expect(TokenType::LParent)); - SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &typeNameToken)); - SLANG_RETURN_ON_FAIL(expect(TokenType::RParent)); + const Token braceToken = m_reader.advanceToken(); - if (typeNameToken.getContent() != node->m_name.getContent()) - { - m_sink->diagnose(typeNameToken, CPPDiagnostics::typeNameDoesntMatch, node->m_name.getContent()); - return SLANG_FAIL; - } - - node->m_reflectionType = ReflectionType::Reflected; + // Push the class scope return pushScope(node); } + SlangResult Parser::_consumeToSync() { @@ -776,33 +794,55 @@ UnownedStringSlice Parser::_concatTokens(TokenReader::ParsingCursor start) SlangResult Parser::_maybeParseType(Index& ioTemplateDepth) { - _consumeTypeModifiers(); - - advanceIfToken(TokenType::Scope); while (true) { - Token identifierToken; - if (!advanceIfToken(TokenType::Identifier, &identifierToken)) + if (m_reader.peekTokenType() == TokenType::Identifier) { - return SLANG_FAIL; + const IdentifierStyle style = m_nodeTree->m_identifierLookup->get(m_reader.peekToken().getContent()); + + if (style != IdentifierStyle::TypeModifier && hasFlag(style, IdentifierFlag::Keyword)) + { + return SLANG_FAIL; + } } - const IdentifierStyle style = m_nodeTree->m_identifierLookup->get(identifierToken.getContent()); - if (hasFlag(style, IdentifierFlag::Keyword)) + _consumeTypeModifiers(); + + advanceIfToken(TokenType::Scope); + while (true) { - return SLANG_FAIL; + Token identifierToken; + if (!advanceIfToken(TokenType::Identifier, &identifierToken)) + { + return SLANG_FAIL; + } + + const IdentifierStyle style = m_nodeTree->m_identifierLookup->get(identifierToken.getContent()); + if (hasFlag(style, IdentifierFlag::Keyword)) + { + return SLANG_FAIL; + } + + if (advanceIfToken(TokenType::Scope)) + { + continue; + } + break; } - if (advanceIfToken(TokenType::Scope)) + if (m_reader.peekTokenType() == TokenType::OpLess) { + SLANG_RETURN_ON_FAIL(_maybeParseTemplateArgs(ioTemplateDepth)); + } + + if (m_reader.peekTokenType() == TokenType::Scope) + { + // Skip the scope and repeat + m_reader.advanceToken(); continue; } - break; - } - if (m_reader.peekTokenType() == TokenType::OpLess) - { - SLANG_RETURN_ON_FAIL(_maybeParseTemplateArgs(ioTemplateDepth)); + break; } // Strip all the consts etc modifiers @@ -856,6 +896,61 @@ SlangResult Parser::_maybeParseType(List<Token>& outToks) return SLANG_OK; } +SlangResult Parser::_parseMarker() +{ + SLANG_ASSERT(m_reader.peekTokenType() == TokenType::Identifier && + _isMarker(m_reader.peekToken().getContent()) && + m_currentScope->isClassLike()); + + ClassLikeNode* node = as<ClassLikeNode>(m_currentScope); + + if (node->m_marker.type != TokenType::Unknown) + { + m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::classMarkerAlreadyFound, node->m_name.getContent()); + m_sink->diagnose(node->m_marker, CPPDiagnostics::previousLocation); + return SLANG_FAIL; + } + + // Set the marker token. + node->m_marker = m_reader.advanceToken(); + + // Looks like it's a marker + UnownedStringSlice slice(node->m_marker.getContent()); + + // Strip the prefix and suffix + slice = UnownedStringSlice(slice.begin() + m_options->m_markPrefix.getLength(), slice.end() - m_options->m_markSuffix.getLength()); + + // Strip ABSTRACT_ if it's there + UnownedStringSlice abstractSlice("ABSTRACT_"); + if (slice.startsWith(abstractSlice)) + { + slice = UnownedStringSlice(slice.begin() + abstractSlice.getLength(), slice.end()); + } + + // TODO: We could strip other stuff or have other heuristics there, but this is + // probably okay for now + + // Set the typeSet + node->m_typeSet = m_nodeTree->getOrAddTypeSet(slice); + + // Okay now looking for ( identifier) + Token typeNameToken; + + SLANG_RETURN_ON_FAIL(expect(TokenType::LParent)); + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &typeNameToken)); + SLANG_RETURN_ON_FAIL(expect(TokenType::RParent)); + + if (typeNameToken.getContent() != node->m_name.getContent()) + { + m_sink->diagnose(typeNameToken, CPPDiagnostics::typeNameDoesntMatch, node->m_name.getContent()); + return SLANG_FAIL; + } + + // If has the marker it is assumed reflected + node->m_reflectionType = ReflectionType::Reflected; + return SLANG_OK; +} + SlangResult Parser::_maybeParseType(UnownedStringSlice& outType) { auto startCursor = m_reader.getCursor(); @@ -965,6 +1060,130 @@ SlangResult Parser::_parseBalanced(DiagnosticSink* sink) } } +SlangResult Parser::_consumeBalancedParens() +{ + SLANG_ASSERT(m_reader.peekTokenType() == TokenType::LParent); + + Index parenCount = 0; + + while (true) + { + const TokenType tokenType = m_reader.peekTokenType(); + + switch (tokenType) + { + case TokenType::LParent: + { + parenCount++; + break; + } + case TokenType::RParent: + { + --parenCount; + // If no more parens then we are done + if (parenCount == 0) + { + m_reader.advanceToken(); + return SLANG_OK; + } + break; + } + case TokenType::EndOfFile: + { + // If we hit the end of the file, then not balanced + return SLANG_FAIL; + } + default: break; + } + + m_reader.advanceToken(); + } +} + +SlangResult Parser::_parseExpression(List<Token>& outExprTokens) +{ + Index parenCount = 0; + Index bracketCount = 0; + + // TODO(JS): NOTE! This doesn't handle an expression that contains a template params in Something<Arg1, 3>, + // because without knowing what Something is, it's not known if < is a comparison or or a 'template' bracket + // + // This can be worked around in the originating source by placing in parens + + while (true) + { + TokenType tokenType = m_reader.peekTokenType(); + + switch (tokenType) + { + case TokenType::LParent: + { + parenCount++; + break; + } + case TokenType::RParent: + { + // If no parens, and nothing else is open then we are done + if (parenCount == 0) + { + if (bracketCount) + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::cannotParseExpression); + return SLANG_FAIL; + } + + return SLANG_OK; + } + --parenCount; + break; + } + case TokenType::LBracket: + { + bracketCount++; + break; + } + case TokenType::RBracket: + { + // If no brackets are open we are done + if (bracketCount == 0) + { + if (parenCount) + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::cannotParseExpression); + return SLANG_FAIL; + } + return SLANG_OK; + } + --bracketCount; + break; + } + case TokenType::EndOfFile: + { + if ((bracketCount | parenCount) == 0) + { + return SLANG_OK; + } + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::cannotParseExpression); + return SLANG_FAIL; + } + case TokenType::RBrace: + case TokenType::Semicolon: + case TokenType::Comma: + { + if ((bracketCount | parenCount) == 0) + { + return SLANG_OK; + } + break; + } + + default: break; + } + + outExprTokens.add(m_reader.advanceToken()); + } +} + SlangResult Parser::_parseTypeDef() { if (!m_currentScope->canContainTypes()) @@ -995,6 +1214,7 @@ SlangResult Parser::_parseTypeDef() RefPtr<TypeDefNode> node = new TypeDefNode; node->m_name = name; + node->m_reflectionType = m_currentScope->getContainedReflectionType(); // Set what aliases too node->m_targetTypeTokens.swapWith(toks); @@ -1004,10 +1224,35 @@ SlangResult Parser::_parseTypeDef() return SLANG_OK; } -SlangResult Parser::_maybeParseField() +SlangResult Parser::_maybeParseContained(Node** outNode) { - // Can only add a field if we are in a class - SLANG_ASSERT(m_currentScope->isClassLike()); + *outNode = nullptr; + + bool isStatic = false; + bool isVirtual = false; + + while (m_reader.peekTokenType() == TokenType::Identifier) + { + const IdentifierStyle style = m_nodeTree->m_identifierLookup->get(m_reader.peekToken().getContent()); + + // Check for virtualness + if (style == IdentifierStyle::Virtual) + { + isVirtual = true; + m_reader.advanceToken(); + continue; + } + + // Check if static + if (style == IdentifierStyle::Static) + { + isStatic = true; + m_reader.advanceToken(); + continue; + } + + break; + } UnownedStringSlice typeName; if (SLANG_FAILED(_maybeParseType(typeName))) @@ -1021,64 +1266,234 @@ SlangResult Parser::_maybeParseField() return SLANG_OK; } - if (m_reader.peekTokenType() != TokenType::Identifier) + // Has a calling convention (must be a function/method) + Token callingConventionToken; + advanceIfStyle(IdentifierStyle::CallingConvention, &callingConventionToken); + + // Expecting a name + Token nameToken; + if (!advanceIfToken(TokenType::Identifier, &nameToken)) { _consumeToSync(); return SLANG_OK; } - Token fieldName = m_reader.advanceToken(); - - if (m_reader.peekTokenType() == TokenType::LBracket) + // Handles other scenarios, but here for catching operator overloading { - auto startCursor = m_reader.getCursor(); - - // If it's not balanced we just assume it's not correct - and ignore - if (SLANG_FAILED(_parseBalanced(nullptr))) + const auto style = m_nodeTree->m_identifierLookup->get(nameToken.getContent()); + if (style != IdentifierStyle::None) { _consumeToSync(); return SLANG_OK; } + } - UnownedStringSlice arraySuffix = _concatTokens(startCursor); + if (m_reader.peekTokenType() == TokenType::LParent) + { + if (!m_currentScope->canContainCallable()) + { + SLANG_RETURN_ON_FAIL(_consumeBalancedParens()); + // Consume everything up to ; or { + SLANG_RETURN_ON_FAIL(_consumeToSync()); + + return SLANG_OK; + } - // The overall type is the typename concated with the arraySuffix - StringBuilder buf; - buf << typeName << arraySuffix; + // Looks like it's a callable + m_reader.advanceToken(); - StringSlicePool* typePool = m_nodeTree->m_typePool; + List<CallableNode::Param> params; - typeName = typePool->getSlice(typePool->add(buf)); - } + if (m_reader.peekTokenType() != TokenType::RParent) + { + while (true) + { + UnownedStringSlice type; + SlangResult res = _maybeParseType(type); - switch (m_reader.peekTokenType()) - { - case TokenType::OpAssign: + if (SLANG_FAILED(res)) + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::expectingType); + return res; + } + + Token paramName; + if (m_reader.peekTokenType() == TokenType::Identifier) + { + paramName = m_reader.advanceToken(); + } + + // Check if we have a default value + auto peekType = m_reader.peekTokenType(); + + if (peekType == TokenType::OpAssign) + { + m_reader.advanceToken(); + + List<Token> exprTokens; + SLANG_RETURN_ON_FAIL(_parseExpression(exprTokens)); + + // Update the peek type + peekType = m_reader.peekTokenType(); + } + + CallableNode::Param param; + param.m_name = nameToken; + param.m_type = type; + + params.add(param); + + if (peekType == TokenType::RParent) + { + break; + } + if (peekType == TokenType::Comma) + { + m_reader.advanceToken(); + continue; + } + + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::expectingToken, ", or ) or ="); + return SLANG_FAIL; + } + } + + // Skip ) + m_reader.advanceToken(); + + // Parse suffix + bool isPure = false; + + // const? + _consumeTypeModifiers(); + + // = 0 ? + if (m_reader.peekTokenType() == TokenType::OpAssign) { - // Special case to handle - // Type operator=(... + m_reader.advanceToken(); + if (m_reader.peekTokenType() == TokenType::IntegerLiteral) + { + Int value = -1; + if (SLANG_SUCCEEDED(StringUtil::parseInt(m_reader.peekToken().getContent(), value)) && + value == 0) + { + isPure = true; + m_reader.advanceToken(); + } + else + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::expectingToken, "0"); + return SLANG_FAIL; + } + } + } + if (m_reader.peekTokenType() == TokenType::Semicolon) + { m_reader.advanceToken(); - if (m_reader.peekTokenType() == TokenType::LParent) + } + else if (m_reader.peekTokenType() == TokenType::LBrace) + { + SLANG_RETURN_ON_FAIL(consumeToClosingBrace()); + } + else + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::expectingToken, "; or {"); + return SLANG_FAIL; + } + + RefPtr<CallableNode> callableNode = new CallableNode; + + callableNode->m_returnType = typeName; + callableNode->m_name = nameToken; + callableNode->m_reflectionType = m_currentScope->getContainedReflectionType(); + + callableNode->m_isVirtual = isVirtual; + callableNode->m_isPure = isPure; + callableNode->m_isStatic = isStatic; + + callableNode->m_params.swapWith(params); + + Node* nodeWithName = m_currentScope->findChild(nameToken.getContent()); + + if (nodeWithName) + { + CallableNode* initialOverload = as<CallableNode>(nodeWithName); + if (!initialOverload) { - // Not a field - break; + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::cannotOverload); + m_sink->diagnose(nodeWithName->getSourceLoc(), CPPDiagnostics::seeDeclarationOf); + return SLANG_FAIL; + } + + callableNode->m_nextOverload = initialOverload->m_nextOverload; + initialOverload->m_nextOverload = initialOverload; + + m_currentScope->addChildIgnoringName(callableNode); + } + else + { + m_currentScope->addChild(callableNode); + } + + *outNode = callableNode; + return SLANG_OK; + } + else + { + // Looks like variable + if (!m_currentScope->canContainFields()) + { + _consumeToSync(); + return SLANG_OK; + } + + if (m_reader.peekTokenType() == TokenType::LBracket) + { + auto startCursor = m_reader.getCursor(); + + // If it's not balanced we just assume it's not correct - and ignore + if (SLANG_FAILED(_parseBalanced(nullptr))) + { + _consumeToSync(); + return SLANG_OK; } + + UnownedStringSlice arraySuffix = _concatTokens(startCursor); + + // The overall type is the typename concated with the arraySuffix + StringBuilder buf; + buf << typeName << arraySuffix; + + StringSlicePool* typePool = m_nodeTree->m_typePool; + + typeName = typePool->getSlice(typePool->add(buf)); } - case TokenType::Semicolon: + + // Check if has a default value + if (advanceIfToken(TokenType::OpAssign)) { - FieldNode* fieldNode = new FieldNode; + List<Token> exprTokens; + SLANG_RETURN_ON_FAIL(_parseExpression(exprTokens)); + } + + // Hit end of field/variable + if (m_reader.peekTokenType() == TokenType::Semicolon) + { + RefPtr<FieldNode> fieldNode = new FieldNode; fieldNode->m_fieldType = typeName; - fieldNode->m_name = fieldName; + fieldNode->m_name = nameToken; fieldNode->m_reflectionType = m_currentScope->getContainedReflectionType(); + fieldNode->m_isStatic = isStatic; m_currentScope->addChild(fieldNode); - break; + + *outNode = fieldNode; + return SLANG_OK; } - default: break; } - + _consumeToSync(); return SLANG_OK; } @@ -1243,6 +1658,11 @@ SlangResult Parser::parse(SourceOrigin* sourceOrigin, const Options* options) switch (style) { + case IdentifierStyle::Template: + { + SLANG_RETURN_ON_FAIL(_consumeTemplate()); + break; + } case IdentifierStyle::PreDeclare: { SLANG_RETURN_ON_FAIL(_parsePreDeclare()); @@ -1310,11 +1730,25 @@ SlangResult Parser::parse(SourceOrigin* sourceOrigin, const Options* options) } else { + // If it's a marker handle it + if (_isMarker(m_reader.peekToken().getContent())) + { + if (!m_currentScope->isClassLike()) + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::classMarkerOutsideOfClass); + return SLANG_FAIL; + } + + SLANG_RETURN_ON_FAIL(_parseMarker()); + break; + } + // Special case the node that's the root of the hierarchy (as far as reflection is concerned) // This could be a field - if (m_currentScope->canContainFields()) + if (m_currentScope->canContainFields() || m_currentScope->canContainCallable()) { - SLANG_RETURN_ON_FAIL(_maybeParseField()); + Node* containedNode = nullptr; + SLANG_RETURN_ON_FAIL(_maybeParseContained(&containedNode)); } else { diff --git a/tools/slang-cpp-extractor/parser.h b/tools/slang-cpp-extractor/parser.h index 5ed862d65..4815821c5 100644 --- a/tools/slang-cpp-extractor/parser.h +++ b/tools/slang-cpp-extractor/parser.h @@ -33,6 +33,11 @@ public: void setKindEnabled(Node::Kind kind, bool isEnabled = true); bool isTypeEnabled(Node::Kind kind) { return (m_nodeTypeEnabled & (NodeTypeBitType(1) << int(kind))) != 0; } + + /// If set classes require a 'marker' to be reflected + void setRequireMarker(bool requireMarker) { m_requireMarker = requireMarker; } + bool getRequireMarker() const { return m_requireMarker; } + void setKindsEnabled(const Node::Kind* kinds, Index kindsCount, bool isEnabled = true); Parser(NodeTree* nodeTree, DiagnosticSink* sink); @@ -48,14 +53,17 @@ protected: SlangResult _parseTypeSet(); SlangResult _maybeParseNode(Node::Kind kind); - SlangResult _maybeParseField(); + SlangResult _maybeParseContained(Node** outNode); SlangResult _parseTypeDef(); SlangResult _parseEnum(); + SlangResult _parseMarker(); SlangResult _maybeParseType(List<Token>& outToks); SlangResult _maybeParseType(UnownedStringSlice& outType); + SlangResult _parseExpression(List<Token>& outExprTokens); + SlangResult _maybeParseType(Index& ioTemplateDepth); SlangResult _maybeParseTemplateArgs(Index& ioTemplateDepth); SlangResult _maybeParseTemplateArg(Index& ioTemplateDepth); @@ -66,9 +74,14 @@ protected: /// Concatenate all tokens from start to the current position UnownedStringSlice _concatTokens(TokenReader::ParsingCursor start); + /// Consume what looks like a template definition + SlangResult _consumeTemplate(); + void _consumeTypeModifiers(); SlangResult _consumeToSync(); + /// Consumes balanced parens. Will return an error if not matched. Assumes starts on opening ( + SlangResult _consumeBalancedParens(); NodeTypeBitType m_nodeTypeEnabled; @@ -82,6 +95,8 @@ protected: NodeTree* m_nodeTree; ///< Shared state between parses. Nodes will be added to this + bool m_requireMarker = true; + const Options* m_options; }; diff --git a/tools/slang-cpp-extractor/unit-test.cpp b/tools/slang-cpp-extractor/unit-test.cpp index a30a3caa7..c239a2a2c 100644 --- a/tools/slang-cpp-extractor/unit-test.cpp +++ b/tools/slang-cpp-extractor/unit-test.cpp @@ -41,12 +41,26 @@ struct TestState }; static const char someSource[] = +"#define SLANG_REFLECTED\n" +"\n" +"SLANG_REFLECTED\n" +"class ISomeInterface\n" +"{\n" +" public:\n" +" virtual int SLANG_MCALL someMethod(int a, int b) const = 0;\n" +" virtual float SLANG_MCALL anotherMethod(float a) = 0;\n" +"};\n" +"\n" "enum SomeEnum\n" "{\n" " Value,\n" " Another = 10,\n" "};\n" -"typedef SomeEnum AliasEnum;\n"; +"typedef SomeEnum AliasEnum;\n" +"void someFunc(int a, float b) { }\n" +"namespace Blah {\n" +"int add(int a, int b) { return a + b; }\n" +"}\n"; /* static */SlangResult UnitTestUtil::run() @@ -65,6 +79,8 @@ static const char someSource[] = SourceOrigin* sourceOrigin = tree.addSourceOrigin(sourceFile, state.m_options); Parser parser(&tree, &state.m_sink); + // We don't require markers to reflect + parser.setRequireMarker(false); { const Node::Kind enableKinds[] = { Node::Kind::Enum, Node::Kind::EnumClass, Node::Kind::EnumCase, Node::Kind::TypeDef }; @@ -72,6 +88,14 @@ static const char someSource[] = } SLANG_RETURN_ON_FAIL(parser.parse(sourceOrigin, &state.m_options)); + + + { + StringBuilder buf; + tree.getRootNode()->dump(0, buf); + + SLANG_UNUSED(buf); + } } return SLANG_OK; |