diff options
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/slang-cpp-extractor/cpp-extractor-main.cpp | 15 | ||||
| -rw-r--r-- | tools/slang-cpp-extractor/diagnostic-defs.h | 3 | ||||
| -rw-r--r-- | tools/slang-cpp-extractor/identifier-lookup.cpp | 35 | ||||
| -rw-r--r-- | tools/slang-cpp-extractor/identifier-lookup.h | 10 | ||||
| -rw-r--r-- | tools/slang-cpp-extractor/node-tree.cpp | 2 | ||||
| -rw-r--r-- | tools/slang-cpp-extractor/options.cpp | 7 | ||||
| -rw-r--r-- | tools/slang-cpp-extractor/options.h | 1 | ||||
| -rw-r--r-- | tools/slang-cpp-extractor/parser.cpp | 525 | ||||
| -rw-r--r-- | tools/slang-cpp-extractor/parser.h | 27 | ||||
| -rw-r--r-- | tools/slang-cpp-extractor/unit-test.cpp | 35 |
10 files changed, 546 insertions, 114 deletions
diff --git a/tools/slang-cpp-extractor/cpp-extractor-main.cpp b/tools/slang-cpp-extractor/cpp-extractor-main.cpp index 371392851..6fbba20c3 100644 --- a/tools/slang-cpp-extractor/cpp-extractor-main.cpp +++ b/tools/slang-cpp-extractor/cpp-extractor-main.cpp @@ -314,6 +314,21 @@ SlangResult App::executeWithArgs(int argc, const char*const* argv) } // namespace CppExtract + +/* +The typical command line for producing generated slang files. Can be determined by setting `dumpCommandLine` belong and compiling. + +``` +-d E:\git\slang-jsmall-nvidia\source\slang\ slang-ast-support-types.h slang-ast-base.h slang-ast-decl.h slang-ast-expr.h slang-ast-modifier.h slang-ast-stmt.h slang-ast-type.h slang-ast-val.h -strip-prefix slang- -o slang-generated -output-fields -mark-suffix _CLASS +``` + +A command line to try and parse the slang.h + +``` +-d E:\git\slang-jsmall-nvidia slang.h -mark-suffix _CLASS -dump -unmarked -unit-test +``` +*/ + int main(int argc, const char*const* argv) { using namespace CppExtract; diff --git a/tools/slang-cpp-extractor/diagnostic-defs.h b/tools/slang-cpp-extractor/diagnostic-defs.h index c7e364291..85e88d64d 100644 --- a/tools/slang-cpp-extractor/diagnostic-defs.h +++ b/tools/slang-cpp-extractor/diagnostic-defs.h @@ -52,6 +52,9 @@ DIAGNOSTIC(100017, Error, cannotParseExpression, "Cannot parse expression") DIAGNOSTIC(100018, Error, cannotOverload, "Cannot overload this declaration"); DIAGNOSTIC(100019, Error, classMarkerOutsideOfClass, "A class/struct marker is found outside of class or struct"); DIAGNOSTIC(100020, Error, classMarkerAlreadyFound, "A class/struct marker already found in type '$0'"); +DIAGNOSTIC(100021, Error, cannotParseType, "Cannot parse type"); +DIAGNOSTIC(100022, Error, destructorNameDoesntMatch, "Destructor name doesn't match class name '$0'"); +DIAGNOSTIC(100023, Error, cannotParseCallable, "Cannot parse callable"); // Command line errors 100100 diff --git a/tools/slang-cpp-extractor/identifier-lookup.cpp b/tools/slang-cpp-extractor/identifier-lookup.cpp index db8ea2846..c429a29a4 100644 --- a/tools/slang-cpp-extractor/identifier-lookup.cpp +++ b/tools/slang-cpp-extractor/identifier-lookup.cpp @@ -27,6 +27,15 @@ using namespace Slang; 0, /// Calling convention IdentifierFlag::Keyword, /// template IdentifierFlag::Keyword, /// static + + IdentifierFlag::Keyword, /// unsigned/signed + + IdentifierFlag::Keyword, /// extern + + 0, /// Callable misc + 0, /// IntegerType int, short, char, long + + IdentifierFlag::Keyword, /// default }; void IdentifierLookup::set(const UnownedStringSlice& name, IdentifierStyle style) @@ -68,7 +77,7 @@ void IdentifierLookup::initDefault(const UnownedStringSlice& markPrefix) // Some keywords { - const char* names[] = { "continue", "if", "case", "break", "catch", "default", "delete", "do", "else", "for", "new", "goto", "return", "switch", "throw", "using", "while", "operator", "explicit"}; + const char* names[] = { "continue", "if", "case", "break", "catch", "delete", "do", "else", "for", "new", "goto", "return", "switch", "throw", "using", "while", "operator", "explicit"}; set(names, SLANG_COUNT_OF(names), IdentifierStyle::Keyword); } @@ -95,9 +104,26 @@ void IdentifierLookup::initDefault(const UnownedStringSlice& markPrefix) { set("virtual", IdentifierStyle::Virtual); - set("SLANG_MCALL", IdentifierStyle::CallingConvention); + set("template", IdentifierStyle::Template); set("static", IdentifierStyle::Static); + set("extern", IdentifierStyle::Extern); + set("default", IdentifierStyle::Default); + } + + { + const char* names[] = { "char", "short", "int", "long"}; + set(names, SLANG_COUNT_OF(names), IdentifierStyle::IntegerType); + } + + { + const char* names[] = { "SLANG_MCALL" }; + set(names, SLANG_COUNT_OF(names), IdentifierStyle::CallingConvention); + } + + { + const char* names[] = { "SLANG_NO_THROW", "inline"}; + set(names, SLANG_COUNT_OF(names), IdentifierStyle::CallableMisc); } // Keywords which introduce types/scopes @@ -119,6 +145,11 @@ void IdentifierLookup::initDefault(const UnownedStringSlice& markPrefix) const char* names[] = { "private", "protected", "public" }; set(names, SLANG_COUNT_OF(names), IdentifierStyle::Access); } + { + const char* names[] = { "signed", "unsigned"}; + + set(names, SLANG_COUNT_OF(names), IdentifierStyle::IntegerModifier); + } } } // namespace CppExtract diff --git a/tools/slang-cpp-extractor/identifier-lookup.h b/tools/slang-cpp-extractor/identifier-lookup.h index 40644a1ba..0d55ba65c 100644 --- a/tools/slang-cpp-extractor/identifier-lookup.h +++ b/tools/slang-cpp-extractor/identifier-lookup.h @@ -37,6 +37,16 @@ enum class IdentifierStyle Static, + IntegerModifier, + + Extern, + + CallableMisc, ///< For SLANG_NO_THROW etc + + IntegerType, ///< Built in integer type + + Default, /// default + CountOf, }; diff --git a/tools/slang-cpp-extractor/node-tree.cpp b/tools/slang-cpp-extractor/node-tree.cpp index 05360e0de..f21d912a7 100644 --- a/tools/slang-cpp-extractor/node-tree.cpp +++ b/tools/slang-cpp-extractor/node-tree.cpp @@ -131,7 +131,7 @@ SlangResult NodeTree::_calcDerivedTypesRec(ScopeNode* inScopeNode, DiagnosticSin else { // Add to it's own typeset - if (classLikeNode->isReflected()) + if (classLikeNode->isReflected() && classLikeNode->m_typeSet) { classLikeNode->m_typeSet->m_baseTypes.add(classLikeNode); } diff --git a/tools/slang-cpp-extractor/options.cpp b/tools/slang-cpp-extractor/options.cpp index 9ec671652..90639da38 100644 --- a/tools/slang-cpp-extractor/options.cpp +++ b/tools/slang-cpp-extractor/options.cpp @@ -114,6 +114,13 @@ SlangResult OptionsParser::parse(int argc, const char*const* argv, DiagnosticSin SLANG_RETURN_ON_FAIL(_parseArgFlag("-unit-test", outOptions.m_runUnitTests)); continue; } + else if (arg == "-unmarked") + { + bool unmarked; + SLANG_RETURN_ON_FAIL(_parseArgFlag("-unmarked", unmarked)); + outOptions.m_requireMark = !unmarked; + continue; + } m_sink->diagnose(SourceLoc(), CPPDiagnostics::unknownOption, arg); return SLANG_FAIL; diff --git a/tools/slang-cpp-extractor/options.h b/tools/slang-cpp-extractor/options.h index 186ac2fbb..e660bc376 100644 --- a/tools/slang-cpp-extractor/options.h +++ b/tools/slang-cpp-extractor/options.h @@ -28,6 +28,7 @@ struct Options bool m_extractDoc = true; ///< If set will try to extract documentation associated with nodes bool m_outputFields = false; ///< When dumping macros also dump field definitions + bool m_requireMark = true; List<String> m_inputPaths; ///< The input paths to the files to be processed diff --git a/tools/slang-cpp-extractor/parser.cpp b/tools/slang-cpp-extractor/parser.cpp index 3f87344ed..7ceb91824 100644 --- a/tools/slang-cpp-extractor/parser.cpp +++ b/tools/slang-cpp-extractor/parser.cpp @@ -141,11 +141,26 @@ SlangResult Parser::pushAnonymousNamespace() m_sourceOrigin->addNode(m_currentScope); } + // Add the to the scope stack so can pop. + m_scopeStack.add(m_currentScope); + return SLANG_OK; } SlangResult Parser::pushScope(ScopeNode* scopeNode) { + // We can only have one 'special' scope. + SLANG_ASSERT(scopeNode || m_scopeStack.getLast()); + + // We keep to track. + m_scopeStack.add(scopeNode); + + // If we pass nullptr, we don't update the current scope. + if (scopeNode == nullptr) + { + return SLANG_OK; + } + if (m_sourceOrigin) { m_sourceOrigin->addNode(scopeNode); @@ -195,12 +210,21 @@ SlangResult Parser::pushScope(ScopeNode* scopeNode) SlangResult Parser::popScope() { - if (m_currentScope->m_parentScope == nullptr) + if (m_scopeStack.getCount() <= 0) { m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::scopeNotClosed); return SLANG_FAIL; } + ScopeNode* topScope = m_scopeStack.getLast(); + m_scopeStack.removeLast(); + + // If the top is nullptr, we don't change the current scope + if (topScope == nullptr) + { + return SLANG_OK; + } + m_currentScope = m_currentScope->m_parentScope; return SLANG_OK; } @@ -558,6 +582,7 @@ SlangResult Parser::_maybeParseNode(Node::Kind kind) // Next is the class name SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &name)); + if (m_reader.peekTokenType() == TokenType::Semicolon) { // pre declaration; @@ -568,7 +593,7 @@ SlangResult Parser::_maybeParseNode(Node::Kind kind) node->m_name = name; // We default to the containing scope for reflection type. - if (!m_requireMarker) + if (!m_options->m_requireMark) { node->m_reflectionType = m_currentScope->getContainedReflectionType(); } @@ -619,7 +644,6 @@ SlangResult Parser::_maybeParseNode(Node::Kind kind) return pushScope(node); } - SlangResult Parser::_consumeToSync() { while (true) @@ -652,7 +676,8 @@ SlangResult Parser::_maybeParseTemplateArg(Index& ioTemplateDepth) { case TokenType::Identifier: { - SLANG_RETURN_ON_FAIL(_maybeParseType(ioTemplateDepth)); + TokenReader::ParsingCursor nameCursor; + SLANG_RETURN_ON_FAIL(_maybeParseType(ioTemplateDepth, nameCursor)); return SLANG_OK; } case TokenType::IntegerLiteral: @@ -725,9 +750,10 @@ SlangResult Parser::_maybeParseTemplateArgs(Index& ioTemplateDepth) } } -void Parser::_consumeTypeModifiers() +SlangResult Parser::_maybeConsume(IdentifierStyle style) { - while (advanceIfStyle(IdentifierStyle::TypeModifier)); + while (advanceIfStyle(style)); + return SLANG_OK; } // True if two of these token types of the same type placed immediately after one another @@ -767,9 +793,59 @@ static bool _tokenConcatNeedsSpace(TokenType prev, TokenType cur) return false; } +void Parser::_getTypeTokens(TokenReader::ParsingCursor start, TokenReader::ParsingCursor nameCursor, List<Token>& outToks) +{ + auto endCursor = m_reader.getCursor(); + m_reader.setCursor(start); + + while (!m_reader.isAtCursor(endCursor)) + { + if (m_reader.getCursor() == nameCursor) + { + m_reader.advanceToken(); + } + else + { + outToks.add(m_reader.advanceToken()); + } + } +} + +UnownedStringSlice Parser::_concatType(TokenReader::ParsingCursor start, TokenReader::ParsingCursor nameCursor) +{ + List<Token> toks; + _getTypeTokens(start, nameCursor, toks); + return _concatTokens(toks.getBuffer(), toks.getCount()); +} + +UnownedStringSlice Parser::_concatTokens(const Token* toks, Index toksCount) +{ + StringBuilder buf; + + TokenType prevTokenType = TokenType::Unknown; + for (Index i = 0; i < toksCount; ++i) + { + const auto token = toks[i]; + + // Check if we need a space between tokens + if (_tokenConcatNeedsSpace(prevTokenType, token.type)) + { + buf << " "; + } + + buf << token.getContent(); + + prevTokenType = token.type; + } + + StringSlicePool* typePool = m_nodeTree->m_typePool; + return typePool->getSlice(typePool->add(buf)); +} + UnownedStringSlice Parser::_concatTokens(TokenReader::ParsingCursor start) { auto endCursor = m_reader.getCursor(); + m_reader.setCursor(start); TokenType prevTokenType = TokenType::Unknown; @@ -792,25 +868,60 @@ UnownedStringSlice Parser::_concatTokens(TokenReader::ParsingCursor start) return typePool->getSlice(typePool->add(buf)); } -SlangResult Parser::_maybeParseType(Index& ioTemplateDepth) +SlangResult Parser::_maybeParseType(Index& ioTemplateDepth, TokenReader::ParsingCursor& outNameCursor) { + outNameCursor = TokenReader::ParsingCursor(); + while (true) { if (m_reader.peekTokenType() == TokenType::Identifier) { const IdentifierStyle style = m_nodeTree->m_identifierLookup->get(m_reader.peekToken().getContent()); - if (style != IdentifierStyle::TypeModifier && hasFlag(style, IdentifierFlag::Keyword)) + if (style == IdentifierStyle::TypeModifier || + style == IdentifierStyle::IntegerModifier || + style == IdentifierStyle::Class || + style == IdentifierStyle::Struct) + { + // These are ok keywords in this context + } + else if (hasFlag(style, IdentifierFlag::Keyword)) { return SLANG_FAIL; } } - _consumeTypeModifiers(); + _maybeConsume(IdentifierStyle::TypeModifier); + if (advanceIfStyle(IdentifierStyle::IntegerModifier)) + { + // Consume the integer typename (if there is one) + const Token peekToken = m_reader.peekToken(); + if (peekToken.type == TokenType::Identifier) + { + const IdentifierStyle style = m_nodeTree->m_identifierLookup->get(peekToken.getContent()); + if (style == IdentifierStyle::IntegerType) + { + m_reader.advanceToken(); + } + } + break; + } + advanceIfToken(TokenType::Scope); while (true) { + // if we have a struct/class prefix in front of a name just consume it. + if (m_reader.peekTokenType() == TokenType::Identifier) + { + const IdentifierStyle style = m_nodeTree->m_identifierLookup->get(m_reader.peekToken().getContent()); + if (style == IdentifierStyle::Class || + style == IdentifierStyle::Struct) + { + m_reader.advanceToken(); + } + } + Token identifierToken; if (!advanceIfToken(TokenType::Identifier, &identifierToken)) { @@ -846,7 +957,7 @@ SlangResult Parser::_maybeParseType(Index& ioTemplateDepth) } // Strip all the consts etc modifiers - _consumeTypeModifiers(); + _maybeConsume(IdentifierStyle::TypeModifier); // It's a reference and we are done if (advanceIfToken(TokenType::OpBitAnd)) @@ -859,21 +970,68 @@ SlangResult Parser::_maybeParseType(Index& ioTemplateDepth) if (advanceIfToken(TokenType::OpMul)) { // Strip all the consts - _consumeTypeModifiers(); + _maybeConsume(IdentifierStyle::TypeModifier); continue; } break; } + if (advanceIfToken(TokenType::LParent)) + { + // TODO(JS): + // Doesn't handle all the modifiers just (*SomeName) + + SLANG_RETURN_ON_FAIL(expect(TokenType::OpMul)); + outNameCursor = m_reader.getCursor(); + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier)); + + SLANG_RETURN_ON_FAIL(expect(TokenType::RParent)); + + // We need to parse and add the params + if (m_reader.peekTokenType() != TokenType::LParent) + { + m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::expectingToken, TokenType::LParent); + return SLANG_FAIL; + } + + // Consume the params + SLANG_RETURN_ON_FAIL(_consumeBalancedParens()); + } + else if (m_reader.peekTokenType() == TokenType::Identifier) + { + auto potentialNameCursor = m_reader.getCursor(); + m_reader.advanceToken(); + if (m_reader.peekTokenType() == TokenType::LBracket) + { + outNameCursor = potentialNameCursor; + while (advanceIfToken(TokenType::LBracket)) + { + List<Token> exprToks; + SLANG_RETURN_ON_FAIL(_parseExpression(exprToks)); + SLANG_RETURN_ON_FAIL(expect(TokenType::RBracket)); + } + } + else + { + // Wasn't an array type..., so rewind + m_reader.setCursor(potentialNameCursor); + } + } + return SLANG_OK; } -SlangResult Parser::_maybeParseType(List<Token>& outToks) +SlangResult Parser::_maybeParseType(List<Token>& outToks, Token& outName) { + // Set to unknown + outName = Token(); + auto startCursor = m_reader.getCursor(); + TokenReader::ParsingCursor nameCursor; + Index templateDepth = 0; - SlangResult res = _maybeParseType(templateDepth); + SlangResult res = _maybeParseType(templateDepth, nameCursor); if (SLANG_FAILED(res) && m_sink->getErrorCount()) { return res; @@ -888,9 +1046,61 @@ SlangResult Parser::_maybeParseType(List<Token>& outToks) auto endCursor = m_reader.getCursor(); m_reader.setCursor(startCursor); - while (!m_reader.isAtCursor(endCursor)) + if (nameCursor.isValid()) + { + while (!m_reader.isAtCursor(endCursor)) + { + if (m_reader.getCursor() == nameCursor) + { + outName = m_reader.advanceToken(); + } + else + { + outToks.add(m_reader.advanceToken()); + } + } + } + else { - outToks.add(m_reader.advanceToken()); + while (!m_reader.isAtCursor(endCursor)) + { + outToks.add(m_reader.advanceToken()); + } + } + + return SLANG_OK; +} + +SlangResult Parser::_parseSpecialMacro() +{ + Token name; + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &name)); + + List<Token> params; + + if (m_reader.peekTokenType() == TokenType::LParent) + { + // Mark the start + auto startCursor = m_reader.getCursor(); + + // Consume the params + SLANG_RETURN_ON_FAIL(_consumeBalancedParens()); + + auto endCursor = m_reader.getCursor(); + m_reader.setCursor(startCursor); + + while (!m_reader.isAtCursor(endCursor)) + { + params.add(m_reader.advanceToken()); + } + } + + // Can do special handling here + const UnownedStringSlice suffix = name.getContent().tail(m_options->m_markPrefix.getLength()); + + if (suffix == "COM_INTERFACE") + { + // TODO(JS): It's a com interface. Extact the GUID } return SLANG_OK; @@ -951,12 +1161,15 @@ SlangResult Parser::_parseMarker() return SLANG_OK; } -SlangResult Parser::_maybeParseType(UnownedStringSlice& outType) +SlangResult Parser::_maybeParseType(UnownedStringSlice& outType, Token& outName) { auto startCursor = m_reader.getCursor(); Index templateDepth = 0; - SlangResult res = _maybeParseType(templateDepth); + + TokenReader::ParsingCursor nameCursor; + + SlangResult res = _maybeParseType(templateDepth, nameCursor); if (SLANG_FAILED(res) && m_sink->getErrorCount()) { return res; @@ -968,8 +1181,23 @@ SlangResult Parser::_maybeParseType(UnownedStringSlice& outType) return SLANG_FAIL; } - // We can build up the out type, from the tokens we found - outType = _concatTokens(startCursor); + if (nameCursor.isValid()) + { + const auto cursor = m_reader.getCursor(); + m_reader.setCursor(nameCursor); + outName = m_reader.peekToken(); + m_reader.setCursor(cursor); + + // Extract the contents + List<Token> toks; + _getTypeTokens(startCursor, nameCursor, toks); + outType = _concatTokens(toks.getBuffer(), toks.getCount()); + } + else + { + // We can build up the out type, from the tokens we found + outType = _concatTokens(startCursor); + } return SLANG_OK; } @@ -1195,25 +1423,27 @@ SlangResult Parser::_parseTypeDef() // Consume the typedef SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier)); + Token nameToken; // Parse the type List<Token> toks; - SLANG_RETURN_ON_FAIL(_maybeParseType(toks)); - - Token name; + SLANG_RETURN_ON_FAIL(_maybeParseType(toks, nameToken)); - // Get the name - SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &name)); + // Followed by the name + if (nameToken.type != TokenType::Identifier) + { + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &nameToken)); + } - if (Node::lookupNameInScope(m_currentScope, name.getContent())) + if (Node::lookupNameInScope(m_currentScope, nameToken.getContent())) { - m_sink->diagnose(name.loc, CPPDiagnostics::identifierAlreadyDefined, name.getContent()); + m_sink->diagnose(nameToken.loc, CPPDiagnostics::identifierAlreadyDefined, nameToken.getContent()); return SLANG_FAIL; } SLANG_RETURN_ON_FAIL(expect(TokenType::Semicolon)); RefPtr<TypeDefNode> node = new TypeDefNode; - node->m_name = name; + node->m_name = nameToken; node->m_reflectionType = m_currentScope->getContainedReflectionType(); // Set what aliases too @@ -1224,10 +1454,31 @@ SlangResult Parser::_parseTypeDef() return SLANG_OK; } + +bool Parser::_isCtor() +{ + bool isCtor = false; + // It's a constructor + if (m_currentScope->isClassLike() && m_reader.peekTokenType() == TokenType::Identifier && + m_reader.peekToken().getContent() == m_currentScope->m_name.getContent()) + { + // We need to check it's followed immediately by ( to be sure it's a ctor + + auto cursor = m_reader.getCursor(); + m_reader.advanceToken(); + isCtor = (m_reader.peekTokenType() == TokenType::LParent); + m_reader.setCursor(cursor); + } + + return isCtor; +} + SlangResult Parser::_maybeParseContained(Node** outNode) { *outNode = nullptr; + _maybeConsume(IdentifierStyle::CallableMisc); + bool isStatic = false; bool isVirtual = false; @@ -1254,31 +1505,67 @@ SlangResult Parser::_maybeParseContained(Node** outNode) break; } + _maybeConsume(IdentifierStyle::CallableMisc); + UnownedStringSlice typeName; - if (SLANG_FAILED(_maybeParseType(typeName))) + Token nameToken; + + bool isConstructor = false; + + if (m_currentScope->isClassLike()) { - if (m_sink->getErrorCount()) + // If it's a dtor + if (advanceIfToken(TokenType::OpBitNot, &nameToken)) { - return SLANG_FAIL; - } + // Dtor + // For Dtor we don't hold the full name just the ~ + Token tok; + SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &tok)); - _consumeToSync(); - return SLANG_OK; + if (tok.getContent() != m_currentScope->m_name.getContent()) + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::destructorNameDoesntMatch, m_currentScope->m_name.getContent()); + return SLANG_FAIL; + } + } + else if (_isCtor()) + { + nameToken = m_reader.advanceToken(); + isConstructor = true; + } } - // Has a calling convention (must be a function/method) - Token callingConventionToken; - advanceIfStyle(IdentifierStyle::CallingConvention, &callingConventionToken); - - // Expecting a name - Token nameToken; - if (!advanceIfToken(TokenType::Identifier, &nameToken)) + // If don't have a name it's not a dtor or ctor, so see if it's a type + if (nameToken.type == TokenType::Unknown) { - _consumeToSync(); - return SLANG_OK; + if (SLANG_FAILED(_maybeParseType(typeName, nameToken))) + { + if (m_sink->getErrorCount()) + { + return SLANG_FAIL; + } + + _consumeToSync(); + return SLANG_OK; + } } + if (nameToken.type == TokenType::Unknown) + { + // Has a calling convention (must be a function/method) + Token callingConventionToken; + advanceIfStyle(IdentifierStyle::CallingConvention, &callingConventionToken); + + // Expecting a name + if (!advanceIfToken(TokenType::Identifier, &nameToken)) + { + _consumeToSync(); + return SLANG_OK; + } + } + // Handles other scenarios, but here for catching operator overloading + if (nameToken.type == TokenType::Identifier) { const auto style = m_nodeTree->m_identifierLookup->get(nameToken.getContent()); if (style != IdentifierStyle::None) @@ -1308,8 +1595,9 @@ SlangResult Parser::_maybeParseContained(Node** outNode) { while (true) { + Token paramName; UnownedStringSlice type; - SlangResult res = _maybeParseType(type); + SlangResult res = _maybeParseType(type, paramName); if (SLANG_FAILED(res)) { @@ -1317,40 +1605,39 @@ SlangResult Parser::_maybeParseContained(Node** outNode) return res; } - Token paramName; - if (m_reader.peekTokenType() == TokenType::Identifier) + if (paramName.type != TokenType::Identifier) { - paramName = m_reader.advanceToken(); + if (m_reader.peekTokenType() == TokenType::Identifier) + { + paramName = m_reader.advanceToken(); + } } - // Check if we have a default value - auto peekType = m_reader.peekTokenType(); - - if (peekType == TokenType::OpAssign) + // If we have a name check for default value + if (paramName.type == TokenType::Identifier && advanceIfToken(TokenType::OpAssign)) { - m_reader.advanceToken(); - + // Check if we have a default value List<Token> exprTokens; SLANG_RETURN_ON_FAIL(_parseExpression(exprTokens)); - - // Update the peek type - peekType = m_reader.peekTokenType(); } CallableNode::Param param; - param.m_name = nameToken; + param.m_name = paramName; param.m_type = type; params.add(param); - if (peekType == TokenType::RParent) - { - break; - } - if (peekType == TokenType::Comma) { - m_reader.advanceToken(); - continue; + const auto peekType = m_reader.peekTokenType(); + if (peekType == TokenType::RParent) + { + break; + } + if (peekType == TokenType::Comma) + { + m_reader.advanceToken(); + continue; + } } m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::expectingToken, ", or ) or ="); @@ -1364,13 +1651,32 @@ SlangResult Parser::_maybeParseContained(Node** outNode) // Parse suffix bool isPure = false; - // const? - _consumeTypeModifiers(); + // const? + _maybeConsume(IdentifierStyle::TypeModifier); - // = 0 ? - if (m_reader.peekTokenType() == TokenType::OpAssign) + if (isConstructor) + { + // Initializer list + if (advanceIfToken(TokenType::Colon)) + { + while (true) + { + auto peekType = m_reader.peekTokenType(); + if (peekType == TokenType::Semicolon || + peekType == TokenType::LBrace || + peekType == TokenType::EndOfFile) + { + break; + } + // Consume + m_reader.advanceToken(); + } + } + } + + // = 0 ? or = default + if (advanceIfToken(TokenType::OpAssign)) { - m_reader.advanceToken(); if (m_reader.peekTokenType() == TokenType::IntegerLiteral) { Int value = -1; @@ -1386,6 +1692,14 @@ SlangResult Parser::_maybeParseContained(Node** outNode) return SLANG_FAIL; } } + else if (advanceIfStyle(IdentifierStyle::Default)) + { + } + else + { + m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::cannotParseCallable); + return SLANG_FAIL; + } } if (m_reader.peekTokenType() == TokenType::Semicolon) @@ -1442,41 +1756,19 @@ SlangResult Parser::_maybeParseContained(Node** outNode) else { // Looks like variable - if (!m_currentScope->canContainFields()) + if (!m_currentScope->canContainFields() || nameToken.type != TokenType::Identifier) { _consumeToSync(); return SLANG_OK; } - if (m_reader.peekTokenType() == TokenType::LBracket) - { - auto startCursor = m_reader.getCursor(); - - // If it's not balanced we just assume it's not correct - and ignore - if (SLANG_FAILED(_parseBalanced(nullptr))) - { - _consumeToSync(); - return SLANG_OK; - } - - UnownedStringSlice arraySuffix = _concatTokens(startCursor); - - // The overall type is the typename concated with the arraySuffix - StringBuilder buf; - buf << typeName << arraySuffix; - - StringSlicePool* typePool = m_nodeTree->m_typePool; - - typeName = typePool->getSlice(typePool->add(buf)); - } - // Check if has a default value if (advanceIfToken(TokenType::OpAssign)) { List<Token> exprTokens; SLANG_RETURN_ON_FAIL(_parseExpression(exprTokens)); } - + // Hit end of field/variable if (m_reader.peekTokenType() == TokenType::Semicolon) { @@ -1636,7 +1928,16 @@ SlangResult Parser::parse(SourceOrigin* sourceOrigin, const Options* options) Lexer lexer; + // Set up the scope stack + m_scopeStack.clear(); + m_currentScope = m_nodeTree->m_rootNode; + m_scopeStack.add(m_currentScope); + + if (!options->m_requireMark) + { + m_currentScope->m_reflectionOverride = ReflectionType::Reflected; + } lexer.initialize(sourceView, m_sink, m_nodeTree->m_namePool, manager->getMemoryArena()); m_tokenList = lexer.lexAllSemanticTokens(); @@ -1652,12 +1953,41 @@ SlangResult Parser::parse(SourceOrigin* sourceOrigin, const Options* options) { switch (m_reader.peekTokenType()) { + case TokenType::OpBitNot: + { + // Handle dtor + if (m_currentScope->isClassLike()) + { + Node* containedNode = nullptr; + SLANG_RETURN_ON_FAIL(_maybeParseContained(&containedNode)); + } + else + { + // consume + m_reader.advanceToken(); + } + break; + } case TokenType::Identifier: { const IdentifierStyle style = m_nodeTree->m_identifierLookup->get(m_reader.peekToken().getContent()); switch (style) { + case IdentifierStyle::Extern: + { + m_reader.advanceToken(); + + Token externType; + SLANG_RETURN_ON_FAIL(expect(TokenType::StringLiteral, &externType)); + + if (advanceIfToken(TokenType::LBrace)) + { + // Push a 'special' scope (which is basically transparent) + pushScope(nullptr); + } + break; + } case IdentifierStyle::Template: { SLANG_RETURN_ON_FAIL(_consumeTemplate()); @@ -1730,8 +2060,10 @@ SlangResult Parser::parse(SourceOrigin* sourceOrigin, const Options* options) } else { + UnownedStringSlice content = m_reader.peekToken().getContent(); + // If it's a marker handle it - if (_isMarker(m_reader.peekToken().getContent())) + if (_isMarker(content)) { if (!m_currentScope->isClassLike()) { @@ -1743,6 +2075,13 @@ SlangResult Parser::parse(SourceOrigin* sourceOrigin, const Options* options) break; } + if (m_options->m_markPrefix.getLength() > 0 && content.startsWith(m_options->m_markPrefix.getUnownedSlice())) + { + SLANG_RETURN_ON_FAIL(_parseSpecialMacro()); + break; + } + + // Special case the node that's the root of the hierarchy (as far as reflection is concerned) // This could be a field if (m_currentScope->canContainFields() || m_currentScope->canContainCallable()) diff --git a/tools/slang-cpp-extractor/parser.h b/tools/slang-cpp-extractor/parser.h index 4815821c5..cf0147e8b 100644 --- a/tools/slang-cpp-extractor/parser.h +++ b/tools/slang-cpp-extractor/parser.h @@ -34,10 +34,6 @@ public: void setKindEnabled(Node::Kind kind, bool isEnabled = true); bool isTypeEnabled(Node::Kind kind) { return (m_nodeTypeEnabled & (NodeTypeBitType(1) << int(kind))) != 0; } - /// If set classes require a 'marker' to be reflected - void setRequireMarker(bool requireMarker) { m_requireMarker = requireMarker; } - bool getRequireMarker() const { return m_requireMarker; } - void setKindsEnabled(const Node::Kind* kinds, Index kindsCount, bool isEnabled = true); Parser(NodeTree* nodeTree, DiagnosticSink* sink); @@ -58,26 +54,33 @@ protected: SlangResult _parseTypeDef(); SlangResult _parseEnum(); SlangResult _parseMarker(); + SlangResult _parseSpecialMacro(); - SlangResult _maybeParseType(List<Token>& outToks); - SlangResult _maybeParseType(UnownedStringSlice& outType); + SlangResult _maybeParseType(List<Token>& outToks, Token& outName); + SlangResult _maybeParseType(UnownedStringSlice& outType, Token& outName); + SlangResult _maybeParseType(Index& ioTemplateDepth, TokenReader::ParsingCursor& outCursor); SlangResult _parseExpression(List<Token>& outExprTokens); - - SlangResult _maybeParseType(Index& ioTemplateDepth); + SlangResult _maybeParseTemplateArgs(Index& ioTemplateDepth); SlangResult _maybeParseTemplateArg(Index& ioTemplateDepth); /// Parse balanced - if a sink is set will report to that sink SlangResult _parseBalanced(DiagnosticSink* sink); + bool _isCtor(); + /// Concatenate all tokens from start to the current position UnownedStringSlice _concatTokens(TokenReader::ParsingCursor start); + UnownedStringSlice _concatTokens(const Token* toks, Index toksCount); + + UnownedStringSlice _concatType(TokenReader::ParsingCursor start, TokenReader::ParsingCursor nameCursor); + + void _getTypeTokens(TokenReader::ParsingCursor start, TokenReader::ParsingCursor nameCursor, List<Token>& outToks); /// Consume what looks like a template definition SlangResult _consumeTemplate(); - - void _consumeTypeModifiers(); + SlangResult _maybeConsume(IdentifierStyle style); SlangResult _consumeToSync(); /// Consumes balanced parens. Will return an error if not matched. Assumes starts on opening ( @@ -88,6 +91,8 @@ protected: TokenList m_tokenList; TokenReader m_reader; + List<ScopeNode*> m_scopeStack; + ScopeNode* m_currentScope; ///< The current scope being processed SourceOrigin* m_sourceOrigin; ///< The source origin that all tokens are in @@ -95,8 +100,6 @@ protected: NodeTree* m_nodeTree; ///< Shared state between parses. Nodes will be added to this - bool m_requireMarker = true; - const Options* m_options; }; diff --git a/tools/slang-cpp-extractor/unit-test.cpp b/tools/slang-cpp-extractor/unit-test.cpp index c239a2a2c..16b4c10dc 100644 --- a/tools/slang-cpp-extractor/unit-test.cpp +++ b/tools/slang-cpp-extractor/unit-test.cpp @@ -29,6 +29,9 @@ struct TestState m_sink.init(&m_sourceManager, Lexer::sourceLocationLexer); m_namePool.setRootNamePool(&m_rootNamePool); + + // We don't require marker + m_options.m_requireMark = false; } RootNamePool m_rootNamePool; @@ -41,9 +44,6 @@ struct TestState }; static const char someSource[] = -"#define SLANG_REFLECTED\n" -"\n" -"SLANG_REFLECTED\n" "class ISomeInterface\n" "{\n" " public:\n" @@ -51,15 +51,30 @@ static const char someSource[] = " virtual float SLANG_MCALL anotherMethod(float a) = 0;\n" "};\n" "\n" +"struct SomeStruct\n" +"{\n" +" SomeStruct() = default;\n" +" SomeStruct(float v = 0.0f):b(v) {}\n" +" ~SomeStruct() {}\n" +" int a = 10; \n" +" float b; \n" +" int another[10];\n" +" const char* yetAnother = nullptr;\n" +"};\n" +"\n" "enum SomeEnum\n" "{\n" " Value,\n" " Another = 10,\n" "};\n" +"\n" +"typedef int (*SomeFunc)(int a);\n" +"\n" "typedef SomeEnum AliasEnum;\n" "void someFunc(int a, float b) { }\n" "namespace Blah {\n" "int add(int a, int b) { return a + b; }\n" +"unsigned add(unsigned a, unsigned b) { return a + b; }\n" "}\n"; @@ -79,16 +94,24 @@ static const char someSource[] = SourceOrigin* sourceOrigin = tree.addSourceOrigin(sourceFile, state.m_options); Parser parser(&tree, &state.m_sink); - // We don't require markers to reflect - parser.setRequireMarker(false); + { const Node::Kind enableKinds[] = { Node::Kind::Enum, Node::Kind::EnumClass, Node::Kind::EnumCase, Node::Kind::TypeDef }; parser.setKindsEnabled(enableKinds, SLANG_COUNT_OF(enableKinds)); } - SLANG_RETURN_ON_FAIL(parser.parse(sourceOrigin, &state.m_options)); + SlangResult res = parser.parse(sourceOrigin, &state.m_options); + + if (state.m_sink.outputBuffer.getLength()) + { + printf("%s\n", state.m_sink.outputBuffer.getBuffer()); + } + if (SLANG_FAILED(res)) + { + return res; + } { StringBuilder buf; |