Use 'balance' for extracting array suffix in C++ extractor (#1346)

* Add support for parsing array types to C++ extractor.

* C++ extractor looks for 'balanced tokens'. Use for extracting array suffixes.

2 files changed, 84 insertions, 32 deletions

diff --git a/tools/slang-cpp-extractor/slang-cpp-extractor-diagnostic-defs.h b/tools/slang-cpp-extractor/slang-cpp-extractor-diagnostic-defs.h
index 715f74c23..7ed2f0691 100644
--- a/tools/slang-cpp-extractor/slang-cpp-extractor-diagnostic-defs.h
+++ b/tools/slang-cpp-extractor/slang-cpp-extractor-diagnostic-defs.h
@@ -19,7 +19,7 @@
 #endif
 
 DIAGNOSTIC(-1, Note, seeDeclarationOf, "see declaration of '$0'")
-DIAGNOSTIC(-1, Note, seeOpenBrace, "see open brace")
+DIAGNOSTIC(-1, Note, seeOpen, "see open $0")
 
 DIAGNOSTIC(1, Error, cannotOpenFile, "cannot open file '$0'.")
 
@@ -33,6 +33,8 @@ DIAGNOSTIC(100005, Error, braceOpenAtEndOfFile, "Brace open at file end")
 DIAGNOSTIC(100006, Error, unexpectedTemplateClose, "Unexpected template close")
 DIAGNOSTIC(100007, Error, superTypeNotFound, "Super type not found for $0")
 DIAGNOSTIC(100008, Error, superTypeNotAType, "Named super type is not a type $0")
+DIAGNOSTIC(100009, Error, unexpectedUnbalancedToken, "Unexpected unbalanced token")
+DIAGNOSTIC(100010, Error, unexpectedEndOfFile, "Unexpected end of file")
 
 // Command line errors 100100
 
diff --git a/tools/slang-cpp-extractor/slang-cpp-extractor-main.cpp b/tools/slang-cpp-extractor/slang-cpp-extractor-main.cpp
index e18601076..814e7ac0e 100644
--- a/tools/slang-cpp-extractor/slang-cpp-extractor-main.cpp
+++ b/tools/slang-cpp-extractor/slang-cpp-extractor-main.cpp
@@ -332,7 +332,8 @@ protected:
     SlangResult _maybeParseTemplateArgs(Index& ioTemplateDepth);
     SlangResult _maybeParseTemplateArg(Index& ioTemplateDepth);
 
-    SlangResult _maybeParseArrayTypeSuffix(UnownedStringSlice& outSuffix);
+        /// Parse balanced - if a sink is set will report to that sink
+    SlangResult _parseBalanced(DiagnosticSink* sink);
 
     SlangResult _calcDerivedTypesRec(Node* node);
     static String _calcMacroOrigin(const String& filePath, const Options& options);
@@ -950,7 +951,7 @@ SlangResult CPPExtractor::consumeToClosingBrace(const Token* inOpenBraceToken)
             case TokenType::EndOfFile:
             {
                 m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::didntFindMatchingBrace);
-                m_sink->diagnose(openToken, CPPDiagnostics::seeOpenBrace);
+                m_sink->diagnose(openToken, CPPDiagnostics::seeOpen);
                 return SLANG_FAIL;
             }
             case TokenType::LBrace:
@@ -1336,46 +1337,91 @@ SlangResult CPPExtractor::_maybeParseType(UnownedStringSlice& outType)
     return SLANG_OK;
 }
 
-SlangResult CPPExtractor::_maybeParseArrayTypeSuffix(UnownedStringSlice& outSuffix)
+static bool _isBalancedOpen(TokenType tokenType)
 {
-    auto startCursor = m_reader.getCursor();
+    return tokenType == TokenType::LBrace ||
+        tokenType == TokenType::LParent ||
+        tokenType == TokenType::LBracket;
+}
+
+static bool _isBalancedClose(TokenType tokenType)
+{
+    return tokenType == TokenType::RBrace ||
+        tokenType == TokenType::RParent ||
+        tokenType == TokenType::RBracket;
+}
+
+static TokenType _getBalancedClose(TokenType tokenType)
+{
+    SLANG_ASSERT(_isBalancedOpen(tokenType));
+    switch (tokenType)
+    {
+        case TokenType::LBrace:         return TokenType::RBrace;
+        case TokenType::LParent:        return TokenType::RParent;
+        case TokenType::LBracket:       return TokenType::RBracket;
+        default:                        return TokenType::Unknown;
+    }
+}
+
+SlangResult CPPExtractor::_parseBalanced(DiagnosticSink* sink)
+{
+    const TokenType openTokenType = m_reader.peekTokenType();
+    if (!_isBalancedOpen(openTokenType))
+    {
+        return SLANG_FAIL;
+    }
+
+    // Save the start token
+    const Token startToken = m_reader.advanceToken();
+    // Get the token type that would close the open
+    const TokenType closeTokenType = _getBalancedClose(openTokenType);
 
     while (true)
     {
-        SLANG_RETURN_ON_FAIL(expect(TokenType::LBracket));
+        const TokenType tokenType = m_reader.peekTokenType();
 
-        while (true)
+        // If we hit the closing token, we are done
+        if (tokenType == closeTokenType)
         {
-            TokenType tokenType = m_reader.peekTokenType();
-            if (tokenType == TokenType::RBracket)
-            {
-                break;
-            }
+            m_reader.advanceToken();
+            return SLANG_OK;
+        }
 
-            if (tokenType == TokenType::EndOfFile ||
-                tokenType == TokenType::LBrace ||
-                tokenType == TokenType::RBrace ||
-                tokenType == TokenType::LBracket)
+        // If we hit a balanced open, recurse 
+        if (_isBalancedOpen(tokenType))
+        {
+            SLANG_RETURN_ON_FAIL(_parseBalanced(sink));
+            continue;
+        }
+
+        // If we hit a close token that doesn't match, then the balancing has gone wrong
+        if (_isBalancedClose(tokenType))
+        {
+            // Only diagnose if required
+            if (sink)
             {
-                return SLANG_FAIL;
+                sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::unexpectedUnbalancedToken);
+                sink->diagnose(startToken, CPPDiagnostics::seeOpen);
             }
-
-            // Okay onto next
-            m_reader.advanceToken();
+            return SLANG_FAIL;
         }
 
-        SLANG_RETURN_ON_FAIL(expect(TokenType::RBracket));
-
-        if (m_reader.peekTokenType() == TokenType::RBracket)
+        // If we hit the end of the file and have not hit the closing token, then
+        // somethings gone wrong
+        if (tokenType == TokenType::EndOfFile)
         {
-            continue;
+            if (sink)
+            {
+                sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::unexpectedEndOfFile);
+                sink->diagnose(startToken, CPPDiagnostics::seeOpen);
+            }
+
+            return SLANG_FAIL;
         }
 
-        break;
+        // Skip the token
+        m_reader.advanceToken();
     }
-
-    outSuffix = _concatTokens(startCursor);
-    return SLANG_OK;
 }
 
 SlangResult CPPExtractor::_maybeParseField()
@@ -1404,14 +1450,18 @@ SlangResult CPPExtractor::_maybeParseField()
 
     if (m_reader.peekTokenType() == TokenType::LBracket)
     {
-        UnownedStringSlice arraySuffix;
+        auto startCursor = m_reader.getCursor();
 
-        if (SLANG_FAILED(_maybeParseArrayTypeSuffix(arraySuffix)))
+        // If it's not balanced we just assume it's not correct - and ignore
+        if (SLANG_FAILED(_parseBalanced(nullptr)))
         {
             _consumeToSync();
-            return m_sink->errorCount ? SLANG_FAIL : SLANG_OK;
+            return SLANG_OK;
         }
-        // The overall type is the typename concatted with the arraySuffix
+
+        UnownedStringSlice arraySuffix = _concatTokens(startCursor);
+
+        // The overall type is the typename concated with the arraySuffix
         StringBuilder buf;
         buf << typeName << arraySuffix;