C++ extractor improvements (#1803)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Split of NodeTree.
Split out FileUtil.
Split out MacroWriter.

* Rename slang-cpp-extractor-main.cpp -> cpp-extractor-main.cpp

* First pass at extractor unit-tests

* Initial parsing of enum.

* Ability to disable/enable parsing of scope types.

* Initial support for typedef.

* Added operator== != to ArrayVIew.
Added test for splitting to unit tests.

* Improve comment in StringUtil.

* Fix comment.

* Fix typo.

1 files changed, 16 insertions, 88 deletions

diff --git a/tools/slang-cpp-extractor/parser.h b/tools/slang-cpp-extractor/parser.h
index 58a13c19f..f0febb728 100644
--- a/tools/slang-cpp-extractor/parser.h
+++ b/tools/slang-cpp-extractor/parser.h
@@ -4,103 +4,19 @@
 #include "diagnostics.h"
 #include "node.h"
 #include "identifier-lookup.h"
+#include "node-tree.h"
 
 #include "../../source/compiler-core/slang-lexer.h"
 
 namespace CppExtract {
 using namespace Slang;
 
-class TypeSet : public RefObject
-{
-public:
-    /// This is the looked up name.
-    UnownedStringSlice m_macroName;    ///< The name extracted from the macro SLANG_ABSTRACT_AST_CLASS -> AST
-
-    String m_typeName;                  ///< The enum type name associated with this type for AST it is ASTNode
-    String m_fileMark;                  ///< This 'mark' becomes of the output filename
-
-    List<ClassLikeNode*> m_baseTypes;   ///< The base types for this type set
-};
-
-class SourceOrigin : public RefObject
-{
-public:
-
-    void addNode(Node* node)
-    {
-        if (auto classLike = as<ClassLikeNode>(node))
-        {
-            SLANG_ASSERT(classLike->m_origin == nullptr);
-            classLike->m_origin = this;
-        }
-
-        m_nodes.add(node);
-    }
-
-    SourceOrigin(SourceFile* sourceFile, const String& macroOrigin) :
-        m_sourceFile(sourceFile),
-        m_macroOrigin(macroOrigin)
-    {}
-
-    String m_macroOrigin;               ///< The macro text is inserted into the macro to identify the origin. It is based on the filename
-    SourceFile* m_sourceFile;           ///< The source file - also holds the path information
-
-    /// All of the nodes defined in this file in the order they were defined
-    /// Note that the same namespace may be listed multiple times.
-    List<RefPtr<Node> > m_nodes;
-};
-
-struct Options;
-class IdentifierLookup;
-
-/* NodeTree holds nodes that have been parsed into a tree rooted on the 'rootNode'.
-Also contains other state associated with or useful to a node tree */
-class NodeTree
-{
-public:
-    friend class Parser;
-        /// Get all of the parsed source origins
-    const List<RefPtr<SourceOrigin> >& getSourceOrigins() const { return m_sourceOrigins; }
-
-    TypeSet* getTypeSet(const UnownedStringSlice& slice);
-    TypeSet* getOrAddTypeSet(const UnownedStringSlice& slice);
-
-    SourceOrigin* addSourceOrigin(SourceFile* sourceFile, const Options& options);
-
-        /// Get all of the type sets
-    const List<RefPtr<TypeSet>>& getTypeSets() const { return m_typeSets; }
-
-        /// Get the root node
-    Node* getRootNode() const { return m_rootNode; }
-
-        /// When parsing we don't lookup all up super types/add derived types. This is because
-        /// we allow files to be processed in any order, so we have to do the type lookup as a separate operation
-    SlangResult calcDerivedTypes(DiagnosticSink* sink);
-
-    NodeTree(StringSlicePool* typePool, NamePool* namePool, IdentifierLookup* identifierLookup);
-
-    static String calcMacroOrigin(const String& filePath, const Options& options);
-
-protected:
-    SlangResult _calcDerivedTypesRec(ScopeNode* node, DiagnosticSink* sink);
-
-    StringSlicePool m_typeSetPool;              ///< Pool for type set names
-    List<RefPtr<TypeSet> > m_typeSets;          ///< The type sets
-
-    IdentifierLookup* m_identifierLookup;
-    StringSlicePool* m_typePool;                ///< Pool for just types
-
-    NamePool* m_namePool;
-
-    RefPtr<ScopeNode> m_rootNode;   ///< The root scope 
-
-    List<RefPtr<SourceOrigin>> m_sourceOrigins;
-};
-
 class Parser
 {
 public:
 
+    typedef uint32_t NodeTypeBitType;
+
     SlangResult expect(TokenType type, Token* outToken = nullptr);
 
     bool advanceIfMarker(Token* outToken = nullptr);
@@ -115,6 +31,10 @@ public:
         /// Parse the contents of the source file
     SlangResult parse(SourceOrigin* sourceOrigin, const Options* options);
 
+    void setTypeEnabled(Node::Type type, bool isEnabled = true);
+    bool isTypeEnabled(Node::Type type) { return (m_nodeTypeEnabled & (NodeTypeBitType(1) << int(type))) != 0; }
+    void setTypesEnabled(const Node::Type* types, Index typesCount, bool isEnabled = true);
+
     Parser(NodeTree* nodeTree, DiagnosticSink* sink);
 
 protected:
@@ -122,15 +42,21 @@ protected:
 
     bool _isMarker(const UnownedStringSlice& name);
 
+    SlangResult _maybeConsumeScope();
+
     SlangResult _parsePreDeclare();
     SlangResult _parseTypeSet();
 
     SlangResult _maybeParseNode(Node::Type type);
     SlangResult _maybeParseField();
 
+    SlangResult _parseTypeDef();
+    SlangResult _parseEnum();
+
+    SlangResult _maybeParseType(List<Token>& outToks);
     SlangResult _maybeParseType(UnownedStringSlice& outType);
 
-    SlangResult _maybeParseType(UnownedStringSlice& outType, Index& ioTemplateDepth);
+    SlangResult _maybeParseType(Index& ioTemplateDepth);
     SlangResult _maybeParseTemplateArgs(Index& ioTemplateDepth);
     SlangResult _maybeParseTemplateArg(Index& ioTemplateDepth);
 
@@ -144,6 +70,8 @@ protected:
 
     SlangResult _consumeToSync();
 
+    NodeTypeBitType m_nodeTypeEnabled;
+
     TokenList m_tokenList;
     TokenReader m_reader;