AST dumping via C++ Extractor reflection (#1348)

* Add support for parsing array types to C++ extractor.

* C++ extractor looks for 'balanced tokens'. Use for extracting array suffixes.

* First pass at field dumping.

* Update project for field dumping.

* WIP AST Dumper.

* More AST dump compiling.

* Fix bug in StringSlicePool where it doesn't use the copy of the UnownedStringSlice in the map.

* Add support for SLANG_RELFECTED and SLANG_UNREFLECTED
More AST dump support.

* Support for hierarchical dumping/flat dumping.
Use SourceWriter to dump.

* Add -dump-ast command line option.

* Add fixes to VS project to incude AST dump.

* Fix compilation on gcc.

* Add fix for type ambiguity issue on x86 VS.

* Fixes from merge of reducing Token size.

* Fix comment about using SourceWriter.

1 files changed, 202 insertions, 115 deletions

diff --git a/tools/slang-cpp-extractor/slang-cpp-extractor-main.cpp b/tools/slang-cpp-extractor/slang-cpp-extractor-main.cpp
index 5118a3a9b..792cf3458 100644
--- a/tools/slang-cpp-extractor/slang-cpp-extractor-main.cpp
+++ b/tools/slang-cpp-extractor/slang-cpp-extractor-main.cpp
@@ -42,6 +42,9 @@ enum class IdentifierStyle
     Namespace,          ///< namespace
     Access,             ///< public, protected, private
 
+    Reflected,
+    Unreflected,
+
     CountOf,
 };
 
@@ -53,6 +56,7 @@ struct IdentifierFlag
         StartScope  = 0x1,          ///< namespace, struct or class
         ClassLike   = 0x2,          ///< Struct or class
         Keyword     = 0x4,
+        Reflection  = 0x8,
     };
 };
 
@@ -67,7 +71,10 @@ static const IdentifierFlags kIdentifierFlags[Index(IdentifierStyle::CountOf)] =
     IdentifierFlag::Keyword | IdentifierFlag::StartScope | IdentifierFlag::ClassLike, /// Class
     IdentifierFlag::Keyword | IdentifierFlag::StartScope | IdentifierFlag::ClassLike, /// Struct
     IdentifierFlag::Keyword | IdentifierFlag::StartScope, /// Namespace
-    IdentifierFlag::Keyword,
+    IdentifierFlag::Keyword,                              /// Access
+    IdentifierFlag::Reflection,                           /// Reflected
+    IdentifierFlag::Reflection,                           /// Unreflected
+
 };
 
 SLANG_FORCE_INLINE IdentifierFlags getFlags(IdentifierStyle style)
@@ -118,6 +125,12 @@ public:
             set(UnownedStringSlice(names[i]), style);
         }
     }
+    void reset()
+    {
+        m_styles.clear();
+        m_pool.clear();
+    }
+
     IdentifierLookup():
         m_pool(StringSlicePool::Style::Empty)
     {
@@ -128,6 +141,12 @@ protected:
     StringSlicePool m_pool;
 };
 
+enum class ReflectionType
+{
+    NotReflected,
+    Reflected,
+};
+
 class SourceOrigin;
 
 class Node : public RefObject
@@ -144,8 +163,11 @@ public:
 
     struct Field
     {
+        bool isReflected() const { return reflectionType == ReflectionType::Reflected; }
+
         UnownedStringSlice type;
         Token name;
+        ReflectionType reflectionType;
     };
 
     enum class BaseType
@@ -193,6 +215,12 @@ public:
         /// Find the last (reflected) derived type
     Node* findLastDerived();
 
+        /// True if reflected
+    bool isReflected() const { return m_reflectionType == ReflectionType::Reflected; }
+
+        /// Gets the reflection for any contained types
+    ReflectionType getContainedReflectionType() const { return m_reflectionType == ReflectionType::NotReflected ? ReflectionType::NotReflected : m_reflectionOverride; }
+
         /// True if has a derived type that is reflected
     bool hasReflectedDerivedType() const;
         /// Stores in out any reflected derived types
@@ -205,7 +233,8 @@ public:
     Node(Type type):
         m_type(type),
         m_parentScope(nullptr),
-        m_isReflected(false),
+        m_reflectionType(ReflectionType::NotReflected),
+        m_reflectionOverride(ReflectionType::Reflected),
         m_superNode(nullptr),
         m_baseType(BaseType::None),
         m_origin(nullptr)
@@ -235,7 +264,11 @@ public:
     SourceOrigin* m_origin;
 
         /// Classes can be traversed, but not reflected. To be reflected they have to contain the marker
-    bool m_isReflected;
+    ReflectionType m_reflectionType;
+
+        /// For child types, fields, how reflection is handled. If this type is not reflected
+    ReflectionType m_reflectionOverride;    
+
         /// The base type of this
     BaseType m_baseType;
 
@@ -393,11 +426,9 @@ void Node::addChild(Node* child)
     child->m_parentScope = this;
     m_children.add(child);
 
-    UnownedStringSlice content = child->m_name.getContent();
-
-    if (content.getLength())
+    if (child->m_name.hasContent())
     {
-        m_childMap.Add(content, child);
+        m_childMap.Add(child->m_name.getContent(), child);
     }
 }
 
@@ -443,7 +474,7 @@ static void _indent(Index indentCount, StringBuilder& out)
 
 void Node::dumpDerived(int indentCount, StringBuilder& out)
 {
-    if (isClassLike() && m_isReflected && m_name.hasContent())
+    if (isClassLike() && isReflected() && m_name.hasContent())
     {
         _indent(indentCount, out);
         out << m_name.getContent() << "\n";
@@ -484,12 +515,12 @@ void Node::dump(int indentCount, StringBuilder& out)
             
             out << typeName << " ";
 
-            if (!m_isReflected)
+            if (!isReflected())
             {
                 out << " (";
             }
             out << m_name.getContent();
-            if (!m_isReflected)
+            if (!isReflected())
             {
                 out << ") ";
             }
@@ -511,8 +542,11 @@ void Node::dump(int indentCount, StringBuilder& out)
 
     for (const Field& field : m_fields)
     {
-        _indent(indentCount + 1, out);
-        out << field.type << " " << field.name.getContent() << "\n";
+        if (field.isReflected())
+        {
+            _indent(indentCount + 1, out);
+            out << field.type << " " << field.name.getContent() << "\n";
+        }
     }
 
     _indent(indentCount, out);
@@ -559,7 +593,7 @@ Index Node::calcDerivedDepth() const
 
 Node* Node::findLastDerived()
 {
-    if (!m_isReflected)
+    if (!isReflected())
     {
         return nullptr;
     }
@@ -594,7 +628,7 @@ bool Node::hasReflectedDerivedType() const
 {
     for (Node* type : m_derivedTypes)
     {
-        if (type->m_isReflected)
+        if (type->isReflected())
         {
             return true;
         }
@@ -607,7 +641,7 @@ void Node::getReflectedDerivedTypes(List<Node*>& out) const
     out.clear();
     for (Node* type : m_derivedTypes)
     {
-        if (type->m_isReflected)
+        if (type->isReflected())
         {
             out.add(type);
         }
@@ -621,7 +655,7 @@ void Node::getReflectedDerivedTypes(List<Node*>& out) const
     for (Index j = 0; j < count; )
     {
         Node* node = ioNodes[j];
-        if (!node->isClassLike() || !node->m_isReflected)
+        if (!node->isClassLike() || !node->isReflected())
         {
             ioNodes.removeAt(j);
             count--;
@@ -651,6 +685,8 @@ struct Options
     bool m_defs = false;            ///< If set will output a '-defs.h' file for each of the input files, that corresponds to previous defs files (although doesn't have fields/RAW)
     bool m_dump = false;            ///< If true will dump to stderr the types/fields and hierarchy it extracted
 
+    bool m_outputFields = false;     ///< When dumping macros also dump field definitions
+
     List<String> m_inputPaths;      ///< The input paths to the files to be processed
         
     String m_outputPath;            ///< The ouput path. Note that the extractor can generate multiple output files, and this will actually be the 'stem' of several files
@@ -766,7 +802,12 @@ SlangResult OptionsParser::parse(int argc, const char*const* argv, DiagnosticSin
             else if (arg == "-defs")
             {
                 outOptions.m_defs = true;
-                continue;
+                continue; 
+            }
+            else if (arg == "-output-fields")
+            {
+                outOptions.m_outputFields = true;
+                break;
             }
             else if (arg == "-strip-prefix")
             {
@@ -809,6 +850,7 @@ CPPExtractor::CPPExtractor(StringSlicePool* typePool, NamePool* namePool, Diagno
     m_identifierLookup(identifierLookup)
 {
     m_rootNode = new Node(Node::Type::Namespace);
+    m_rootNode->m_reflectionType = ReflectionType::Reflected;
 }
 
 bool CPPExtractor::_isMarker(const UnownedStringSlice& name)
@@ -1038,6 +1080,9 @@ SlangResult CPPExtractor::_maybeParseNode(Node::Type type)
     RefPtr<Node> node(new Node(type));
     node->m_name = name;
 
+    // Defaults to not reflected
+    SLANG_ASSERT(!node->isReflected());
+
     if (advanceIfToken(TokenType::Colon))
     {
         // Could have public
@@ -1068,8 +1113,6 @@ SlangResult CPPExtractor::_maybeParseNode(Node::Type type)
             m_reader.advanceToken();
         }
 
-        // Node does define a class, but it's not reflected
-        node->m_isReflected = false;
         return pushNode(node);
     }
 
@@ -1094,7 +1137,6 @@ SlangResult CPPExtractor::_maybeParseNode(Node::Type type)
             case TokenType::Identifier:  break;
             case TokenType::RBrace:
             {
-                node->m_isReflected = false;
                 SLANG_RETURN_ON_FAIL(pushNode(node));
                 SLANG_RETURN_ON_FAIL(popBrace());
                 m_reader.advanceToken();
@@ -1102,7 +1144,6 @@ SlangResult CPPExtractor::_maybeParseNode(Node::Type type)
             }
             default:
             {
-                node->m_isReflected = false;
                 SLANG_RETURN_ON_FAIL(pushNode(node));
                 return SLANG_OK;
             }
@@ -1114,8 +1155,6 @@ SlangResult CPPExtractor::_maybeParseNode(Node::Type type)
             break;
         }
 
-        // Looks like a class, but looks like non-reflected
-        node->m_isReflected = false;
         // We still need to add the node,
         SLANG_RETURN_ON_FAIL(pushNode(node));
         return SLANG_OK;
@@ -1134,7 +1173,7 @@ SlangResult CPPExtractor::_maybeParseNode(Node::Type type)
         return SLANG_FAIL;
     }
     
-    node->m_isReflected = true;
+    node->m_reflectionType = ReflectionType::Reflected;
     return pushNode(node);
 }
 
@@ -1478,6 +1517,7 @@ SlangResult CPPExtractor::_maybeParseField()
             Node::Field field;
             field.type = typeName;
             field.name = fieldName;
+            field.reflectionType = m_currentNode->getContainedReflectionType();
 
             m_currentNode->m_fields.add(field);
             break;
@@ -1538,8 +1578,8 @@ SlangResult CPPExtractor::parse(SourceFile* sourceFile, const Options* options)
         {
             case TokenType::Identifier:
             {
-                IdentifierStyle style = m_identifierLookup->get(m_reader.peekToken().getContent());
-
+                const IdentifierStyle style = m_identifierLookup->get(m_reader.peekToken().getContent());
+                
                 switch (style)
                 {
                     case IdentifierStyle::BaseClass:
@@ -1555,10 +1595,31 @@ SlangResult CPPExtractor::parse(SourceFile* sourceFile, const Options* options)
                         node->m_name = nameToken;
                         node->m_baseType = Node::BaseType::Marked;
 
+                        // Classes defined this way are not reflected, as the mark means the type exists, but isn't visible
+                        node->m_reflectionType = ReflectionType::NotReflected;
+
                         SLANG_RETURN_ON_FAIL(pushNode(node));
                         popBrace();
                         break;
                     }
+                    case IdentifierStyle::Reflected:
+                    {
+                        m_reader.advanceToken();
+                        if (m_currentNode)
+                        {
+                            m_currentNode->m_reflectionOverride = ReflectionType::Reflected;
+                        }
+                        break;
+                    }
+                    case IdentifierStyle::Unreflected:
+                    {
+                        m_reader.advanceToken();
+                        if (m_currentNode)
+                        {
+                            m_currentNode->m_reflectionOverride = ReflectionType::NotReflected;
+                        }
+                        break;
+                    }
                     case IdentifierStyle::Root:
                     {
                         if (m_currentNode && m_currentNode->isClassLike())
@@ -1660,7 +1721,7 @@ SlangResult CPPExtractor::_calcDerivedTypesRec(Node* node)
             Node* superType = parentScope->findChild(node->m_super.getContent());
             if (!superType)
             {
-                if (node->m_isReflected)
+                if (node->isReflected())
                 {
                     m_sink->diagnose(node->m_name, CPPDiagnostics::superTypeNotFound, node->m_name.getContent());
                     return SLANG_FAIL;
@@ -1775,48 +1836,18 @@ public:
         m_slicePool(StringSlicePool::Style::Default)
     {
         m_namePool.setRootNamePool(rootNamePool);
-
-        // Some keywords
-        {
-            const char* names[] = { "virtual", "typedef", "continue", "if", "case", "break", "catch", "default", "delete", "do", "else", "for", "new", "goto", "return", "switch", "throw", "using", "while" };
-            m_identifierLookup.set(names, SLANG_COUNT_OF(names), IdentifierStyle::Keyword);
-        }
-
-        // Type modifier keywords
-        {
-            const char* names[] = { "const", "volatile" };
-            m_identifierLookup.set(names, SLANG_COUNT_OF(names), IdentifierStyle::TypeModifier);
-        }
-
-        // Special markers
-        {
-            m_identifierLookup.set("SLANG_CLASS_ROOT", IdentifierStyle::Root);
-            m_identifierLookup.set("SLANG_REFLECT_BASE_CLASS", IdentifierStyle::BaseClass);
-        }
-
-        // Keywords which introduce types/scopes
-        {
-            m_identifierLookup.set("struct", IdentifierStyle::Struct);
-            m_identifierLookup.set("class", IdentifierStyle::Class);
-            m_identifierLookup.set("namespace", IdentifierStyle::Namespace);
-        }
-
-        // Keywords that control access
-        {
-            const char* names[] = { "private", "protected", "public" };
-            m_identifierLookup.set(names, SLANG_COUNT_OF(names), IdentifierStyle::Access);
-        }
     }
 
 protected:
-    
+
+        /// Called to set up identifier lookup. Must be performed after options are initials
+    static void _initIdentifierLookup(const Options& options, IdentifierLookup& outLookup);
+
     NamePool m_namePool;
 
     Options m_options;
     DiagnosticSink* m_sink;
     SourceManager* m_sourceManager;
-    IdentifierLookup m_identifierLookup;
-
     
     StringSlicePool m_slicePool;
 };
@@ -1864,7 +1895,7 @@ SlangResult CPPExtractorApp::calcDef(CPPExtractor& extractor, SourceOrigin* orig
 
     for (Node* node : origin->m_nodes)
     {
-        if (node->isClassLike() && node->m_isReflected)
+        if (node->isClassLike() && node->isReflected())
         {
             if (node->m_marker.getContent().indexOf(UnownedStringSlice::fromLiteral("ABSTRACT")) >= 0)
             {
@@ -1878,8 +1909,6 @@ SlangResult CPPExtractorApp::calcDef(CPPExtractor& extractor, SourceOrigin* orig
     return SLANG_OK;
 }
 
-
-
 SlangResult CPPExtractorApp::calcChildrenHeader(CPPExtractor& extractor, StringBuilder& out)
 {
     const List<Node*>& baseTypes = extractor.getBaseTypes();
@@ -1948,6 +1977,46 @@ SlangResult CPPExtractorApp::calcChildrenHeader(CPPExtractor& extractor, StringB
             }
             out << "\n\n";
         }
+
+        if (m_options.m_outputFields)
+        {
+            out << "\n\n /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!! FIELDS !!!!!!!!!!!!!!!!!!!!!!!!!!!! */\n\n";
+
+            for (Node* node : nodes)
+            {
+                // Define the derived types
+                out << "#define " << m_options.m_prefixMark << "FIELDS_" << reflectTypeName << "_" << node->m_name.getContent() << "(_x_, _param_)";
+
+                if (node->m_fields.getCount() > 0)
+                {
+                    out << "\\\n";
+
+                    const Index fieldsCount = node->m_fields.getCount();
+                    bool previousField = false;
+                    for (Index j = 0; j < fieldsCount; ++j) 
+                    {
+                        const auto& field = node->m_fields[j];
+                        
+                        if (field.isReflected())
+                        {
+                            if (previousField)
+                            {
+                                out << "\\\n";
+                            }
+
+                            _indent(1, out);
+
+                            // NOTE! We put the type field in brackets, such that there is no issue with templates containing a comma.
+                            // If stringified
+                            out << "_x_(" << field.name.getContent() << ", (" << field.type << "), _param_)";
+                            previousField = true;
+                        }
+                    }
+                }
+                
+                out << "\n\n";
+            }
+        }
     }
 
     return SLANG_OK;
@@ -2010,33 +2079,6 @@ SlangResult CPPExtractorApp::calcHeader(CPPExtractor& extractor, StringBuilder&
             out << "};\n\n";
         }
 
-#if 0
-        out << "\n";
-        out << "enum class " << reflectTypeName << "Last\n";
-        out << "{\n";
-
-        for (Node* node : nodes)
-        {
-            SLANG_ASSERT(node->isClassLike());
-            // If it's not reflected we don't output, in the enum list
-            if (!node->m_isReflected)
-            {
-                continue;
-            }
-
-            Node* lastDerived = node->findLastDerived();
-            if (lastDerived)
-            {
-                // Okay first we are going to output the enum values
-                const Index depth = node->calcDerivedDepth() - 1;
-                _indent(depth, out);
-                out << node->m_name.Content << " = int(" << reflectTypeName << "Type::" << lastDerived->m_name.Content << "),\n";
-            }
-        }
-
-        out << "};\n\n";
-#endif
-
         // Predeclare the classes
         {
             out << "// Predeclare\n\n";
@@ -2044,7 +2086,7 @@ SlangResult CPPExtractorApp::calcHeader(CPPExtractor& extractor, StringBuilder&
             {
                 SLANG_ASSERT(node->isClassLike());
                 // If it's not reflected we don't output, in the enum list
-                if (node->m_isReflected)
+                if (node->isReflected())
                 {
                     const char* type = (node->m_type == Node::Type::ClassType) ? "class" : "struct";
                     out << type << " " << node->m_name.getContent() << ";\n";
@@ -2052,24 +2094,6 @@ SlangResult CPPExtractorApp::calcHeader(CPPExtractor& extractor, StringBuilder&
             }
         }
 
-#if 0
-        out << "struct " << reflectTypeName << "Super\n";
-        out << "{\n";
-
-        for (Node* node : nodes)
-        {
-            // If it's not reflected we don't output, in the enum list
-            if (node->m_isReflected && node->m_superNode)
-            {
-                _indent(1, out);
-                // We concat _Super so the typedef are distinct from the ones being referenced
-                out << "typedef " << node->m_superNode->m_name.Content << " " << node->m_name.Content << "_Super;\n";
-            }
-        }
-
-        out << "};\n";
-#endif
-
         // Do the macros for each of the types
 
         {
@@ -2129,7 +2153,7 @@ SlangResult CPPExtractorApp::calcHeader(CPPExtractor& extractor, StringBuilder&
                 }
                 out << marker << ", ";
 
-                if (node->m_baseType != Node::BaseType::None || node->m_superNode && node->m_superNode->m_isReflected == false)
+                if (node->m_baseType != Node::BaseType::None || node->m_superNode && node->m_superNode->isReflected() == false)
                 {
                     out << "BASE, ";
                 }
@@ -2164,7 +2188,7 @@ SlangResult CPPExtractorApp::calcHeader(CPPExtractor& extractor, StringBuilder&
 
         for (Node* node : origin->m_nodes)
         {
-            if (!(node->m_isReflected && node->isClassLike()))
+            if (!(node->isReflected() && node->isClassLike()))
             {
                 continue;
             }
@@ -2208,7 +2232,6 @@ SlangResult CPPExtractorApp::writeDefs(CPPExtractor& extractor)
 
 SlangResult CPPExtractorApp::writeOutput(CPPExtractor& extractor)
 {
-    
     String path;
     if (m_options.m_inputDirectory.getLength())
     {
@@ -2257,13 +2280,77 @@ SlangResult CPPExtractorApp::writeOutput(CPPExtractor& extractor)
     return SLANG_OK;
 }
 
+/* static */void CPPExtractorApp::_initIdentifierLookup(const Options& options, IdentifierLookup& outLookup)
+{
+    outLookup.reset();
+
+    // Some keywords
+    {
+        const char* names[] = { "virtual", "typedef", "continue", "if", "case", "break", "catch", "default", "delete", "do", "else", "for", "new", "goto", "return", "switch", "throw", "using", "while" };
+        outLookup.set(names, SLANG_COUNT_OF(names), IdentifierStyle::Keyword);
+    }
+
+    // Type modifier keywords
+    {
+        const char* names[] = { "const", "volatile" };
+        outLookup.set(names, SLANG_COUNT_OF(names), IdentifierStyle::TypeModifier);
+    }
+
+    // Special markers
+    {
+        {
+            StringBuilder buf;
+            buf << options.m_prefixMark;
+            buf << "CLASS_ROOT";
+
+            outLookup.set(buf.getUnownedSlice(), IdentifierStyle::Root);
+        }
+        {
+            StringBuilder buf;
+            buf << options.m_prefixMark;
+            buf << "REFLECT_BASE_CLASS";
+
+            outLookup.set(buf.getUnownedSlice(), IdentifierStyle::BaseClass);
+        }
+        {
+            StringBuilder buf;
+            buf << options.m_prefixMark;
+            buf << "REFLECTED";
+
+            outLookup.set(buf.getUnownedSlice(), IdentifierStyle::Reflected);
+        }
+        {
+            StringBuilder buf;
+            buf << options.m_prefixMark;
+            buf << "UNREFLECTED";
+
+            outLookup.set(buf.getUnownedSlice(), IdentifierStyle::Unreflected);
+        }
+    }
+
+
+    // Keywords which introduce types/scopes
+    {
+        outLookup.set("struct", IdentifierStyle::Struct);
+        outLookup.set("class", IdentifierStyle::Class);
+        outLookup.set("namespace", IdentifierStyle::Namespace);
+    }
+
+    // Keywords that control access
+    {
+        const char* names[] = { "private", "protected", "public" };
+        outLookup.set(names, SLANG_COUNT_OF(names), IdentifierStyle::Access);
+    }
+}
 
 SlangResult CPPExtractorApp::execute(const Options& options)
 {
     m_options = options;
 
-    
-    CPPExtractor extractor(&m_slicePool, &m_namePool, m_sink, &m_identifierLookup);
+    IdentifierLookup identifierLookup;
+    _initIdentifierLookup(options, identifierLookup);
+
+    CPPExtractor extractor(&m_slicePool, &m_namePool, m_sink, &identifierLookup);
 
     // Read in each of the input files
     for (Index i = 0; i < m_options.m_inputPaths.getCount(); ++i)