Initial support for documentation extraction in C++ (#2156)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Split doc extractor such that can be used in C++ extractor.

* Compiles. Update the stdlib docs.

* Fix issue on release builds.

* Add support for extracting documentation to C++ extractor.

* Dump out markup.
Make enum value backing type take tokens.

* Node::Type -> Node::Kind

* More improvements around Node::Type -> Node::Kind

9 files changed, 375 insertions, 131 deletions

diff --git a/tools/slang-cpp-extractor/cpp-extractor-main.cpp b/tools/slang-cpp-extractor/cpp-extractor-main.cpp
index d1ec47e69..9db61f8e0 100644
--- a/tools/slang-cpp-extractor/cpp-extractor-main.cpp
+++ b/tools/slang-cpp-extractor/cpp-extractor-main.cpp
@@ -20,6 +20,7 @@
 #include "../../source/compiler-core/slang-diagnostic-sink.h"
 #include "../../source/compiler-core/slang-name.h"
 #include "../../source/compiler-core/slang-name-convention-util.h"
+#include "../../source/compiler-core/slang-doc-extractor.h"
 
 #include "node.h"
 #include "diagnostics.h"
@@ -63,6 +64,8 @@ public:
 
 protected:
 
+    SlangResult _extractDoc(NodeTree* nodeTree);
+
     NamePool m_namePool;
 
     Options m_options;
@@ -73,6 +76,116 @@ protected:
 };
 
 
+// Work out an appropriate search type for a node type.
+//
+// TODO(JS):
+// NOTE! Currently extractor doesn't extract callable types and so doesn't extract callable types parameters
+static DocMarkupExtractor::SearchStyle _getSearchStyle(Node* node)
+{
+    typedef DocMarkupExtractor::SearchStyle SearchStyle;
+
+    if (!node->getSourceLoc().isValid())
+    {
+        return SearchStyle::None;
+    }
+
+    switch (node->m_kind)
+    {
+        case Node::Kind::Invalid:
+        {
+            return SearchStyle::None;
+        }
+        case Node::Kind::Field:
+        {
+            return SearchStyle::Variable;
+        }
+        case Node::Kind::EnumCase:
+        {
+            return SearchStyle::EnumCase;
+        }
+        case Node::Kind::TypeDef:
+        {
+            return SearchStyle::Variable;
+        }
+        default: break;
+    }
+
+    // Default is to only allow before.
+    return SearchStyle::Before;
+}
+
+SlangResult App::_extractDoc(NodeTree* nodeTree)
+{
+    // Find all of the nodes
+    List<Node*> nodes;
+    // Add the root
+    nodes.add(nodeTree->getRootNode());
+
+    // Traverse all nodes
+    for (Index startIndex = 0; startIndex < nodes.getCount(); ++startIndex)
+    {
+        Node* node = nodes[startIndex];
+
+        ScopeNode* scopeNode = as<ScopeNode>(node);
+
+        if (scopeNode)
+        {
+            for (Node* child : scopeNode->m_children)
+            {
+                nodes.add(child);
+            }
+        }
+    }
+
+    // Find out what to find
+
+    List<DocMarkupExtractor::SearchItemInput> inputItems;
+
+    for (Node* node : nodes)
+    {
+        auto searchStyle = _getSearchStyle(node);
+
+        DocMarkupExtractor::SearchItemInput inputItem;
+        inputItem.searchStyle = searchStyle;
+        inputItem.sourceLoc = node->getSourceLoc();
+
+        inputItems.add(inputItem);
+    }
+
+    List<DocMarkupExtractor::SearchItemOutput> outputItems;
+
+    List<SourceView*> views;
+
+    DocMarkupExtractor extractor;
+
+    SLANG_RETURN_ON_FAIL(extractor.extract(inputItems.getBuffer(), inputItems.getCount(), m_sourceManager, m_sink, views, outputItems));
+
+    // Put what was extracted into the nodes
+    {
+        const Index count = inputItems.getCount();
+        SLANG_ASSERT(count == outputItems.getCount() && count == nodes.getCount());
+
+        for (Index i = 0; i < count; ++i)
+        {
+            const auto& outputItem = outputItems[i];
+
+            // We need to use the index used for input, because in output they can be reordered.
+            const auto inputIndex = outputItem.inputIndex;
+            const auto& inputItem = inputItems[inputIndex];
+
+            if (inputItem.searchStyle != DocMarkupExtractor::SearchStyle::None && outputItem.text.getLength())
+            {
+                Node* node = nodes[inputIndex];
+
+                node->m_markup = outputItem.text;
+                node->m_markupVisibility = outputItem.visibilty;
+            }
+        }
+    }
+
+    return SLANG_OK;
+}
+
 SlangResult App::execute(const Options& options)
 {
     m_options = options;
@@ -143,6 +256,11 @@ SlangResult App::execute(const Options& options)
         }
     }
 
+    if (options.m_extractDoc)
+    {
+        SLANG_RETURN_ON_FAIL(_extractDoc(&tree));
+    }
+
     // Dump out the tree
     if (options.m_dump)
     {
diff --git a/tools/slang-cpp-extractor/macro-writer.cpp b/tools/slang-cpp-extractor/macro-writer.cpp
index 2259f1800..c4245a646 100644
--- a/tools/slang-cpp-extractor/macro-writer.cpp
+++ b/tools/slang-cpp-extractor/macro-writer.cpp
@@ -214,7 +214,7 @@ SlangResult MacroWriter::calcTypeHeader(NodeTree* tree, TypeSet* typeSet, String
 
     for (Node* scopeNode : baseScopePath)
     {
-        SLANG_ASSERT(scopeNode->m_type == Node::Type::Namespace);
+        SLANG_ASSERT(scopeNode->m_kind == Node::Kind::Namespace);
         out << "namespace " << scopeNode->m_name.getContent() << " {\n";
     }
 
@@ -262,7 +262,7 @@ SlangResult MacroWriter::calcTypeHeader(NodeTree* tree, TypeSet* typeSet, String
             // If it's not reflected we don't output, in the enum list
             if (node->isReflected())
             {
-                const char* type = (node->m_type == Node::Type::ClassType) ? "class" : "struct";
+                const char* type = (node->m_kind == Node::Kind::ClassType) ? "class" : "struct";
                 out << type << " " << node->m_name.getContent() << ";\n";
             }
         }
diff --git a/tools/slang-cpp-extractor/node-tree.cpp b/tools/slang-cpp-extractor/node-tree.cpp
index 3002a7abd..05360e0de 100644
--- a/tools/slang-cpp-extractor/node-tree.cpp
+++ b/tools/slang-cpp-extractor/node-tree.cpp
@@ -18,7 +18,7 @@ NodeTree::NodeTree(StringSlicePool* typePool, NamePool* namePool, IdentifierLook
     m_identifierLookup(identifierLookup),
     m_typeSetPool(StringSlicePool::Style::Empty)
 {
-    m_rootNode = new ScopeNode(Node::Type::Namespace);
+    m_rootNode = new ScopeNode(Node::Kind::Namespace);
     m_rootNode->m_reflectionType = ReflectionType::Reflected;
 }
 
diff --git a/tools/slang-cpp-extractor/node.cpp b/tools/slang-cpp-extractor/node.cpp
index 3b2403816..2074b7c41 100644
--- a/tools/slang-cpp-extractor/node.cpp
+++ b/tools/slang-cpp-extractor/node.cpp
@@ -3,6 +3,7 @@
 #include "file-util.h"
 
 #include "../../source/core/slang-string-util.h"
+#include "../../source/core/slang-string-escape-util.h"
 
 namespace CppExtract {
 
@@ -10,6 +11,40 @@ namespace CppExtract {
 
 SLANG_FORCE_INLINE static void _indent(Index indentCount, StringBuilder& out) { FileUtil::indent(indentCount, out); }
 
+void Node::dumpMarkup(int indentCount, StringBuilder& out)
+{
+    if (m_markup.getLength() <= 0)
+    {
+        return;
+    }
+
+    List<UnownedStringSlice> lines;
+    StringUtil::calcLines(m_markup.getUnownedSlice(), lines);
+
+    // Remove empty lines from the end
+    while (lines.getCount())
+    {
+        auto lastLine = lines.getLast();
+        if (lastLine.trim().getLength() == 0)
+        {
+            lines.removeLast();
+            continue;
+        }
+        break;
+    }
+
+    if (lines.getCount() == 0)
+    {
+        return;
+    }
+
+    for (auto line : lines)
+    {
+        _indent(indentCount, out);
+        out << "// " << line << "\n";
+    }
+}
+
 ScopeNode* Node::getRootScope()
 {
     if (m_parentScope)
@@ -47,7 +82,7 @@ void Node::calcAbsoluteName(StringBuilder& outName) const
             outName << "::";
         }
 
-        if (node->m_type == Type::AnonymousNamespace)
+        if (node->m_kind == Kind::AnonymousNamespace)
         {
             outName << "{Anonymous}";
         }
@@ -119,7 +154,7 @@ void Node::calcAbsoluteName(StringBuilder& outName) const
     for (Node* node : scope->m_children)
     {
         EnumNode* enumNode = as<EnumNode>(node);
-        if (enumNode && enumNode->m_type == Node::Type::Enum)
+        if (enumNode && enumNode->m_kind == Node::Kind::Enum)
         {
             Node** nodePtr = enumNode->m_childMap.TryGetValue(name);
             if (nodePtr)
@@ -251,7 +286,7 @@ ScopeNode* ScopeNode::getAnonymousNamespace()
 {
     if (!m_anonymousNamespace)
     {
-        m_anonymousNamespace = new ScopeNode(Type::AnonymousNamespace);
+        m_anonymousNamespace = new ScopeNode(Kind::AnonymousNamespace);
         m_anonymousNamespace->m_parentScope = this;
         m_children.add(m_anonymousNamespace);
     }
@@ -263,7 +298,7 @@ void ScopeNode::addChild(Node* child)
 {
     SLANG_ASSERT(child->m_parentScope == nullptr);
     // Can't add anonymous namespace this way - should be added via getAnonymousNamespace
-    SLANG_ASSERT(child->m_type != Type::AnonymousNamespace);
+    SLANG_ASSERT(child->m_kind != Kind::AnonymousNamespace);
 
     child->m_parentScope = this;
     m_children.add(child);
@@ -298,15 +333,17 @@ void ScopeNode::calcScopeDepthFirst(List<Node*>& outNodes)
 
 void ScopeNode::dump(int indentCount, StringBuilder& out)
 {
+    dumpMarkup(indentCount, out);
+
     _indent(indentCount, out);
 
-    switch (m_type)
+    switch (m_kind)
     {
-        case Type::AnonymousNamespace:
+        case Kind::AnonymousNamespace:
         {
             out << "namespace {\n";
         }
-        case Type::Namespace:
+        case Kind::Namespace:
         {
             if (m_name.hasContent())
             {
@@ -331,17 +368,66 @@ void ScopeNode::dump(int indentCount, StringBuilder& out)
 
 /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! EnumCaseNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */
 
+/* Returns true if needs space between the tokens.
+It determines this based on the locs, and if they contain something between them.
+*/
+static bool _needsSpace(const Token& prevTok, const Token& tok)
+{
+    auto prevLoc = prevTok.getLoc();
+    auto loc = tok.getLoc();
+
+    auto prevContent = prevTok.getContent();
+    auto content = tok.getContent();
+
+    if (prevLoc + prevContent.getLength() == loc)
+    {
+        return false;
+    }
+
+    return true;
+}
+
+
+static void _dumpTokens(const Token* toks, Index count, StringBuilder& out)
+{
+    if (count > 0)
+    {
+        out << toks[0].getContent();
+
+        for (Index i = 1; i < count; ++i)
+        {
+            const auto& prevToken = toks[i - 1];
+            const auto& token = toks[i];
+
+            if (_needsSpace(prevToken, token))
+            {
+                out << " ";
+            }
+
+            out << token.getContent();
+        }
+    }
+}
+
+static void _dumpTokens(const List<Token>& toks, StringBuilder& out)
+{
+    _dumpTokens(toks.getBuffer(), toks.getCount(), out);
+}
+
+
 void EnumCaseNode::dump(int indent, StringBuilder& out)
 {
     if (isReflected())
     {
+        dumpMarkup(indent, out);
+
         _indent(indent, out);
         out << m_name.getContent();
 
-        if (m_value.type != TokenType::Invalid)
+        if (m_valueTokens.getCount())
         {
             out << " = ";
-            out << m_value.getContent();
+            _dumpTokens(m_valueTokens, out);
         }
 
         out << ",\n";
@@ -354,16 +440,13 @@ void TypeDefNode::dump(int indent, StringBuilder& out)
 {
     if (isReflected())
     {
+        dumpMarkup(indent, out);
+
         _indent(indent, out);
 
         out << "typedef ";
-
-        for (auto& tok : m_targetTypeTokens)
-        {
-            out << tok.getContent() << " ";
-        }
-
-        out << m_name.getContent() << ";\n";
+        _dumpTokens(m_targetTypeTokens, out);
+        out << " " << m_name.getContent() << ";\n";
     }
 }
 
@@ -376,11 +459,13 @@ void EnumNode::dump(int indent, StringBuilder& out)
         return;
     }
 
+    dumpMarkup(indent, out);
+
     _indent(indent, out);
 
     out << "enum ";
 
-    if (m_type == Type::EnumClass)
+    if (m_kind == Kind::EnumClass)
     {
         out << "class ";
     }
@@ -390,9 +475,10 @@ void EnumNode::dump(int indent, StringBuilder& out)
         out << m_name.getContent();
     }
 
-    if (m_backingToken.type != TokenType::Invalid)
-    {
-        out << " : " << m_backingToken.getContent();
+    if (m_backingTokens.getCount() > 0)
+    { 
+        out << " : ";
+        _dumpTokens(m_backingTokens, out);
     }
 
     out << "\n";
@@ -414,6 +500,8 @@ void FieldNode::dump(int indent, StringBuilder& out)
 {
     if (isReflected())
     {
+        dumpMarkup(indent, out);
+
         _indent(indent, out);
         out << m_fieldType << " " << m_name.getContent() << "\n";
     }
@@ -506,9 +594,11 @@ void ClassLikeNode::getReflectedDerivedTypes(List<ClassLikeNode*>& out) const
 
 void ClassLikeNode::dump(int indentCount, StringBuilder& out)
 {
+    dumpMarkup(indentCount, out);
+
     _indent(indentCount, out);
 
-    const char* typeName = (m_type == Type::StructType) ? "struct" : "class";
+    const char* typeName = (m_kind == Kind::StructType) ? "struct" : "class";
 
     out << typeName << " ";
 
diff --git a/tools/slang-cpp-extractor/node.h b/tools/slang-cpp-extractor/node.h
index c741024e4..f649f1adb 100644
--- a/tools/slang-cpp-extractor/node.h
+++ b/tools/slang-cpp-extractor/node.h
@@ -3,6 +3,8 @@
 
 #include "diagnostics.h"
 
+#include "../../source/compiler-core/slang-doc-extractor.h"
+
 namespace CppExtract {
 using namespace Slang;
 
@@ -21,7 +23,7 @@ struct ScopeNode;
 class Node : public RefObject
 {
 public:
-    enum class Type : uint8_t
+    enum class Kind : uint8_t
     {
         Invalid,
 
@@ -42,29 +44,49 @@ public:
         CountOf,
     };
 
-    enum class TypeRange
+    enum class KindRange
     {
-        ScopeStart = int(Type::StructType),
-        ScopeEnd = int(Type::AnonymousNamespace),
+        ScopeStart = int(Kind::StructType),
+        ScopeEnd = int(Kind::AnonymousNamespace),
+
+        ClassLikeStart = int(Kind::StructType),
+        ClassLikeEnd = int(Kind::ClassType),
 
-        ClassLikeStart = int(Type::StructType),
-        ClassLikeEnd = int(Type::ClassType),
+        ScopeTypeStart = int(Kind::StructType),
+        ScopeTypeEnd = int(Kind::EnumClass),
 
-        EnumStart = int(Type::Enum),
-        EnumEnd = int(Type::EnumClass),
+        OtherTypeStart = int(Kind::TypeDef),
+        OtherTypeEnd = int(Kind::TypeDef),
+
+        EnumStart = int(Kind::Enum),
+        EnumEnd = int(Kind::EnumClass),
     };
 
-    static bool isScopeType(Type type) { return int(type) >= int(TypeRange::ScopeStart) && int(type) <= int(TypeRange::ScopeEnd); }
-    static bool isClassLikeType(Type type) { return int(type) >= int(TypeRange::ClassLikeStart) &&  int(type) <= int(TypeRange::ClassLikeEnd); }
-    static bool isEnumLikeType(Type type) { return int(type) >= int(TypeRange::EnumStart) &&  int(type) <= int(TypeRange::EnumEnd); }
-    static bool canAcceptTypes(Type type)
+        /// Returns true if kind can cast to this type
+        /// Used for implementing as<T> casting
+    static bool isOfKind(Kind type) { return true; }
+
+    static bool isKindScope(Kind kind) { return int(kind) >= int(KindRange::ScopeStart) && int(kind) <= int(KindRange::ScopeEnd); }
+    static bool isKindClassLike(Kind kind) { return int(kind) >= int(KindRange::ClassLikeStart) &&  int(kind) <= int(KindRange::ClassLikeEnd); }
+    static bool isKindEnumLike(Kind kind) { return int(kind) >= int(KindRange::EnumStart) &&  int(kind) <= int(KindRange::EnumEnd); }
+
+        /// It a type, but doesn't have a scope
+    static bool isKindOtherType(Kind kind) { return int(kind) >= int(KindRange::OtherTypeStart) && int(kind) <= int(KindRange::OtherTypeEnd); }
+        /// Is a type and has a scope
+    static bool isKindScopeType(Kind kind) { return int(kind) >= int(KindRange::ScopeTypeStart) && int(kind) <= int(KindRange::ScopeTypeEnd); }
+
+        /// True if the kind is any type
+    static bool isKindType(Kind kind) { return isKindOtherType(kind) || isKindScopeType(kind); }
+
+        /// True if the kind can accept contained types
+    static bool canKindContainTypes(Kind type)
     {
         switch (type)
         {
-            case Type::StructType:
-            case Type::ClassType:
-            case Type::Namespace:
-            case Type::AnonymousNamespace:
+            case Kind::StructType:
+            case Kind::ClassType:
+            case Kind::Namespace:
+            case Kind::AnonymousNamespace:
             {
                 return true;
             }
@@ -73,9 +95,11 @@ public:
         return false;
     }
 
-    static bool isType(Type type) { return true; }
+    bool isClassLike() const { return isKindClassLike(m_kind); }
 
-    bool isClassLike() const { return isClassLikeType(m_type); }
+        /// These are useful for the filter
+    static bool isClassLikeAndReflected(Node* node) { return node->isClassLike() && node->isReflected(); }
+    static bool isClassLike(Node* node) { return isKindClassLike(node->m_kind); }
 
     virtual void dump(int indent, StringBuilder& out) = 0;
 
@@ -94,13 +118,12 @@ public:
         /// True if reflected
     bool isReflected() const { return m_reflectionType == ReflectionType::Reflected; }
 
+    SourceLoc getSourceLoc() const { return m_name.getLoc(); }
+
     ScopeNode* getRootScope();
 
     typedef bool(*Filter)(Node* node);
-
-    static bool isClassLikeAndReflected(Node* node) { return node->isClassLike() && node->isReflected(); }
-    static bool isClassLike(Node* node) { return isClassLikeType(node->m_type); }
-
+    
     template <typename T>
     static void filter(Filter filter, List<T*>& io) { const Node* _isNodeDerived = (T*)nullptr; SLANG_UNUSED(_isNodeDerived); filterImpl(filter, reinterpret_cast<List<Node*>&>(io)); }
 
@@ -122,15 +145,21 @@ public:
 
     static void splitPath(const UnownedStringSlice& slice, List<UnownedStringSlice>& outSplitPath);
 
-    Node(Type type) :
-        m_type(type),
+        /// If markup is specified dump it
+    void dumpMarkup(int indent, StringBuilder& out);
+
+    Node(Kind type) :
+        m_kind(type),
         m_parentScope(nullptr),
         m_reflectionType(ReflectionType::NotReflected)
     {
     }
 
-    Type m_type;                        ///< The type of node this is
-    ReflectionType m_reflectionType;    /// Classes can be traversed, but not reflected. To be reflected they have to contain the marker
+    Kind m_kind;                        ///< The kind of node this is
+    ReflectionType m_reflectionType;    ///< Classes can be traversed, but not reflected. To be reflected they have to contain the marker
+    
+    MarkupVisibility m_markupVisibility = MarkupVisibility::Public;         ///< The visibility of the markup
+    String m_markup;                                                        ///< Documentation associated with this node
 
     Token m_name;                       ///< The name of this scope/type    
 
@@ -141,15 +170,15 @@ struct ScopeNode : public Node
 {
     typedef Node Super;
 
-    static bool isType(Type type) { return isScopeType(type); }
+    static bool isOfKind(Kind kind) { return isKindScope(kind); }
 
     virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE;
     virtual void calcScopeDepthFirst(List<Node*>& outNodes) SLANG_OVERRIDE;
 
         /// True if can accept fields (class like types can)
-    bool acceptsFields() const { return isClassLike(); }
+    bool canContainFields() const { return isClassLike(); }
         /// True if the scope can accept types
-    bool acceptsTypes() const { return canAcceptTypes(m_type); }
+    bool canContainTypes() const { return canKindContainTypes(m_kind); }
 
         /// Gets the reflection for any contained types
     ReflectionType getContainedReflectionType() const { return m_reflectionType == ReflectionType::NotReflected ? ReflectionType::NotReflected : m_reflectionOverride; }
@@ -163,8 +192,8 @@ struct ScopeNode : public Node
         /// Gets the anonymous namespace associated with this scope
     ScopeNode* getAnonymousNamespace();
 
-    ScopeNode(Type type) :
-        Super(type),
+    ScopeNode(Kind kind) :
+        Super(kind),
         m_reflectionOverride(ReflectionType::Reflected),
         m_anonymousNamespace(nullptr)
     {
@@ -187,12 +216,12 @@ struct FieldNode : public Node
 {
     typedef Node Super;
 
-    static bool isType(Type type) { return type == Type::Field; }
+    static bool isOfKind(Kind type) { return type == Kind::Field; }
 
     virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE;
 
     FieldNode() :
-        Super(Type::Field)
+        Super(Kind::Field)
     {
     }
 
@@ -205,7 +234,7 @@ struct ClassLikeNode : public ScopeNode
 {
     typedef ScopeNode Super;
 
-    static bool isType(Type type) { return isClassLikeType(type); }
+    static bool isOfKind(Kind kind) { return isKindClassLike(kind); }
 
         /// Add a node that is derived from this
     void addDerived(ClassLikeNode* derived);
@@ -231,13 +260,13 @@ struct ClassLikeNode : public ScopeNode
     // Node Impl
     virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE;
 
-    ClassLikeNode(Type type) :
-        Super(type),
+    ClassLikeNode(Kind kind) :
+        Super(kind),
         m_origin(nullptr),
         m_typeSet(nullptr),
         m_superNode(nullptr)
     {
-        SLANG_ASSERT(type == Type::ClassType || type == Type::StructType);
+        SLANG_ASSERT(kind == Kind::ClassType || kind == Kind::StructType);
     }
 
     SourceOrigin* m_origin;                             ///< Defines where this was uniquely defined. 
@@ -256,43 +285,44 @@ struct EnumCaseNode : public Node
 {
     typedef Node Super;
 
-    static bool isType(Type type) { return type == Type::EnumCase; }
+    static bool isOfKind(Kind kind) { return kind == Kind::EnumCase; }
 
     virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE;
 
     EnumCaseNode():
-        Super(Type::EnumCase)
+        Super(Kind::EnumCase)
     {
     }
 
-    Token m_value;              ///< If not defined will be invalid
+    // Tokens that make up the value. If not defined will be empty
+    List<Token> m_valueTokens;
 };
 
 struct EnumNode : public ScopeNode
 {
     typedef ScopeNode Super;
-    static bool isType(Type type) { return isEnumLikeType(type); }
+    static bool isOfKind(Kind kind) { return isKindEnumLike(kind); }
 
     virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE;
 
-    EnumNode(Type type):
-        Super(type)
+    EnumNode(Kind kind):
+        Super(kind)
     {
-        SLANG_ASSERT(isEnumLikeType(type));
+        SLANG_ASSERT(isKindEnumLike(kind));
     }
 
-    Token m_backingToken;
+    List<Token> m_backingTokens;
 };
 
 struct TypeDefNode : public Node
 {
     typedef Node Super;
-    static bool isType(Type type) { return type == Type::TypeDef; }
+    static bool isOfKind(Kind kind) { return kind == Kind::TypeDef; }
 
     virtual void dump(int indent, StringBuilder& out) SLANG_OVERRIDE;
 
     TypeDefNode():
-        Super(Type::TypeDef)
+        Super(Kind::TypeDef)
     {
     }
 
@@ -300,7 +330,7 @@ struct TypeDefNode : public Node
 };
 
 template <typename T>
-T* as(Node* node) { return (node && T::isType(node->m_type)) ? static_cast<T*>(node) : nullptr; }
+T* as(Node* node) { return (node && T::isOfKind(node->m_kind)) ? static_cast<T*>(node) : nullptr; }
 
 } // CppExtract
 
diff --git a/tools/slang-cpp-extractor/options.h b/tools/slang-cpp-extractor/options.h
index 14ef9ecfc..186ac2fbb 100644
--- a/tools/slang-cpp-extractor/options.h
+++ b/tools/slang-cpp-extractor/options.h
@@ -25,6 +25,7 @@ struct Options
     bool m_defs = false;            ///< If set will output a '-defs.h' file for each of the input files, that corresponds to previous defs files (although doesn't have fields/RAW)
     bool m_dump = false;            ///< If true will dump to stderr the types/fields and hierarchy it extracted
     bool m_runUnitTests = false;    ///< If true will run internal unit tests
+    bool m_extractDoc = true;       ///< If set will try to extract documentation associated with nodes
 
     bool m_outputFields = false;     ///< When dumping macros also dump field definitions
 
diff --git a/tools/slang-cpp-extractor/parser.cpp b/tools/slang-cpp-extractor/parser.cpp
index 97c156f2f..29bc517f5 100644
--- a/tools/slang-cpp-extractor/parser.cpp
+++ b/tools/slang-cpp-extractor/parser.cpp
@@ -11,7 +11,7 @@ namespace CppExtract {
 using namespace Slang;
 
 // If fails then we need more bits to identify types
-SLANG_COMPILE_TIME_ASSERT(int(Node::Type::CountOf) <= 8 * sizeof(uint32_t));
+SLANG_COMPILE_TIME_ASSERT(int(Node::Kind::CountOf) <= 8 * sizeof(uint32_t));
 
 // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Parser !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
@@ -21,34 +21,39 @@ Parser::Parser(NodeTree* nodeTree, DiagnosticSink* sink) :
     m_nodeTypeEnabled(0)
 {
     // Enable types by default
-    const Node::Type defaultEnabled[] =
-    {
-        Node::Type::ClassType,
-        Node::Type::StructType,
-        Node::Type::Namespace,
-        Node::Type::AnonymousNamespace,
-        Node::Type::Field,
+    const Node::Kind defaultEnabled[] =
+    {
+        Node::Kind::ClassType,
+        Node::Kind::StructType,
+        Node::Kind::Namespace,
+        Node::Kind::AnonymousNamespace,
+        Node::Kind::Field,
+
+        // These are disabled by default because AST uses macro magic to build up the types
+        // Node::Type::TypeDef,
+        // Node::Type::Enum,
+        // Node::Type::EnumClass,
     };
-    setTypesEnabled(defaultEnabled, SLANG_COUNT_OF(defaultEnabled));
+    setKindsEnabled(defaultEnabled, SLANG_COUNT_OF(defaultEnabled));
 }
 
-void Parser::setTypeEnabled(Node::Type type, bool isEnabled )
+void Parser::setKindEnabled(Node::Kind kind, bool isEnabled)
 {
     if (isEnabled)
     {
-        m_nodeTypeEnabled |= (NodeTypeBitType(1) << int(type));
+        m_nodeTypeEnabled |= (NodeTypeBitType(1) << int(kind));
     }
     else
     {
-        m_nodeTypeEnabled &= ~(NodeTypeBitType(1) << int(type));
+        m_nodeTypeEnabled &= ~(NodeTypeBitType(1) << int(kind));
     }
 }
 
-void Parser::setTypesEnabled(const Node::Type* types, Index typesCount, bool isEnabled)
+void Parser::setKindsEnabled(const Node::Kind* kinds, Index kindsCount, bool isEnabled)
 {
-    for (Index i = 0; i < typesCount; ++i)
+    for (Index i = 0; i < kindsCount; ++i)
     {
-        setTypeEnabled(types[i], isEnabled);
+        setKindEnabled(kinds[i], isEnabled);
     }
 }
 
@@ -157,9 +162,9 @@ SlangResult Parser::pushScope(ScopeNode* scopeNode)
                 return SLANG_FAIL;
             }
 
-            if (foundNode->m_type == Node::Type::Namespace)
+            if (foundNode->m_kind == Node::Kind::Namespace)
             {
-                if (foundNode->m_type != scopeNode->m_type)
+                if (foundNode->m_kind != scopeNode->m_kind)
                 {
                     // Different types can't work
                     m_sink->diagnose(m_reader.peekToken(), CPPDiagnostics::typeAlreadyDeclared, scopeNode->m_name.getContent());
@@ -274,13 +279,13 @@ SlangResult Parser::_parseEnum()
     // consume enum
     SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &enumToken));
 
-    if (!m_currentScope->acceptsTypes())
+    if (!m_currentScope->canContainTypes())
     {
         m_sink->diagnose(enumToken.loc, CPPDiagnostics::cannotDeclareTypeInScope);
         return SLANG_FAIL;
     }
 
-    Node::Type type = Node::Type::Enum;
+    Node::Kind kind = Node::Kind::Enum;
 
     Token nameToken;
     if (advanceIfToken(TokenType::Identifier, &nameToken))
@@ -289,7 +294,7 @@ SlangResult Parser::_parseEnum()
 
         if (style == IdentifierStyle::Class)
         {
-            type = Node::Type::EnumClass;
+            kind = Node::Kind::EnumClass;
             SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &nameToken));
         }
         else if (style == IdentifierStyle::None)
@@ -303,7 +308,7 @@ SlangResult Parser::_parseEnum()
         }
     }
 
-    RefPtr<EnumNode> node = new EnumNode(type);
+    RefPtr<EnumNode> node = new EnumNode(kind);
     node->m_name = nameToken;
 
     if (advanceIfToken(TokenType::Colon))
@@ -328,9 +333,9 @@ SlangResult Parser::_parseEnum()
         // for our uses here.
         // If we can't find the type, we could assume it's size is undefined
 
-        if (backingTokens.getCount() == 1)
+        if (backingTokens.getCount() > 0)
         {
-            node->m_backingToken = backingTokens[0];
+            node->m_backingTokens.swapWith(backingTokens);
         }
     }
 
@@ -345,7 +350,7 @@ SlangResult Parser::_parseEnum()
             {
                 // Strictly speaking we should check the backing type etc, match, but for now ignore and assume it's ok
 
-                if (node->m_type == type)
+                if (node->m_kind == kind)
                 {
                     return SLANG_OK;
                 }
@@ -400,9 +405,9 @@ SlangResult Parser::_parseEnum()
                 valueTokens.add(m_reader.advanceToken());
             }
 
-            if (valueTokens.getCount() == 1)
+            if (valueTokens.getCount() > 0)
             {
-                caseNode->m_value = valueTokens[0];
+                caseNode->m_valueTokens.swapWith(valueTokens);
             }
         }
 
@@ -422,12 +427,12 @@ SlangResult Parser::_parseEnum()
     return popScope();
 }
 
-SlangResult Parser::_maybeParseNode(Node::Type type)
+SlangResult Parser::_maybeParseNode(Node::Kind kind)
 {
     // We are looking for
     // struct/class identifier [: [public|private|protected] Identifier ] { [public|private|proctected:]* marker ( identifier );
 
-    if (type == Node::Type::Namespace)
+    if (kind == Node::Kind::Namespace)
     {
         // consume namespace
         SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier));
@@ -442,7 +447,7 @@ SlangResult Parser::_maybeParseNode(Node::Type type)
             if (advanceIfToken(TokenType::LBrace))
             {
                 // Okay looks like we are opening a namespace
-                RefPtr<ScopeNode> node(new ScopeNode(Node::Type::Namespace));
+                RefPtr<ScopeNode> node(new ScopeNode(Node::Kind::Namespace));
                 node->m_name = name;
                 // Push the node
                 return pushScope(node);
@@ -452,14 +457,14 @@ SlangResult Parser::_maybeParseNode(Node::Type type)
         // Just ignore it then
         return SLANG_OK;
     }
-    else if (Node::isEnumLikeType(type))
+    else if (Node::isKindEnumLike(kind))
     {
         return _parseEnum();
     }
 
     // Must be class | struct
 
-    SLANG_ASSERT(type == Node::Type::ClassType || type == Node::Type::StructType);
+    SLANG_ASSERT(kind == Node::Kind::ClassType || kind == Node::Kind::StructType);
 
     Token name;
 
@@ -474,7 +479,7 @@ SlangResult Parser::_maybeParseNode(Node::Type type)
         return SLANG_OK;
     }
 
-    RefPtr<ClassLikeNode> node(new ClassLikeNode(type));
+    RefPtr<ClassLikeNode> node(new ClassLikeNode(kind));
     node->m_name = name;
 
     // Defaults to not reflected
@@ -962,7 +967,7 @@ SlangResult Parser::_parseBalanced(DiagnosticSink* sink)
 
 SlangResult Parser::_parseTypeDef()
 {
-    if (!m_currentScope->acceptsTypes())
+    if (!m_currentScope->canContainTypes())
     {
         m_sink->diagnose(m_reader.peekLoc(), CPPDiagnostics::cannotDeclareTypeInScope);
         return SLANG_FAIL;
@@ -1078,16 +1083,16 @@ SlangResult Parser::_maybeParseField()
     return SLANG_OK;
 }
 
-/* static */Node::Type Parser::_toNodeType(IdentifierStyle style)
+/* static */Node::Kind Parser::_toNodeKind(IdentifierStyle style)
 {
     switch (style)
     {
-        case IdentifierStyle::Class:        return Node::Type::ClassType;
-        case IdentifierStyle::Struct:       return Node::Type::StructType;
-        case IdentifierStyle::Namespace:    return Node::Type::Namespace;
-        case IdentifierStyle::Enum:         return Node::Type::Enum;
-        case IdentifierStyle::TypeDef:      return Node::Type::TypeDef;
-        default: return Node::Type::Invalid;
+        case IdentifierStyle::Class:        return Node::Kind::ClassType;
+        case IdentifierStyle::Struct:       return Node::Kind::StructType;
+        case IdentifierStyle::Namespace:    return Node::Kind::Namespace;
+        case IdentifierStyle::Enum:         return Node::Kind::Enum;
+        case IdentifierStyle::TypeDef:      return Node::Kind::TypeDef;
+        default: return Node::Kind::Invalid;
     }
 }
 
@@ -1118,7 +1123,7 @@ SlangResult Parser::_parsePreDeclare()
     SLANG_RETURN_ON_FAIL(expect(TokenType::Comma));
 
     // Get the type of type
-    Node::Type nodeType;
+    Node::Kind nodeKind;
     {
         Token typeToken;
         SLANG_RETURN_ON_FAIL(expect(TokenType::Identifier, &typeToken));
@@ -1130,7 +1135,7 @@ SlangResult Parser::_parsePreDeclare()
             m_sink->diagnose(typeToken, CPPDiagnostics::expectingTypeKeyword, typeToken.getContent());
             return SLANG_FAIL;
         }
-        nodeType = _toNodeType(style);
+        nodeKind = _toNodeKind(style);
     }
 
     Token name;
@@ -1145,12 +1150,12 @@ SlangResult Parser::_parsePreDeclare()
 
     SLANG_RETURN_ON_FAIL(expect(TokenType::RParent));
 
-    switch (nodeType)
+    switch (nodeKind)
     {
-        case Node::Type::ClassType:
-        case Node::Type::StructType:
+        case Node::Kind::ClassType:
+        case Node::Kind::StructType:
         {
-            RefPtr<ClassLikeNode> node(new ClassLikeNode(nodeType));
+            RefPtr<ClassLikeNode> node(new ClassLikeNode(nodeKind));
 
             node->m_name = name;
             node->m_super = super;
@@ -1274,7 +1279,7 @@ SlangResult Parser::parse(SourceOrigin* sourceOrigin, const Options* options)
                     }
                     case IdentifierStyle::TypeDef:
                     {
-                        if (isTypeEnabled(Node::Type::TypeDef))
+                        if (isTypeEnabled(Node::Kind::TypeDef))
                         {
                             SLANG_RETURN_ON_FAIL(_parseTypeDef());
                         }
@@ -1291,12 +1296,12 @@ SlangResult Parser::parse(SourceOrigin* sourceOrigin, const Options* options)
 
                         if (flags & IdentifierFlag::StartScope)
                         {
-                            Node::Type type = _toNodeType(style);
-                            SLANG_ASSERT(type != Node::Type::Invalid);
+                            Node::Kind kind = _toNodeKind(style);
+                            SLANG_ASSERT(kind != Node::Kind::Invalid);
 
-                            if (isTypeEnabled(type))
+                            if (isTypeEnabled(kind))
                             {
-                                SLANG_RETURN_ON_FAIL(_maybeParseNode(type));
+                                SLANG_RETURN_ON_FAIL(_maybeParseNode(kind));
                             }
                             else
                             {
@@ -1307,7 +1312,7 @@ SlangResult Parser::parse(SourceOrigin* sourceOrigin, const Options* options)
                         {
                             // Special case the node that's the root of the hierarchy (as far as reflection is concerned)
                             // This could be a field
-                            if (m_currentScope->acceptsFields())
+                            if (m_currentScope->canContainFields())
                             {
                                 SLANG_RETURN_ON_FAIL(_maybeParseField());
                             }
diff --git a/tools/slang-cpp-extractor/parser.h b/tools/slang-cpp-extractor/parser.h
index f0febb728..5ed862d65 100644
--- a/tools/slang-cpp-extractor/parser.h
+++ b/tools/slang-cpp-extractor/parser.h
@@ -31,14 +31,14 @@ public:
         /// Parse the contents of the source file
     SlangResult parse(SourceOrigin* sourceOrigin, const Options* options);
 
-    void setTypeEnabled(Node::Type type, bool isEnabled = true);
-    bool isTypeEnabled(Node::Type type) { return (m_nodeTypeEnabled & (NodeTypeBitType(1) << int(type))) != 0; }
-    void setTypesEnabled(const Node::Type* types, Index typesCount, bool isEnabled = true);
+    void setKindEnabled(Node::Kind kind, bool isEnabled = true);
+    bool isTypeEnabled(Node::Kind kind) { return (m_nodeTypeEnabled & (NodeTypeBitType(1) << int(kind))) != 0; }
+    void setKindsEnabled(const Node::Kind* kinds, Index kindsCount, bool isEnabled = true);
 
     Parser(NodeTree* nodeTree, DiagnosticSink* sink);
 
 protected:
-    static Node::Type _toNodeType(IdentifierStyle style);
+    static Node::Kind _toNodeKind(IdentifierStyle style);
 
     bool _isMarker(const UnownedStringSlice& name);
 
@@ -47,7 +47,7 @@ protected:
     SlangResult _parsePreDeclare();
     SlangResult _parseTypeSet();
 
-    SlangResult _maybeParseNode(Node::Type type);
+    SlangResult _maybeParseNode(Node::Kind kind);
     SlangResult _maybeParseField();
 
     SlangResult _parseTypeDef();
diff --git a/tools/slang-cpp-extractor/unit-test.cpp b/tools/slang-cpp-extractor/unit-test.cpp
index 7648e4e50..a30a3caa7 100644
--- a/tools/slang-cpp-extractor/unit-test.cpp
+++ b/tools/slang-cpp-extractor/unit-test.cpp
@@ -67,8 +67,8 @@ static const char someSource[] =
         Parser parser(&tree, &state.m_sink);
 
         {
-            const Node::Type enableTypes[] = { Node::Type::Enum, Node::Type::EnumClass, Node::Type::EnumCase, Node::Type::TypeDef };
-            parser.setTypesEnabled(enableTypes, SLANG_COUNT_OF(enableTypes));
+            const Node::Kind enableKinds[] = { Node::Kind::Enum, Node::Kind::EnumClass, Node::Kind::EnumCase, Node::Kind::TypeDef };
+            parser.setKindsEnabled(enableKinds, SLANG_COUNT_OF(enableKinds));
         }
 
         SLANG_RETURN_ON_FAIL(parser.parse(sourceOrigin, &state.m_options));