Initial support for documentation extraction in C++ (#2156)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Split doc extractor such that can be used in C++ extractor.

* Compiles. Update the stdlib docs.

* Fix issue on release builds.

* Add support for extracting documentation to C++ extractor.

* Dump out markup.
Make enum value backing type take tokens.

* Node::Type -> Node::Kind

* More improvements around Node::Type -> Node::Kind

1 files changed, 111 insertions, 21 deletions

diff --git a/tools/slang-cpp-extractor/node.cpp b/tools/slang-cpp-extractor/node.cpp
index 3b2403816..2074b7c41 100644
--- a/tools/slang-cpp-extractor/node.cpp
+++ b/tools/slang-cpp-extractor/node.cpp
@@ -3,6 +3,7 @@
 #include "file-util.h"
 
 #include "../../source/core/slang-string-util.h"
+#include "../../source/core/slang-string-escape-util.h"
 
 namespace CppExtract {
 
@@ -10,6 +11,40 @@ namespace CppExtract {
 
 SLANG_FORCE_INLINE static void _indent(Index indentCount, StringBuilder& out) { FileUtil::indent(indentCount, out); }
 
+void Node::dumpMarkup(int indentCount, StringBuilder& out)
+{
+    if (m_markup.getLength() <= 0)
+    {
+        return;
+    }
+
+    List<UnownedStringSlice> lines;
+    StringUtil::calcLines(m_markup.getUnownedSlice(), lines);
+
+    // Remove empty lines from the end
+    while (lines.getCount())
+    {
+        auto lastLine = lines.getLast();
+        if (lastLine.trim().getLength() == 0)
+        {
+            lines.removeLast();
+            continue;
+        }
+        break;
+    }
+
+    if (lines.getCount() == 0)
+    {
+        return;
+    }
+
+    for (auto line : lines)
+    {
+        _indent(indentCount, out);
+        out << "// " << line << "\n";
+    }
+}
+
 ScopeNode* Node::getRootScope()
 {
     if (m_parentScope)
@@ -47,7 +82,7 @@ void Node::calcAbsoluteName(StringBuilder& outName) const
             outName << "::";
         }
 
-        if (node->m_type == Type::AnonymousNamespace)
+        if (node->m_kind == Kind::AnonymousNamespace)
         {
             outName << "{Anonymous}";
         }
@@ -119,7 +154,7 @@ void Node::calcAbsoluteName(StringBuilder& outName) const
     for (Node* node : scope->m_children)
     {
         EnumNode* enumNode = as<EnumNode>(node);
-        if (enumNode && enumNode->m_type == Node::Type::Enum)
+        if (enumNode && enumNode->m_kind == Node::Kind::Enum)
         {
             Node** nodePtr = enumNode->m_childMap.TryGetValue(name);
             if (nodePtr)
@@ -251,7 +286,7 @@ ScopeNode* ScopeNode::getAnonymousNamespace()
 {
     if (!m_anonymousNamespace)
     {
-        m_anonymousNamespace = new ScopeNode(Type::AnonymousNamespace);
+        m_anonymousNamespace = new ScopeNode(Kind::AnonymousNamespace);
         m_anonymousNamespace->m_parentScope = this;
         m_children.add(m_anonymousNamespace);
     }
@@ -263,7 +298,7 @@ void ScopeNode::addChild(Node* child)
 {
     SLANG_ASSERT(child->m_parentScope == nullptr);
     // Can't add anonymous namespace this way - should be added via getAnonymousNamespace
-    SLANG_ASSERT(child->m_type != Type::AnonymousNamespace);
+    SLANG_ASSERT(child->m_kind != Kind::AnonymousNamespace);
 
     child->m_parentScope = this;
     m_children.add(child);
@@ -298,15 +333,17 @@ void ScopeNode::calcScopeDepthFirst(List<Node*>& outNodes)
 
 void ScopeNode::dump(int indentCount, StringBuilder& out)
 {
+    dumpMarkup(indentCount, out);
+
     _indent(indentCount, out);
 
-    switch (m_type)
+    switch (m_kind)
     {
-        case Type::AnonymousNamespace:
+        case Kind::AnonymousNamespace:
         {
             out << "namespace {\n";
         }
-        case Type::Namespace:
+        case Kind::Namespace:
         {
             if (m_name.hasContent())
             {
@@ -331,17 +368,66 @@ void ScopeNode::dump(int indentCount, StringBuilder& out)
 
 /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! EnumCaseNode !!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */
 
+/* Returns true if needs space between the tokens.
+It determines this based on the locs, and if they contain something between them.
+*/
+static bool _needsSpace(const Token& prevTok, const Token& tok)
+{
+    auto prevLoc = prevTok.getLoc();
+    auto loc = tok.getLoc();
+
+    auto prevContent = prevTok.getContent();
+    auto content = tok.getContent();
+
+    if (prevLoc + prevContent.getLength() == loc)
+    {
+        return false;
+    }
+
+    return true;
+}
+
+
+static void _dumpTokens(const Token* toks, Index count, StringBuilder& out)
+{
+    if (count > 0)
+    {
+        out << toks[0].getContent();
+
+        for (Index i = 1; i < count; ++i)
+        {
+            const auto& prevToken = toks[i - 1];
+            const auto& token = toks[i];
+
+            if (_needsSpace(prevToken, token))
+            {
+                out << " ";
+            }
+
+            out << token.getContent();
+        }
+    }
+}
+
+static void _dumpTokens(const List<Token>& toks, StringBuilder& out)
+{
+    _dumpTokens(toks.getBuffer(), toks.getCount(), out);
+}
+
+
 void EnumCaseNode::dump(int indent, StringBuilder& out)
 {
     if (isReflected())
     {
+        dumpMarkup(indent, out);
+
         _indent(indent, out);
         out << m_name.getContent();
 
-        if (m_value.type != TokenType::Invalid)
+        if (m_valueTokens.getCount())
         {
             out << " = ";
-            out << m_value.getContent();
+            _dumpTokens(m_valueTokens, out);
         }
 
         out << ",\n";
@@ -354,16 +440,13 @@ void TypeDefNode::dump(int indent, StringBuilder& out)
 {
     if (isReflected())
     {
+        dumpMarkup(indent, out);
+
         _indent(indent, out);
 
         out << "typedef ";
-
-        for (auto& tok : m_targetTypeTokens)
-        {
-            out << tok.getContent() << " ";
-        }
-
-        out << m_name.getContent() << ";\n";
+        _dumpTokens(m_targetTypeTokens, out);
+        out << " " << m_name.getContent() << ";\n";
     }
 }
 
@@ -376,11 +459,13 @@ void EnumNode::dump(int indent, StringBuilder& out)
         return;
     }
 
+    dumpMarkup(indent, out);
+
     _indent(indent, out);
 
     out << "enum ";
 
-    if (m_type == Type::EnumClass)
+    if (m_kind == Kind::EnumClass)
     {
         out << "class ";
     }
@@ -390,9 +475,10 @@ void EnumNode::dump(int indent, StringBuilder& out)
         out << m_name.getContent();
     }
 
-    if (m_backingToken.type != TokenType::Invalid)
-    {
-        out << " : " << m_backingToken.getContent();
+    if (m_backingTokens.getCount() > 0)
+    { 
+        out << " : ";
+        _dumpTokens(m_backingTokens, out);
     }
 
     out << "\n";
@@ -414,6 +500,8 @@ void FieldNode::dump(int indent, StringBuilder& out)
 {
     if (isReflected())
     {
+        dumpMarkup(indent, out);
+
         _indent(indent, out);
         out << m_fieldType << " " << m_name.getContent() << "\n";
     }
@@ -506,9 +594,11 @@ void ClassLikeNode::getReflectedDerivedTypes(List<ClassLikeNode*>& out) const
 
 void ClassLikeNode::dump(int indentCount, StringBuilder& out)
 {
+    dumpMarkup(indentCount, out);
+
     _indent(indentCount, out);
 
-    const char* typeName = (m_type == Type::StructType) ? "struct" : "class";
+    const char* typeName = (m_kind == Kind::StructType) ? "struct" : "class";
 
     out << typeName << " ";