Modify markdown writing to produce readthedocs pages (#6904)

This commit makes multple changes to the slang doc markdown writer to make the stdlib reference pages usable with readthedocs.
- Adds tables of contents at the bottom of every page with children. TOCs contain category landing pages where applicable and contain uncategorized child pages
- Changes links on the pages to use relative paths instead of absolute paths
- Changes anchor tags to use HTML syntax instead of markdown syntax

The overall TOC on readthedocs will look the same as the TOC seen on the existing docs website, and I have verfified that links and anchors will still work.

The tables of contents generated for use on readthedocs would be visible on the existing docs website.
This change wraps the TOCs in a comment block, so that it will be hidden from view.
The readthedocs build script can filter out the comments to unhide the TOC inside and still properly render it there with it remaining hidden when viewed elsewhere.

---------

Co-authored-by: slangbot <186143334+slangbot@users.noreply.github.com>

1 files changed, 82 insertions, 21 deletions

diff --git a/source/slang/slang-doc-markdown-writer.cpp b/source/slang/slang-doc-markdown-writer.cpp
index df8a8bfb8..ae0ddc1a5 100644
--- a/source/slang/slang-doc-markdown-writer.cpp
+++ b/source/slang/slang-doc-markdown-writer.cpp
@@ -100,6 +100,15 @@ String getDocPath(const DocumentationConfig& config, String path)
     return config.rootDir + Path::getPathWithoutExt(path);
 }
 
+String getTocTreeEntry(const String& name, const String& fromPath, const String& toPath)
+{
+    StringBuilder sb;
+    // Format: name <path>
+    sb << name;
+    sb << " <" << Path::getPathWithoutExt(Path::getRelativePath(fromPath, toPath)) << ">\n";
+    return sb.produceString();
+}
+
 void DocMarkdownWriter::_appendAsBullets(
     const List<NameAndText>& values,
     bool insertLinkForName,
@@ -109,6 +118,18 @@ void DocMarkdownWriter::_appendAsBullets(
     for (const auto& value : values)
     {
         out << "#### ";
+        if (value.decl)
+        {
+            // Add anchor ID for the decl.
+            if (as<GenericTypeParamDeclBase>(value.decl))
+            {
+                out << " <a id=\"typeparam-" << getText(value.decl->getName()) << "\"></a>";
+            }
+            else
+            {
+                out << " <a id=\"decl-" << getText(value.decl->getName()) << "\"></a>";
+            }
+        }
         const String& name = value.name;
         auto path = findLinkForToken(m_currentPage, name);
         if (name.getLength())
@@ -124,7 +145,7 @@ void DocMarkdownWriter::_appendAsBullets(
                 out.appendChar(wrapChar);
                 if (path.getLength())
                 {
-                    out << "](" << getDocPath(m_config, path) << ")";
+                    out << "](" << Path::getPathWithoutExt(path) << ")";
                 }
             }
             else
@@ -146,18 +167,6 @@ void DocMarkdownWriter::_appendAsBullets(
                 }
             }
         }
-        if (value.decl)
-        {
-            // Add anchor ID for the decl.
-            if (as<GenericTypeParamDeclBase>(value.decl))
-            {
-                out << " {#typeparam-" << getText(value.decl->getName()) << "}";
-            }
-            else
-            {
-                out << " {#decl-" << getText(value.decl->getName()) << "}";
-            }
-        }
         if (value.text.getLength())
         {
             out.appendChar('\n');
@@ -208,7 +217,10 @@ void DocMarkdownWriter::_appendAsBullets(const List<String>& values, char wrapCh
         }
         if (path.getLength())
         {
-            out << "](" << getDocPath(m_config, path) << ")";
+            out << "]("
+                << Path::getPathWithoutExt(
+                       Path::getRelativePath(Path::getParentDirectory(m_currentPage->path), path))
+                << ")";
         }
         out << "\n";
     }
@@ -1926,7 +1938,7 @@ void DocMarkdownWriter::writeAggType(
 
         if (assocTypeDecls.getCount())
         {
-            out << toSlice("# Associated types\n\n");
+            out << toSlice("## Associated types\n\n");
 
             for (AssocTypeDecl* assocTypeDecl : assocTypeDecls)
             {
@@ -1977,7 +1989,7 @@ void DocMarkdownWriter::writeAggType(
         _getDeclsOfType<PropertyDecl>(this, page, properties);
         if (properties.getCount())
         {
-            out << toSlice("## Properties\n\n");
+            out << toSlice("## m_currentPage->path\n\n");
             _appendAsBullets(_getAsNameAndTextList(properties), true, 0);
             out << toSlice("\n");
         }
@@ -2151,7 +2163,9 @@ String DocMarkdownWriter::translateToMarkdownWithLinks(String text, bool strictC
                 sb.append("[");
                 sb << escapeMarkdownText(tokenContent.getUnownedSlice());
                 sb.append("](");
-                sb.append(getDocPath(m_config, page->path));
+                sb.append(Path::getPathWithoutExt(Path::getRelativePath(
+                    Path::getParentDirectory(m_currentPage->path),
+                    page->path)));
                 if (sectionName.getLength())
                     sb << "#" << sectionName;
                 sb.append(")");
@@ -2239,7 +2253,10 @@ String DocMarkdownWriter::translateToHTMLWithLinks(Decl* decl, String text)
             if (page)
             {
                 sb.append("<a href=\"");
-                sb.append(getDocPath(m_config, page->path));
+                sb.append(Path::getPathWithoutExt(Path::getRelativePath(
+                    Path::getParentDirectory(m_currentPage->path),
+                    page->path)));
+                sb.append(".html");
                 if (sectionName.getLength())
                     sb << "#" << sectionName;
                 sb.append("\"");
@@ -2698,7 +2715,9 @@ void DocMarkdownWriter::generateSectionIndexPage(DocumentPage* page)
     for (auto child : page->children)
     {
         sb << "- [" << escapeMarkdownText(child->shortName) << "]("
-           << getDocPath(m_config, child->path) << ")\n";
+           << Path::getPathWithoutExt(
+                  Path::getRelativePath(Path::getParentDirectory(page->path), child->path))
+           << ")\n";
     }
 }
 
@@ -2872,6 +2891,34 @@ void writeTOCChildren(
     categoryNames.sort();
     auto parentPath = Path::getParentDirectory(page->path);
     parentPath.append("/");
+
+    // Create toctree for index pages
+    if (page->path.endsWith("index.md"))
+    {
+        StringBuilder& tocSB = page->get();
+        tocSB << "\n<!-- RTD-TOC-START\n";
+        tocSB << "```{toctree}\n:titlesonly:\n:hidden:\n\n";
+
+        // Add category landing pages to the toctree
+        for (auto& cat : categoryNames)
+        {
+            // Skip non-categorized pages
+            if (cat.getLength() == 0)
+                continue;
+
+            String landingPagePath = parentPath + cat + ".md";
+            tocSB << getTocTreeEntry(writer->m_categories[cat], parentPath, landingPagePath);
+        }
+
+        // Add uncategorized pages to the toctree
+        for (auto child : categories[""])
+        {
+            tocSB << getTocTreeEntry(child->shortName, parentPath, child->path);
+        }
+        tocSB << "```\n";
+        tocSB << "RTD-TOC-END -->\n";
+    }
+
     for (auto& cat : categoryNames)
     {
         // Skip non-categorized pages first.
@@ -2900,9 +2947,23 @@ void writeTOCChildren(
                                << "\n\nThis category contains the following declarations:\n\n";
         for (auto child : categories[cat])
         {
-            landingPage->contentSB << "#### [" << writer->escapeMarkdownText(child->title) << "]("
-                                   << getDocPath(config, child->path) << ")\n\n";
+            landingPage->contentSB
+                << "#### [" << writer->escapeMarkdownText(child->title) << "]("
+                << Path::getPathWithoutExt(
+                       Path::getRelativePath(Path::getParentDirectory(page->path), child->path))
+                << ")\n\n";
         }
+
+        // Add the toctree for the category landing page.
+        landingPage->contentSB << "\n<!-- RTD-TOC-START\n";
+        landingPage->contentSB << "```{toctree}\n:titlesonly:\n:hidden:\n\n";
+        for (auto child : categories[cat])
+        {
+            landingPage->contentSB << getTocTreeEntry(child->shortName, parentPath, child->path);
+        }
+        landingPage->contentSB << "```\n";
+        landingPage->contentSB << "RTD-TOC-END -->\n";
+
         page->children.add(landingPage);
     }