auto-generate toctree from flytesnacks index.md docs (#4587)

* auto-generate toctree from flytesnacks index.md docs

Signed-off-by: Niels Bantilan <[email protected]>

* fix environment_setup ref

Signed-off-by: Niels Bantilan <[email protected]>

* make ListTableToc more generic

Signed-off-by: Niels Bantilan <[email protected]>

---------

Signed-off-by: Niels Bantilan <[email protected]>

docs/_ext/import_projects.py

@@ -1,3 +1,4 @@
+import inspect
 import os
 import re
 import shutil
@@ -9,8 +10,11 @@
 from typing import Optional, List, Union
 
 from git import Repo
+from docutils import nodes
+from docutils.statemachine import StringList, string2lines
 from sphinx.application import Sphinx
 from sphinx.config import Config
+from sphinx.util.docutils import SphinxDirective
 
 __version__ = "0.0.0"
 
@@ -20,6 +24,7 @@ class ImportProjectsConfig:
     clone_dir: str
     flytekit_api_dir: str
     source_regex_mapping: dict = field(default_factory=dict)
+    list_table_toc: List[str] = field(default_factory=list)
 
 
 @dataclass
@@ -32,6 +37,46 @@ class Project:
     refresh: bool = False
 
 
+TOC_TEMPLATE = """
+```{{toctree}}
+:maxdepth: 1
+:hidden:
+{toc}
+```
+"""
+
+TABLE_TEMPLATE = """
+```{{list-table}}
+{content}
+```
+"""
+
+
+class ListTableToc(SphinxDirective):
+    """Custom directive to convert list-table into both list-table and toctree."""
+
+    has_content = True
+
+    def run(self) -> list:
+        return [self.parse()]
+
+    def parse(self):
+        """Parses the list-table and adds a toctree"""
+        toc = ""
+
+        # finds all doc references in the form <doc/path>
+        for doc in re.findall(r"<(.+)>", self.block_text):
+            toc += f"\n{doc}"
+
+        container = nodes.container("")
+        toc = inspect.cleandoc(TOC_TEMPLATE.format(toc=toc))
+        table = inspect.cleandoc(TABLE_TEMPLATE.format(content=self.block_text))
+        content = f"{toc}\n\n{table}"
+
+        self.state.nested_parse(StringList(string2lines(content)), 0, container)
+        return container
+
+
 def update_sys_path_for_flytekit(import_project_config: ImportProjectsConfig):
     # create flytekit/_version.py file manually
     with open(f"{import_project_config.flytekit_api_dir}/flytekit/_version.py", "w") as f:
@@ -132,10 +177,24 @@ def replace_refs_in_docstrings(
         lines[i] = text
 
 
+def add_list_table_toc(app: Sphinx, docname: str, source: List[str]):
+    """This replaces list-table directives in specific documents with list-table-toc.
+
+    This is important for automatically generating a toctree and list-table from
+    a list-table.
+    """
+    if docname in app.config.import_projects_config["list_table_toc"]:
+        text = source[0]
+        text = re.sub(r"{list-table}", r"{list-table-toc}", text)
+        source[0] = text
+
+
 def setup(app: Sphinx) -> dict:
     app.add_config_value("import_projects_config", None, False)
     app.add_config_value("import_projects", None, False)
+    app.add_directive("list-table-toc", ListTableToc)
     app.connect("config-inited", import_projects, priority=0)
+    app.connect("source-read", add_list_table_toc, priority=0)
 
     return {
         "version": __version__,

docs/conf.py

@@ -128,10 +128,7 @@
     "flytesnacks/bioinformatics_examples.md",
     "flytesnacks/feature_engineering.md",
     "flytesnacks/flyte_lab.md",
-    "flytesnacks/integrations.md",
     "flytesnacks/ml_training.md",
-    "flytesnacks/tutorials.md",
-    "flytesnacks/userguide.md",
     "flytesnacks/README.md",
     "flytekit/**/README.md",
     "flytekit/_templates/**",
@@ -313,8 +310,10 @@
 # These patterns are used to replace values in source files that are imported
 # from other repos.
 REPLACE_PATTERNS = {
+    r"<flyte:deployment/index>": r"</deployment/index>",
     r"<flytectl:index>": r"</flytectl_overview>",
     INTERSPHINX_REFS_PATTERN: INTERSPHINX_REFS_REPLACE,
+    r"<auto_examples": r"<flytesnacks/examples",
     r"<protos/docs/core/core:taskmetadata>": r"<ref_flyteidl.core.TaskMetadata>",
     r"<protos/docs/core/core:tasktemplate>": r"<ref_flyteidl.core.TaskTemplate>",
     r"<flytesnacks/examples": r"</flytesnacks/examples",
@@ -325,12 +324,19 @@
     r"{ref}`bioinformatics <bioinformatics>`": r"bioinformatics",
     PROTO_REF_PATTERN: PROTO_REF_REPLACE,
     r"/protos/docs/service/index": r"/protos/docs/service/service",
+    r"<weather_forecasting>": r"</flytesnacks/weather_forecasting>",
+    r"<environment_setup>": r"</flytesnacks/environment_setup>"
 }
 
 import_projects_config = {
     "clone_dir": "_projects",
     "flytekit_api_dir": "_src/flytekit/",
     "source_regex_mapping": REPLACE_PATTERNS,
+    "list_table_toc": [
+        "flytesnacks/userguide",
+        "flytesnacks/tutorials",
+        "flytesnacks/integrations",
+    ],
 }
 
 # Define these environment variables to use local copies of the projects. This

docs/index.md

@@ -79,12 +79,12 @@ contribute its architecture and design. You can also access the
 * - {doc}`🔤 Intro to Flyte <introduction>`
   - Get your first workflow running, learn about the Flyte development lifecycle
     and core use cases.
-* - {doc}`📖 User Guide <userguide>`
+* - {doc}`📖 User Guide <flytesnacks/userguide>`
   - A comprehensive view of Flyte's functionality for data and ML practitioners.
-* - {doc}`📚 Tutorials <tutorials>`
+* - {doc}`📚 Tutorials <flytesnacks/tutorials>`
   - End-to-end examples of Flyte for data/feature engineering, machine learning,
     bioinformatics, and more.
-* - {doc}`🔌 Integrations <integrations>`
+* - {doc}`🔌 Integrations <flytesnacks/integrations>`
   - Leverage a rich ecosystem of third-party tools and libraries to make your
     Flyte workflows even more effective.
 * - {ref}`🚀 Deployment Guide <deployment>`
@@ -145,9 +145,9 @@ Core Use Cases <flytesnacks/getting_started/core_use_cases>
 :name: examples-guides
 :hidden:
 
-User Guide <userguide>
-Tutorials <tutorials>
-Integrations <integrations>
+User Guide <flytesnacks/userguide>
+Tutorials <flytesnacks/tutorials>
+Integrations <flytesnacks/integrations>
 ```
 
 ```{toctree}