pulp · pedro-psb · Jun 10, 2024 · Jun 7, 2024
diff --git a/pyproject.toml b/pyproject.toml
@@ -12,6 +12,7 @@ dependencies = [
     "mkdocs-macros-plugin",
     "mkdocs-site-urls",
     "mkdocs-literate-nav",
+    "bs4",
     "importlib_resources",
     "httpx",
     "rich",

diff --git a/src/pulp_docs/mkdocs_hooks.py b/src/pulp_docs/mkdocs_hooks.py
@@ -3,6 +3,7 @@
 
 See: https://www.mkdocs.org/user-guide/configuration/#hooks
 """
+from bs4 import BeautifulSoup as bs
 
 
 def on_serve(server, config, builder):
@@ -16,3 +17,43 @@ def on_serve(server, config, builder):
     server.unwatch(tmpdir)
     server.unwatch(mkdocs_yml)
     return server
+
+
+REDOC_HEADER = """
+<link href="https://fonts.googleapis.com/css?family=Montserrat:300,400,700|Roboto:300,400,700" rel="stylesheet">
+<style>
+  body {
+    margin: 0;
+    padding: 0;
+  }
+</style>
+"""
+
+REDOC_TAG_TEMPLATE = """
+<redoc spec-url='%s'></redoc>
+"""
+
+REDOC_SCRIPT = """
+<script src="https://cdn.jsdelivr.net/npm/redoc@next/bundles/redoc.standalone.js"> </script>
+"""
+
+
+def on_post_page(output, page, config):
+    if basepath := page.meta.get("restapi_json_file"):
+        redoc_tag = REDOC_TAG_TEMPLATE % basepath
+        bs_page = bs(output, "html.parser")
+
+        # Append <head>scripts
+        bs_page.html.head.append(bs(REDOC_HEADER, "html.parser"))
+
+        # Replace main content-container with <redoc> tag
+        main_container = bs_page.find_all("div", class_="md-main__inner")[0]
+        main_container.replace_with(bs(redoc_tag, "html.parser"))
+
+        # Append <script> tag at the end of body
+        bs_page.html.body.append(bs(REDOC_SCRIPT, "html.parser"))
+
+        # Remove footer (looks weird)
+        footer = bs_page.find_all(class_="md-footer")[0]
+        footer.decompose()
+        return str(bs_page)
diff --git a/src/pulp_docs/mkdocs_macros.py b/src/pulp_docs/mkdocs_macros.py
@@ -17,12 +17,12 @@
 
 import json
 import logging
-import os
 import shutil
 import tempfile
 import time
 from pathlib import Path
 
+import httpx
 import rich
 
 from pulp_docs.cli import Config
@@ -89,6 +89,7 @@ def prepare_repositories(TMPDIR: Path, repos: Repos, config: Config):
     # Download/copy source code to tmpdir
     repo_sources = TMPDIR / "repo_sources"
     repo_docs = TMPDIR / "repo_docs"
+    api_src_dir = TMPDIR / "api_json"
     shutil.rmtree(repo_sources, ignore_errors=True)
     shutil.rmtree(repo_docs, ignore_errors=True)
 
@@ -102,10 +103,13 @@ def prepare_repositories(TMPDIR: Path, repos: Repos, config: Config):
         else:
             this_src_dir = repo_sources / repo_or_pkg.subpackage_of / repo_or_pkg.name
 
+        # restapi
+        if repo_or_pkg.type in ("content", "core"):
+            _download_api_json(api_src_dir, repo_or_pkg.name)
+            _generate_rest_api_page(this_src_dir, repo_or_pkg.name, repo_or_pkg.title)
+
         # install and post-process
-        _place_doc_files(this_src_dir, this_docs_dir, repo_or_pkg)
-        if repo_or_pkg.type == "content":
-            _generate_rest_api_page(this_docs_dir, repo_or_pkg.name, repo_or_pkg.title)
+        _place_doc_files(this_src_dir, this_docs_dir, repo_or_pkg, api_src_dir)
 
         end = time.perf_counter()
         log.info(f"{repo_or_pkg.name} completed in {end - start:.2} sec")
@@ -124,7 +128,31 @@ def prepare_repositories(TMPDIR: Path, repos: Repos, config: Config):
     return (repo_docs, repo_sources)
 
 
-def _place_doc_files(src_dir: Path, docs_dir: Path, repo: Repo):
+def _download_api_json(api_dir: Path, repo_name: str):
+    api_json_path = api_dir / f"{repo_name}/api.json"
+    if api_json_path.exists():
+        log.info(f"{repo_name} api.json already downloaded.")
+        return
+
+    log.info(f"Downloading api.json for {repo_name}")
+    api_url_1 = "https://docs.pulpproject.org/{repo_name}/api.json"
+    api_url_2 = "https://docs.pulpproject.org/{repo_name}/_static/api.json"
+    response = httpx.get(api_url_1.format(repo_name=repo_name))
+    if response.is_error:
+        response = httpx.get(api_url_2.format(repo_name=repo_name))
+    if response.is_error:
+        raise Exception("Couldnt get rest api page")
+
+    # Schema overrides for better display
+    json_file_content = response.json()
+    json_file_content["info"]["title"] = f"{repo_name} API"
+
+    api_json_path.parent.mkdir(parents=True, exist_ok=True)
+    api_json_path.write_text(json.dumps(json_file_content))
+    log.info("Done.")
+
+
+def _place_doc_files(src_dir: Path, docs_dir: Path, repo: Repo, api_src_dir: Path):
     """
     Copy only doc-related files from src_dir to doc_dir.
 
@@ -148,6 +176,14 @@ def _place_doc_files(src_dir: Path, docs_dir: Path, repo: Repo):
         Path(docs_dir / "docs").mkdir(parents=True)
         repo.status.has_staging_docs = False
 
+    # Setup rest Api
+    if repo.type in ("content", "core"):
+        api_json = api_src_dir / f"{repo.name}/api.json"
+        api_md_page = src_dir / "restapi.md"
+        # breakpoint()
+        shutil.copy(api_json, docs_dir / "api.json")
+        shutil.copy(api_md_page, docs_dir / "restapi.md")
+
     # Get changelog
     repo.status.has_changelog = False
     changes_dir = Path(docs_dir)
@@ -175,15 +211,20 @@ def _place_doc_files(src_dir: Path, docs_dir: Path, repo: Repo):
     )
 
 
+RESTAPI_TEMPLATE = """\
+---
+restapi_json_file: "../api.json"
+---
+"""
+
+
 def _generate_rest_api_page(docs_dir: Path, repo_name: str, repo_title: str):
     """Create page that contain a link to the rest api, based on the project url template"""
     log.info("Generating REST_API page")
-    rest_api_page = docs_dir / "docs" / "rest_api.md"
-    rest_api_page.touch()
-    restapi_url = RESTAPI_URL_TEMPLATE.format(repo_name)
-    md_title = f"# {repo_title} REST Api"
-    md_body = f"[{restapi_url}]({restapi_url})"
-    rest_api_page.write_text(f"{md_title}\n\n{md_body}")
+    rest_api_page = docs_dir / "restapi.md"
+    rest_api_page.parent.mkdir(parents=True, exist_ok=True)
+    # rest_api_page.write_text(RESTAPI_TEMPLATE.format(repo_title=repo_title))
+    rest_api_page.write_text(RESTAPI_TEMPLATE)
 
 
 def print_user_repo(repos: Repos, config: Config):
@@ -316,13 +357,23 @@ def get_repos(repo_type="content"):
         repos_list = sorted(
             repos.get_repos(repo_types=_repo_type), key=lambda x: x.title
         )
+        MD_LINK = "[{title}](site:{path})"
+        GITHUB_LINK = (
+            "<a href='https://github.com/{owner}/{name}' target='blank'>{title}</a>"
+        )
         repos_data = [
             {
                 "title": repo.title,
                 "version": "3.12.1",
-                "rest_api_url": f"https://docs.pulpproject.org/{repo.name}/restapi.html",
-                "codebase_url": f"https://github.com/{repo.owner}/{repo.name}",
-                "changelog_url": f"site:{repo.name}/changes/",
+                "restapi_link": MD_LINK.format(
+                    title="REST API", path=f"{repo.name}/restapi/"
+                ),
+                "changes_link": MD_LINK.format(
+                    title="Changelog", path=f"{repo.name}/changes/"
+                ),
+                "codebase_link": GITHUB_LINK.format(
+                    title="Codebase", owner=repo.owner, name=repo.name
+                ),
             }
             for repo in repos_list
         ]

diff --git a/src/pulp_docs/navigation.py b/src/pulp_docs/navigation.py
@@ -77,13 +77,6 @@ def grouped_by_persona(tmpdir: Path, repos: Repos):
         {"Overview": f"{SECTION_HOST}/docs/sections/help/index.md"},
         {"Community": f"{SECTION_HOST}/docs/sections/help/community/"},
         {"More": f"{SECTION_HOST}/docs/sections/help/more/"},
-        {
-            "Changelogs": [
-                {"Core": f.changes_grouping(CHANGES_PATH, repo_types=["core"])},
-                {"Plugins": f.changes_grouping(CHANGES_PATH, repo_types=["content"])},
-                {"Extra": f.changes_grouping(CHANGES_PATH, repo_types=["other"])},
-            ]
-        },
     ]
 
     # Main Section

diff --git a/src/pulp_docs/repository.py b/src/pulp_docs/repository.py
@@ -6,14 +6,14 @@
 
 from __future__ import annotations
 
+import json
 import logging
 import os
 import shutil
 import subprocess
 import tarfile
 import tempfile
 import typing as t
-from collections import ChainMap, defaultdict
 from dataclasses import dataclass, field
 from io import BytesIO
 from pathlib import Path
@@ -116,13 +116,13 @@ def download(self, dest_dir: Path, clear_cache: bool = False) -> str:
         # from remote
         elif not cached_repo.exists():
             log_header = "Downloading from remote"
+            src_copy_path = DOWNLOAD_CACHE_DIR / self.name
             download_from = download_from_gh_main(
-                DOWNLOAD_CACHE_DIR / self.name,
+                src_copy_path,
                 self.owner,
                 self.name,
                 self.branch_in_use,
             )
-            src_copy_path = DOWNLOAD_CACHE_DIR / self.name
 
         # copy from source/cache to pulp-docs workdir
         log.info(f"{log_header}: source={download_from}, copied_from={src_copy_path}")
@@ -133,6 +133,7 @@ def download(self, dest_dir: Path, clear_cache: bool = False) -> str:
             src_copy_path,
             dest_dir,
             ignore=shutil.ignore_patterns(*ignore_patterns),
+            dirs_exist_ok=True,
         )
 
         self.status.download_source = str(download_from)

diff --git a/src/pulp_docs/utils/aggregation.py b/src/pulp_docs/utils/aggregation.py
@@ -85,17 +85,26 @@ def repo_grouping(
             if index_path.exists():
                 repo_nav.append({"Overview": str(index_path.relative_to(self.tmpdir))})
 
+            # Add content type for a repo (guides,tutorials,etc)
             for content_type in selected_content:
-                # Get repo files from content-type and persona
                 lookup_path = self._parse_template_str(
                     template_str, repo.name, content_type
                 )
                 _repo_content = self._add_literate_nav_dir(lookup_path)
 
-                # Prevent rendering content-type section if there are no files
-                if _repo_content:
+                if _repo_content:  # No content section if there are no files
                     content_type_title = Names.get(content_type)
                     repo_nav.append({content_type_title: _repo_content})  # type: ignore
+
+            # Add changelog and restapi
+            if "/user/" in template_str:
+                CHANGES_PATH = f"{repo.name}/changes.md"
+                RESTAPI_PATH = f"{repo.name}/restapi.md"
+                if repo.type in ("content", "core"):
+                    repo_nav.append({"REST API": RESTAPI_PATH})
+                repo_nav.append({"Changelog": CHANGES_PATH})
+
+            # Add navigation to Repo, if one exsits
             if repo_nav:
                 group_nav.append({repo.title: repo_nav})
         return group_nav or ["#"]
@@ -158,4 +167,3 @@ def _parse_template_str(
             kwargs["content"] = content_type
 
         return self.tmpdir / template_str.format(**kwargs)
-
diff --git a/staging_docs/sections/help/index.md b/staging_docs/sections/help/index.md
@@ -11,18 +11,18 @@ Don't hesistate to contact us!
 
 ---
 
-## Quick Links
+## Quick Links (WIP)
 
 {% for repo_type in ("core", "content") %}
-Repo | Version | Rest API | Github Page | Changelog
---- | --- | --- | --- | --- 
+Repo | Version | Links | &nbsp; | &nbsp;
+--- | --- | --- | --- | ---
 {% for repo in get_repos(repo_type) -%}
-{{ repo.title }} | `{{ repo.version }}` | <a href="{{ repo.rest_api_url}}" target="_blank">:link:</a> | [:link:]({{ repo.codebase_url }}) | [:link:]({{ repo.changelog_url }})
+{{ repo.title }} | `{{ repo.version }}` | {{ repo.restapi_link }} | {{ repo.codebase_link }} | {{ repo.changes_link }}
 {% endfor %}
 {% endfor %}
 
-Repo | Version | Code (Github) | Changelog
---- | --- | --- | --- 
+Repo | Version | Links | &nbsp;
+--- | --- | --- | ---
 {% for repo in get_repos("other") -%}
-{{ repo.title }} | `{{ repo.version }}` | [:link:]({{ repo.codebase_url }}) | [:link:]({{ repo.changelog_url }})
+{{ repo.title }} | `{{ repo.version }}` | {{ repo.codebase_link }} | {{ repo.changes_link }}
 {% endfor %}