Misc cleanup and removal. (#357)

Among other, removal of prev_end argument that is not needed. closes
#356

papyri/render.py

@@ -374,10 +374,17 @@ async def _write_index(self, html_dir):
         if html_dir:
             (html_dir / "index.html").write_text(await self.index())
 
-    async def virtual(self, module, node_name: str):
+    async def virtual(self, module: str, node_name: str):
+        """
+        Endpoint that  look for all nodes of a specific type in all
+        the known pages and render them.
+        """
+        _module: Optional[str]
         if module == "*":
-            module = None
-        items = list(self.store.glob((module, None, None, None)))
+            _module = None
+        else:
+            _module = module
+        items = list(self.store.glob((_module, None, None, None)))
 
         visitor = TreeVisitor([getattr(take2, node_name)])
         acc = []
@@ -444,31 +451,37 @@ async def _write_gallery(self, config):
                     config.output_dir / module / version / "gallery" / "index.html"
                 ).write_text(data)
 
-    async def gallery(self, package, version, ext=""):
+    async def gallery(self, package: str, version: str, ext=""):
         # TODO FIX
+        _package: Optional[str]
+        _version: Optional[str]
         if ":" in package:
-            package, _ = package.split(":")
-        if package == version == "*":
-            package = version = None
+            _package, _ = package.split(":")
+        else:
+            _package = package
+        _version = version
+        if _package == "*" and _version == "*":
+            _package = None
+            _version = None
 
         figmap = defaultdict(lambda: [])
         assert isinstance(self.store, GraphStore)
-        if package is not None:
+        if _package is not None:
             meta = encoder.decode(
-                self.store.get_meta(Key(package, version, None, None))
+                self.store.get_meta(Key(_package, _version, None, None))
             )
         else:
             meta = {"logo": None}
         logo = meta["logo"]
-        res = self.store.glob((package, version, "assets", None))
-        backrefs = set()
+        res = self.store.glob((_package, _version, "assets", None))
+        backrefs: set[tuple[str, str, str, str]] = set()
         for key in res:
             brs = {tuple(x) for x in self.store.get_backref(key)}
             backrefs = backrefs.union(brs)
 
-        for key in backrefs:
-            data = encoder.decode(self.store.get(Key(*key)))
-            if "examples" in key:
+        for key2 in backrefs:
+            data = encoder.decode(self.store.get(Key(*key2)))
+            if "examples" in key2:
                 continue
             # TODO: examples can actuallly be just Sections.
             assert isinstance(data, IngestedBlobs)
@@ -477,14 +490,14 @@ async def gallery(self, package, version, ext=""):
             for k in [
                 u.value for u in i.example_section_data if u.__class__.__name__ == "Fig"
             ]:
-                package, v, kind, _path = key
+                package, v, kind, _path = key2
                 # package, filename, link
                 impath = f"{self.prefix}{k.module}/{k.version}/img/{k.path}"
                 link = f"{self.prefix}/{package}/{v}/api/{_path}"
                 # figmap.append((impath, link, name)
                 figmap[package].append((impath, link, _path))
 
-        glist = self.store.glob((package, version, "examples", None))
+        glist = self.store.glob((package, _version, "examples", None))
         for target_key in glist:
             section = encoder.decode(self.store.get(target_key))
 
@@ -504,7 +517,7 @@ class D:
 
         doc = D()
         pap_keys = self.store.glob((None, None, "meta", "papyri.cbor"))
-        parts = {package: []}
+        parts: Any = {package: []}
         for pk in pap_keys:
             mod, ver, kind, identifier = pk
             parts[package].append((RefInfo(mod, ver, "api", mod), mod))
@@ -516,7 +529,7 @@ class D:
             module=package,
             parts_mods=parts.get(package, []),
             parts=list(parts.items()),
-            version=version,
+            version=_version,
             parts_links=defaultdict(lambda: ""),
             doc=doc,
         )
@@ -550,6 +563,10 @@ class D:
         )
 
     def _myst_root(self, doc: IngestedBlobs) -> MRoot:
+        """
+        Convert a internal IngestedBlob document into a MyST tree
+        for rendering.
+        """
         to_suppress = []
         myst_acc: List[Any] = []
         if doc.signature:
@@ -645,8 +662,9 @@ def render_one(
             module = qa.split(".")[0]
             return template.render(
                 current_type=current_type,
-                doc=doc,
                 myst_root=root,
+                item_line=doc.item_line,
+                item_file=doc.item_file,
                 logo=meta.get("logo", None),
                 version=meta["version"],
                 module=module,
@@ -906,7 +924,13 @@ async def _write_narrative_files(self, config):
         for _, (module, version, _, path) in progress(
             narrative, description="Rendering Narrative..."
         ):
-            data = await self._serve_narrative(module, version, path)
+            try:
+                data = await self._serve_narrative(module, version, path)
+            except Exception as e:
+                e.add_note(
+                    f"rendering {module=} {version=} {path=}",
+                )
+                raise
             if config.output_dir:
                 (config.output_dir / module / version / "docs").mkdir(
                     parents=True, exist_ok=True
@@ -1084,7 +1108,24 @@ async def serve_static(path):
 
 
 class Resolver:
-    def __init__(self, store, prefix: str, extension: str):
+    # mapping from package name to existing (version(s))
+    # currently multiple version nor really supported.
+    # this is used when we request to reach to a page from any version of a
+    # library to know which one we should look into.
+    version: Dict[str, str]
+
+    # the rendering might under prefix, depending on how it is hosted.
+    # This should thus be prepended to generated urls.
+    # it should end and start with a `/`.
+    prefix: str
+
+    # Extension that need to be added to the end of the url, why many server
+    # are fine omitting `.html`, it might be necessary when resolving for
+    # statically generated contents.
+
+    extension: str
+
+    def __init__(self, store, prefix: str, extension: str) -> None:
         """
         Given a RefInfo to an object, resolve it to a full http-link
         with the current configuration.
@@ -1110,42 +1151,67 @@ def __init__(self, store, prefix: str, extension: str):
         self.prefix = prefix
         self.extension = extension
 
-        self.version: Dict[str, str] = {}
+        self.version = {}
 
         for p, v in {
             (package, version)
             for (package, version, _, _) in self.store.glob((None, "*", "meta", None))
         }:
             if p in self.version:
                 pass
-                # todo, likely parse version here if possible.
+                # TODO:, likely parse version here if possible.
                 # maxv = max(v, self.version[p])
                 # print("multiple version for package", p, "Trying most recent", maxv)
             self.version[p] = v
 
-    def exists_resolve(self, info) -> Tuple[bool, Optional[str]]:
+    def exists_resolve(self, info: RefInfo) -> Tuple[bool, Optional[str]]:
+        """Resolve a RefInfo to a URL, additionally return wether the link item exists
+
+        Return
+        ------
+        exists: boolean
+            Whether  the target document exists
+        url : str|None
+            If exists, URL where target document can be found
+
+        """
         module, version_number, kind, path = info
         if kind == "api":
             kind = "module"
         if version_number == "*":
             if info.module in self.version:
                 version_number = self.version[info.module]
 
-        i2 = RefInfo(module, version_number, kind, path)
-        # TODO: Fix
+        query_ref = RefInfo(module, version_number, kind, path)
+        # TODO: Fix, for example in IPython narrative docs the
+        # toctree point to ``pr/*`` which is a subfolder we don't support yet:
+        #
+        # .. toctree::
+        #     :maxdepth: 2
+        #     :glob:
+        #
+        #     pr/*
         if kind == "?":
             return False, None
-        sgi = self.store.glob(i2)
+        sgi = self.store.glob(query_ref)
         if sgi:
-            exists, url = self._resolve(i2)
+            assert len(sgi) == 1, (
+                "we have no reason to have more than one reference",
+                sgi,
+            )
+            exists, url = self._resolve(query_ref)
             return exists, url
 
         # we may want to find older versions.
 
         return False, None
 
-    def resolve(self, info) -> str:
-        return self._resolve(info)[1]
+    def resolve(self, info: RefInfo) -> Optional[str]:
+        exists, url = self.exists_resolve(info)
+        # TODO: this is moslty used to render navigation we should make sure that
+        # links are resolved and exists before rendering.
+        # assert exists, f"{info=} doe not exists"
+        return url
 
     def _resolve(self, info) -> Tuple[bool, str]:
         """
@@ -1157,8 +1223,9 @@ def _resolve(self, info) -> Tuple[bool, str]:
             "api",
             "examples",
             "assets",
-            "?",
+            # "?",
             "docs",
+            # TODO:
             "to-resolve",
         ), repr(info)
         # assume same package/version for now.

papyri/templates/html.tpl.j2

@@ -51,14 +51,11 @@ they belong to, and scaled with the number of references pointing them</p>
 <hr>
 
 {%if meta.get('github_slug')%}
-    GitHub : <a class='external' href='https://github.com/{{meta.github_slug}}/blob/{{meta.tag}}/{{doc.item_file}}#L{{doc.item_line}}'>{{doc.item_file}}#{{doc.item_line}}</a>
+    GitHub : <a class='external' href='https://github.com/{{meta.github_slug}}/blob/{{meta.tag}}/{{item_file}}#L{{item_line}}'>{{item_file}}#{{item_line}}</a>
 {% else %}
-    File: {{doc.item_file}}#{{doc.item_line}}
+    File: {{item_file}}#{{item_line}}
 {%endif%}<br/>
 
-type: {{doc.item_type}} <br/>
-Commit: <br/>
-
 {% if graph is defined %}
     {{d3script(graph)}}
 {% endif %}

papyri/tree.py

@@ -607,34 +607,34 @@ def _toctree_handler(self, argument, options, content):
         for line in content.splitlines():
             line = line.strip()
             if line == "self":
-                # TODO
+                # TODO in toctree one line can refer to self
                 continue
             if "<" in line and line.endswith(">"):
-                title, link = line[:-1].split("<")
+                title, url = line[:-1].split("<")
                 title = title.strip()
-                assert "<" not in link
-                toc.append([title, link])
-                l = Link(
+                assert "<" not in url
+                toc.append([title, url])
+                link = Link(
                     title,
-                    reference=RefInfo(module="", version="", kind="?", path=link),
+                    reference=RefInfo(module="", version="", kind="?", path=url),
                     kind="exists",
                     exists=True,
                     anchor=None,
                 )
-                RESOLVER.add_reference(l, link)
-                lls.append(l)
+                RESOLVER.add_reference(link, url)
+                lls.append(link)
             else:
                 assert "<" not in line
                 toc.append([None, line])
-                l = Link(
+                link = Link(
                     line,
                     reference=RefInfo(module="", version="", kind="?", path=line),
                     kind="exists",
                     exists=True,
                     anchor=None,
                 )
-                RESOLVER.add_reference(l, line)
-                lls.append(l)
+                RESOLVER.add_reference(link, line)
+                lls.append(link)
 
         self._tocs.append(toc)
 
@@ -660,7 +660,7 @@ def replace_MMystDirective(self, myst_directive: MMystDirective):
         return [myst_directive]
 
     def replace_BlockDirective(self, block_directive: MMystDirective):
-        assert False, "we shoudl never reach there"
+        assert False, "we should never reach there"
 
     def _resolve(self, loc, text):
         """