diff --git a/papyri/render.py b/papyri/render.py
index 7f6dab3a..a9e4a23e 100644
--- a/papyri/render.py
+++ b/papyri/render.py
@@ -374,10 +374,17 @@ async def _write_index(self, html_dir):
if html_dir:
(html_dir / "index.html").write_text(await self.index())
- async def virtual(self, module, node_name: str):
+ async def virtual(self, module: str, node_name: str):
+ """
+ Endpoint that look for all nodes of a specific type in all
+ the known pages and render them.
+ """
+ _module: Optional[str]
if module == "*":
- module = None
- items = list(self.store.glob((module, None, None, None)))
+ _module = None
+ else:
+ _module = module
+ items = list(self.store.glob((_module, None, None, None)))
visitor = TreeVisitor([getattr(take2, node_name)])
acc = []
@@ -444,31 +451,37 @@ async def _write_gallery(self, config):
config.output_dir / module / version / "gallery" / "index.html"
).write_text(data)
- async def gallery(self, package, version, ext=""):
+ async def gallery(self, package: str, version: str, ext=""):
# TODO FIX
+ _package: Optional[str]
+ _version: Optional[str]
if ":" in package:
- package, _ = package.split(":")
- if package == version == "*":
- package = version = None
+ _package, _ = package.split(":")
+ else:
+ _package = package
+ _version = version
+ if _package == "*" and _version == "*":
+ _package = None
+ _version = None
figmap = defaultdict(lambda: [])
assert isinstance(self.store, GraphStore)
- if package is not None:
+ if _package is not None:
meta = encoder.decode(
- self.store.get_meta(Key(package, version, None, None))
+ self.store.get_meta(Key(_package, _version, None, None))
)
else:
meta = {"logo": None}
logo = meta["logo"]
- res = self.store.glob((package, version, "assets", None))
- backrefs = set()
+ res = self.store.glob((_package, _version, "assets", None))
+ backrefs: set[tuple[str, str, str, str]] = set()
for key in res:
brs = {tuple(x) for x in self.store.get_backref(key)}
backrefs = backrefs.union(brs)
- for key in backrefs:
- data = encoder.decode(self.store.get(Key(*key)))
- if "examples" in key:
+ for key2 in backrefs:
+ data = encoder.decode(self.store.get(Key(*key2)))
+ if "examples" in key2:
continue
# TODO: examples can actuallly be just Sections.
assert isinstance(data, IngestedBlobs)
@@ -477,14 +490,14 @@ async def gallery(self, package, version, ext=""):
for k in [
u.value for u in i.example_section_data if u.__class__.__name__ == "Fig"
]:
- package, v, kind, _path = key
+ package, v, kind, _path = key2
# package, filename, link
impath = f"{self.prefix}{k.module}/{k.version}/img/{k.path}"
link = f"{self.prefix}/{package}/{v}/api/{_path}"
# figmap.append((impath, link, name)
figmap[package].append((impath, link, _path))
- glist = self.store.glob((package, version, "examples", None))
+ glist = self.store.glob((package, _version, "examples", None))
for target_key in glist:
section = encoder.decode(self.store.get(target_key))
@@ -504,7 +517,7 @@ class D:
doc = D()
pap_keys = self.store.glob((None, None, "meta", "papyri.cbor"))
- parts = {package: []}
+ parts: Any = {package: []}
for pk in pap_keys:
mod, ver, kind, identifier = pk
parts[package].append((RefInfo(mod, ver, "api", mod), mod))
@@ -516,7 +529,7 @@ class D:
module=package,
parts_mods=parts.get(package, []),
parts=list(parts.items()),
- version=version,
+ version=_version,
parts_links=defaultdict(lambda: ""),
doc=doc,
)
@@ -550,6 +563,10 @@ class D:
)
def _myst_root(self, doc: IngestedBlobs) -> MRoot:
+ """
+ Convert a internal IngestedBlob document into a MyST tree
+ for rendering.
+ """
to_suppress = []
myst_acc: List[Any] = []
if doc.signature:
@@ -645,8 +662,9 @@ def render_one(
module = qa.split(".")[0]
return template.render(
current_type=current_type,
- doc=doc,
myst_root=root,
+ item_line=doc.item_line,
+ item_file=doc.item_file,
logo=meta.get("logo", None),
version=meta["version"],
module=module,
@@ -906,7 +924,13 @@ async def _write_narrative_files(self, config):
for _, (module, version, _, path) in progress(
narrative, description="Rendering Narrative..."
):
- data = await self._serve_narrative(module, version, path)
+ try:
+ data = await self._serve_narrative(module, version, path)
+ except Exception as e:
+ e.add_note(
+ f"rendering {module=} {version=} {path=}",
+ )
+ raise
if config.output_dir:
(config.output_dir / module / version / "docs").mkdir(
parents=True, exist_ok=True
@@ -1084,7 +1108,24 @@ async def serve_static(path):
class Resolver:
- def __init__(self, store, prefix: str, extension: str):
+ # mapping from package name to existing (version(s))
+ # currently multiple version nor really supported.
+ # this is used when we request to reach to a page from any version of a
+ # library to know which one we should look into.
+ version: Dict[str, str]
+
+ # the rendering might under prefix, depending on how it is hosted.
+ # This should thus be prepended to generated urls.
+ # it should end and start with a `/`.
+ prefix: str
+
+ # Extension that need to be added to the end of the url, why many server
+ # are fine omitting `.html`, it might be necessary when resolving for
+ # statically generated contents.
+
+ extension: str
+
+ def __init__(self, store, prefix: str, extension: str) -> None:
"""
Given a RefInfo to an object, resolve it to a full http-link
with the current configuration.
@@ -1110,7 +1151,7 @@ def __init__(self, store, prefix: str, extension: str):
self.prefix = prefix
self.extension = extension
- self.version: Dict[str, str] = {}
+ self.version = {}
for p, v in {
(package, version)
@@ -1118,12 +1159,22 @@ def __init__(self, store, prefix: str, extension: str):
}:
if p in self.version:
pass
- # todo, likely parse version here if possible.
+ # TODO:, likely parse version here if possible.
# maxv = max(v, self.version[p])
# print("multiple version for package", p, "Trying most recent", maxv)
self.version[p] = v
- def exists_resolve(self, info) -> Tuple[bool, Optional[str]]:
+ def exists_resolve(self, info: RefInfo) -> Tuple[bool, Optional[str]]:
+ """Resolve a RefInfo to a URL, additionally return wether the link item exists
+
+ Return
+ ------
+ exists: boolean
+ Whether the target document exists
+ url : str|None
+ If exists, URL where target document can be found
+
+ """
module, version_number, kind, path = info
if kind == "api":
kind = "module"
@@ -1131,21 +1182,36 @@ def exists_resolve(self, info) -> Tuple[bool, Optional[str]]:
if info.module in self.version:
version_number = self.version[info.module]
- i2 = RefInfo(module, version_number, kind, path)
- # TODO: Fix
+ query_ref = RefInfo(module, version_number, kind, path)
+ # TODO: Fix, for example in IPython narrative docs the
+ # toctree point to ``pr/*`` which is a subfolder we don't support yet:
+ #
+ # .. toctree::
+ # :maxdepth: 2
+ # :glob:
+ #
+ # pr/*
if kind == "?":
return False, None
- sgi = self.store.glob(i2)
+ sgi = self.store.glob(query_ref)
if sgi:
- exists, url = self._resolve(i2)
+ assert len(sgi) == 1, (
+ "we have no reason to have more than one reference",
+ sgi,
+ )
+ exists, url = self._resolve(query_ref)
return exists, url
# we may want to find older versions.
return False, None
- def resolve(self, info) -> str:
- return self._resolve(info)[1]
+ def resolve(self, info: RefInfo) -> Optional[str]:
+ exists, url = self.exists_resolve(info)
+ # TODO: this is moslty used to render navigation we should make sure that
+ # links are resolved and exists before rendering.
+ # assert exists, f"{info=} doe not exists"
+ return url
def _resolve(self, info) -> Tuple[bool, str]:
"""
@@ -1157,8 +1223,9 @@ def _resolve(self, info) -> Tuple[bool, str]:
"api",
"examples",
"assets",
- "?",
+ # "?",
"docs",
+ # TODO:
"to-resolve",
), repr(info)
# assume same package/version for now.
diff --git a/papyri/templates/html.tpl.j2 b/papyri/templates/html.tpl.j2
index 068ad119..4b0aae64 100644
--- a/papyri/templates/html.tpl.j2
+++ b/papyri/templates/html.tpl.j2
@@ -51,14 +51,11 @@ they belong to, and scaled with the number of references pointing them
{%if meta.get('github_slug')%}
- GitHub : {{doc.item_file}}#{{doc.item_line}}
+ GitHub : {{item_file}}#{{item_line}}
{% else %}
- File: {{doc.item_file}}#{{doc.item_line}}
+ File: {{item_file}}#{{item_line}}
{%endif%}
-type: {{doc.item_type}}
-Commit:
-
{% if graph is defined %}
{{d3script(graph)}}
{% endif %}
diff --git a/papyri/tree.py b/papyri/tree.py
index 1a3561a7..3e2da512 100644
--- a/papyri/tree.py
+++ b/papyri/tree.py
@@ -607,34 +607,34 @@ def _toctree_handler(self, argument, options, content):
for line in content.splitlines():
line = line.strip()
if line == "self":
- # TODO
+ # TODO in toctree one line can refer to self
continue
if "<" in line and line.endswith(">"):
- title, link = line[:-1].split("<")
+ title, url = line[:-1].split("<")
title = title.strip()
- assert "<" not in link
- toc.append([title, link])
- l = Link(
+ assert "<" not in url
+ toc.append([title, url])
+ link = Link(
title,
- reference=RefInfo(module="", version="", kind="?", path=link),
+ reference=RefInfo(module="", version="", kind="?", path=url),
kind="exists",
exists=True,
anchor=None,
)
- RESOLVER.add_reference(l, link)
- lls.append(l)
+ RESOLVER.add_reference(link, url)
+ lls.append(link)
else:
assert "<" not in line
toc.append([None, line])
- l = Link(
+ link = Link(
line,
reference=RefInfo(module="", version="", kind="?", path=line),
kind="exists",
exists=True,
anchor=None,
)
- RESOLVER.add_reference(l, line)
- lls.append(l)
+ RESOLVER.add_reference(link, line)
+ lls.append(link)
self._tocs.append(toc)
@@ -660,7 +660,7 @@ def replace_MMystDirective(self, myst_directive: MMystDirective):
return [myst_directive]
def replace_BlockDirective(self, block_directive: MMystDirective):
- assert False, "we shoudl never reach there"
+ assert False, "we should never reach there"
def _resolve(self, loc, text):
"""
diff --git a/papyri/ts.py b/papyri/ts.py
index 609279cc..ae58ecf2 100644
--- a/papyri/ts.py
+++ b/papyri/ts.py
@@ -261,7 +261,6 @@ def _targetify(self, acc):
def visit(self, node):
self.depth += 1
acc = []
- prev_end = None
# TODO: FIX
if node.type == "ERROR":
# print(f'ERROR node: {self.as_text(c)!r}, skipping')
@@ -280,28 +279,27 @@ def visit(self, node):
f"visit_{kind} not found while visiting {node}::\n{self.as_text(c)!r}"
)
meth = getattr(self, "visit_" + kind)
- new_children = meth(c, prev_end=prev_end)
+ new_children = meth(c)
acc.extend(new_children)
- prev_end = c.end_point
self.depth -= 1
acc = self._compressor(acc)
acc = self._targetify(acc)
return acc
- def visit_citation(self, node, prev_end=None):
+ def visit_citation(self, node):
# raise VisitCitationNotImplementedError()
# just hlines, like ------
return [Unimplemented("citation", self.as_text(node))]
- def visit_citation_reference(self, node, prev_end=None):
+ def visit_citation_reference(self, node):
raise VisitCitationReferenceNotImplementedError()
# just hlines, like ------
return []
- def visit_transition(self, node, prev_end=None):
+ def visit_transition(self, node):
return [MThematicBreak()]
- def visit_reference(self, node, prev_end=None):
+ def visit_reference(self, node):
"""
TODO:
@@ -324,7 +322,7 @@ def visit_reference(self, node, prev_end=None):
assert trailing in ("_", "__")
return [Directive(_text, None, None)]
- def visit_interpreted_text(self, node, prev_end=None):
+ def visit_interpreted_text(self, node):
if len(node.children) == 2:
role, text = node.children
assert role.type == "role"
@@ -374,10 +372,10 @@ def visit_interpreted_text(self, node, prev_end=None):
)
return [t]
- def visit_standalone_hyperlink(self, node, prev_end=None):
+ def visit_standalone_hyperlink(self, node):
return self.visit_text(node)
- def visit_text(self, node, prev_end=None):
+ def visit_text(self, node):
text = self.bytes[node.start_byte : node.end_byte].decode()
assert not text.startswith(":func:"), breakpoint()
t = MText(text)
@@ -385,7 +383,7 @@ def visit_text(self, node, prev_end=None):
t.end_byte = node.end_byte
return [t]
- def visit_whitespace(self, node, prev_end=None):
+ def visit_whitespace(self, node):
content = self.bytes[node.start_byte : node.end_byte].decode()
# assert set(content) == {' '}, repr(content)
t = MText(" " * len(content))
@@ -394,21 +392,21 @@ def visit_whitespace(self, node, prev_end=None):
# print(' '*self.depth*4, t, node.start_byte, node.end_byte)
return [t]
- def visit_literal(self, node, prev_end=None):
+ def visit_literal(self, node):
text = self.bytes[node.start_byte + 2 : node.end_byte - 2].decode()
assert "\n\n" not in text
t = MInlineCode(text.replace("\n", " "))
# print(' '*self.depth*4, t)
return [t]
- def visit_literal_block(self, node, prev_end=None):
+ def visit_literal_block(self, node):
datas = self.bytes[node.start_byte : node.end_byte].decode()
first_offset = node.start_point[1]
datas = " " * first_offset + datas
b = MCode(dedent(datas))
return [b]
- def visit_bullet_list(self, node, prev_end=None):
+ def visit_bullet_list(self, node):
myst_acc = []
for list_item in node.children:
assert list_item.type == "list_item"
@@ -420,7 +418,7 @@ def visit_bullet_list(self, node, prev_end=None):
myst_acc.append(MListItem(False, self.visit(body)))
return [MList(ordered=False, start=1, spread=False, children=myst_acc)]
- def visit_section(self, node, prev_end=None):
+ def visit_section(self, node):
# print(' '*self.depth*4, '->', node)
# print(' '*self.depth*4, '::',self.bytes[node.start_byte: node.end_byte].decode())
if node.children[0].type == "adornment":
@@ -478,10 +476,10 @@ def visit_section(self, node, prev_end=None):
# print(' '*self.depth*4, '->', node)
return [Section([], title, level=level)]
- def visit_block_quote(self, node, prev_end=None):
+ def visit_block_quote(self, node):
return [MBlockquote(self.visit(node))]
- def visit_paragraph(self, node, prev_end=None):
+ def visit_paragraph(self, node):
sub = self.visit(node.with_whitespace())
acc = []
acc2 = []
@@ -498,24 +496,24 @@ def visit_paragraph(self, node, prev_end=None):
p = MParagraph(compress_word(acc))
return [p, *acc2]
- def visit_line_block(self, node, prev_end=None):
+ def visit_line_block(self, node):
# TODO
# e.g: numpy/doc/source/user/c-info.how-to-extend.rst
print("Skipping node", self.as_text(node))
return []
- def visit_substitution_reference(self, node, prev_end=None):
+ def visit_substitution_reference(self, node):
# TODO
return [SubstitutionRef(self.as_text(node))]
- def visit_doctest_block(self, node, prev_end=None) -> List[MCode]:
+ def visit_doctest_block(self, node) -> List[MCode]:
# TODO
- return self.visit_literal_block(node, prev_end)
+ return self.visit_literal_block(node)
- def visit_field(self, node, prev_end=None):
+ def visit_field(self, node):
return []
- def visit_field_list(self, node, prev_end=None) -> List[FieldList]:
+ def visit_field_list(self, node) -> List[FieldList]:
acc = []
lens = {len(f.children) for f in node.children}
@@ -544,7 +542,7 @@ def visit_field_list(self, node, prev_end=None) -> List[FieldList]:
else:
raise ValueError("mixed len...")
- def visit_enumerated_list(self, node, prev_end=None):
+ def visit_enumerated_list(self, node):
myst_acc = []
for list_item in node.children:
assert list_item.type == "list_item"
@@ -552,7 +550,7 @@ def visit_enumerated_list(self, node, prev_end=None):
myst_acc.append(MListItem(False, self.visit(body)))
return [MList(ordered=True, start=1, spread=False, children=myst_acc)]
- def visit_target(self, node, prev_end=None):
+ def visit_target(self, node):
# TODO:
# raise VisitTargetNotImplementedError()
# self.as_text(node)
@@ -563,21 +561,21 @@ def visit_target(self, node, prev_end=None):
return [Unimplemented("untarget", self.as_text(name))]
return [Unimplemented("target", self.as_text(node))]
- # def visit_arguments(self, node, prev_end=None):
+ # def visit_arguments(self, node):
# assert False
# return []
- def visit_attribution(self, node, prev_end):
+ def visit_attribution(self, node):
# TODO:
print("attribution not implemented")
return [Unimplemented("inline_target", self.as_text(node))]
- def visit_inline_target(self, node, prev_end):
+ def visit_inline_target(self, node):
# TODO:
print("inline_target not implemented")
return [Unimplemented("inline_target", self.as_text(node))]
- def visit_directive(self, node, prev_end=None):
+ def visit_directive(self, node):
"""
Main entry point for directives.
@@ -588,7 +586,6 @@ def visit_directive(self, node, prev_end=None):
----------
node: Node
The directive to parse
- prev_end: Unknown
Returns
-------
@@ -697,12 +694,12 @@ def visit_directive(self, node, prev_end=None):
directive = MMystDirective(role, argument, dict(options), content)
return [directive]
- def visit_footnote_reference(self, node, prev_end=None):
+ def visit_footnote_reference(self, node):
# TODO
# assert False, self.bytes[node.start_byte : node.end_byte].decode()
return []
- def visit_emphasis(self, node, prev_end=None):
+ def visit_emphasis(self, node):
# TODO
return [
MEmphasis(
@@ -710,7 +707,7 @@ def visit_emphasis(self, node, prev_end=None):
)
]
- def visit_substitution_definition(self, node, prev_end=None):
+ def visit_substitution_definition(self, node):
assert len(node.children) == 3
_dotdot, sub, directive = node.children
assert self.bytes[_dotdot.start_byte : _dotdot.end_byte].decode() == ".."
@@ -723,25 +720,25 @@ def visit_substitution_definition(self, node, prev_end=None):
)
]
- def visit_comment(self, node, prev_end=None):
+ def visit_comment(self, node):
# TODO
return [MComment(self.bytes[node.start_byte : node.end_byte].decode())]
# raise VisitCommentNotImplementedError()
- def visit_strong(self, node, prev_end=None):
+ def visit_strong(self, node):
return [
MStrong(
[MText(self.bytes[node.start_byte + 2 : node.end_byte - 2].decode())]
)
]
- def visit_footnote(self, node, prev_end=None):
+ def visit_footnote(self, node):
# TODO
# that is actually used for references
# assert False, self.bytes[node.start_byte : node.end_byte].decode()
return [Unimplemented("footnote", self.as_text(node))]
- def visit_ERROR(self, node, prev_end=None):
+ def visit_ERROR(self, node):
"""
Called with parsing error nodes.
"""
@@ -749,7 +746,7 @@ def visit_ERROR(self, node, prev_end=None):
# raise TreeSitterParseError()
return []
- def visit_definition_list(self, node, prev_end=None):
+ def visit_definition_list(self, node):
acc = []
for list_item in node.children:
assert list_item.type == "list_item"