From 1a204d3a6dbeaeb072e16fae75edc999be9c1a5f Mon Sep 17 00:00:00 2001 From: Egor Dmitriev Date: Tue, 5 Oct 2021 22:15:17 +0200 Subject: [PATCH] [FEATURE] Added inline database / table support --- notionsci/cli/notion.py | 2 +- notionsci/connections/notion/client.py | 6 ++++- .../connections/notion/structures/blocks.py | 22 +++++++++++++++--- .../connections/notion/structures/content.py | 2 +- .../notion/structures/properties.py | 23 ++++++++++++++++--- notionsci/utils/markdown.py | 2 +- notionsci/utils/serialization.py | 13 +++++++++++ 7 files changed, 60 insertions(+), 10 deletions(-) diff --git a/notionsci/cli/notion.py b/notionsci/cli/notion.py index 95cb65d..687fcad 100644 --- a/notionsci/cli/notion.py +++ b/notionsci/cli/notion.py @@ -109,7 +109,7 @@ def download_md(page: ID, output: str): notion = config.connections.notion.client() page = notion.page_get(page) - notion.load_children(page, recursive=True) + notion.load_children(page, recursive=True, databases=True) path = os.path.join(output, f'{sanitize_filename(page.get_title())}.md') if os.path.isdir(output) else output content = page.to_markdown(MarkdownContext()) diff --git a/notionsci/connections/notion/client.py b/notionsci/connections/notion/client.py index b1ab67c..1ed4dc8 100644 --- a/notionsci/connections/notion/client.py +++ b/notionsci/connections/notion/client.py @@ -3,6 +3,7 @@ from notion_client import Client +from notionsci.connections.notion import BlockType from notionsci.connections.notion.structures import Database, SortObject, QueryFilter, format_query_args, ContentObject, \ PropertyType, Page, ID, QueryResult, Block from notionsci.utils import strip_none_field, filter_none_dict @@ -150,10 +151,13 @@ def block_retrieve_all_children( query_fn=lambda **args: self.block_retrieve_children(**args) ) - def load_children(self, item: Union[Page, Block], recursive=False): + def load_children(self, item: Union[Page, Block], recursive=False, databases=False): children: List[Block] = list(self.block_retrieve_all_children(item.id)) item.set_children(children) if recursive: for child in children: if child.has_children and child.get_children() is None: self.load_children(child, recursive) + elif databases and child.type == BlockType.child_database: + child.child_database.database = self.database_get(child.id) + child.child_database.children = list(self.database_query_all(child.id)) # eager load (mayby dont?) diff --git a/notionsci/connections/notion/structures/blocks.py b/notionsci/connections/notion/structures/blocks.py index e513538..ae5f77e 100644 --- a/notionsci/connections/notion/structures/blocks.py +++ b/notionsci/connections/notion/structures/blocks.py @@ -4,12 +4,13 @@ from enum import Enum from typing import Optional, List, Union +import pandas as pd from dataclass_dict_convert import dataclass_dict_convert from stringcase import snakecase from notionsci.connections.notion.structures.common import FileObject, RichText, ID from notionsci.utils import ForwardRefConvertor, ListConvertor, ToMarkdownMixin, MarkdownBuilder, MarkdownContext, \ - chain_to_markdown + chain_to_markdown, ignore_fields from notionsci.utils.markdown import MarkdownListType @@ -252,13 +253,28 @@ def to_markdown(self, context: MarkdownContext) -> str: return MarkdownBuilder.code(content, self.language) -@dataclass_dict_convert(dict_letter_case=snakecase) +@dataclass_dict_convert( + dict_letter_case=snakecase, + **ignore_fields(['database', 'children']) +) @dataclass class ChildDatabaseBlock(ToMarkdownMixin): title: str + database: Optional['notionsci.connections.notion.Database'] = None + children: Optional[List['notionsci.connections.notion.Page']] = None def to_markdown(self, context: MarkdownContext) -> str: - return f'Child Database {self.title} !!!!\n' + if self.database and self.children: + df = pd.DataFrame([ + { + prop_name: child.get_property(prop_name).to_markdown(context) + for prop_name, prop in self.database.properties.items() + } + for child in self.children + ]) + return MarkdownBuilder.table(df) + else: + return f'Table {self.title}\n' FileBlock = FileObject diff --git a/notionsci/connections/notion/structures/content.py b/notionsci/connections/notion/structures/content.py index d522b74..394915e 100644 --- a/notionsci/connections/notion/structures/content.py +++ b/notionsci/connections/notion/structures/content.py @@ -70,6 +70,7 @@ class ContentObject: object: str id: Optional[ID] = None parent: Optional[Parent] = None + url: Optional[str] = None icon: EmojiFileType = None cover: Optional[FileObject] = None @@ -85,7 +86,6 @@ class ContentObject: @dataclass class Page(ContentObject, ToMarkdownMixin, ChildrenMixin, HasPropertiesMixin[Property]): object: str = 'page' - url: Optional[str] = None properties: Dict[str, Property] = field(default_factory=dict) diff --git a/notionsci/connections/notion/structures/properties.py b/notionsci/connections/notion/structures/properties.py index cc07048..c37c102 100644 --- a/notionsci/connections/notion/structures/properties.py +++ b/notionsci/connections/notion/structures/properties.py @@ -7,7 +7,7 @@ from stringcase import snakecase from notionsci.connections.notion.structures.common import RichText, Color, ID -from notionsci.utils import ExplicitNone +from notionsci.utils import ExplicitNone, ToMarkdownMixin, MarkdownContext class PropertyType(Enum): @@ -76,15 +76,26 @@ def object_to_text_value(raw_value: Any): return raw_value +def object_to_markdown(raw_value: Any, context: MarkdownContext, sep=' '): + if isinstance(raw_value, list): + return sep.join([object_to_markdown(v, context) for v in raw_value]) + elif isinstance(raw_value, RichText): + return raw_value.to_markdown(context) + elif isinstance(raw_value, SelectValue): + return raw_value.name + return raw_value + + ## Property Definition Types @dataclass_dict_convert(dict_letter_case=snakecase) @dataclass -class Property: +class Property(ToMarkdownMixin): type: PropertyType - id: Optional[str] = None + id: Optional[str] = None title: Optional[TitleValue] = None + rich_text: Optional[RichTextValue] = None number: Optional[NumberValue] = None select: Optional[SelectValue] = None @@ -116,6 +127,12 @@ def raw_value(self): def value(self): return object_to_text_value(self.raw_value()) + def to_markdown(self, context: MarkdownContext) -> str: + return object_to_markdown( + self._value(), context, + sep=',+ ' if self.type == PropertyType.multi_select else ' ' + ) + @staticmethod def as_title(text: str) -> 'Property': return Property( diff --git a/notionsci/utils/markdown.py b/notionsci/utils/markdown.py index 5d8b5a5..77f1d75 100644 --- a/notionsci/utils/markdown.py +++ b/notionsci/utils/markdown.py @@ -155,7 +155,7 @@ def table(data: pd.DataFrame, alignments: Dict[str, str] = None): if alignments: for col, align in alignments.items(): data.style.set_properties(subset=[col], **{'text-align': align}) - return data.to_markdown() + return data.to_markdown(index=False) @staticmethod def code(content, language): diff --git a/notionsci/utils/serialization.py b/notionsci/utils/serialization.py index c260494..06bb105 100644 --- a/notionsci/utils/serialization.py +++ b/notionsci/utils/serialization.py @@ -48,3 +48,16 @@ def convert_to_dict(self, val: Any) -> Union[Dict, List, int, float, str, bool]: def ignore_unknown(field): pass + + +def ignore_fields(fields): + return dict( + custom_to_dict_convertors={ + field: lambda x: None + for field in fields + }, + custom_from_dict_convertors={ + field: lambda x: None + for field in fields + } + ) \ No newline at end of file