diff --git a/.github/workflows/rename_project.yml b/.github/workflows/rename_project.yml deleted file mode 100644 index 3c9d962..0000000 --- a/.github/workflows/rename_project.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Rename the project from template - -on: [push] - -permissions: write-all - -jobs: - rename-project: - if: ${{ !contains (github.repository, '/python-project-template') }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - # by default, it uses a depth of 1 - # this fetches all history so that we can read each commit - fetch-depth: 0 - ref: ${{ github.head_ref }} - - - run: echo "REPOSITORY_NAME=$(echo '${{ github.repository }}' | awk -F '/' '{print $2}' | tr '-' '_' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV - shell: bash - - - run: echo "REPOSITORY_URLNAME=$(echo '${{ github.repository }}' | awk -F '/' '{print $2}')" >> $GITHUB_ENV - shell: bash - - - run: echo "REPOSITORY_OWNER=$(echo '${{ github.repository }}' | awk -F '/' '{print $1}')" >> $GITHUB_ENV - shell: bash - - - name: Is this still a template - id: is_template - run: echo "::set-output name=is_template::$(ls .github/template.yml &> /dev/null && echo true || echo false)" - - - name: Rename the project - if: steps.is_template.outputs.is_template == 'true' - run: | - echo "Renaming the project with -a(author) ${{ env.REPOSITORY_OWNER }} -n(name) ${{ env.REPOSITORY_NAME }} -u(urlname) ${{ env.REPOSITORY_URLNAME }}" - .github/rename_project.sh -a ${{ env.REPOSITORY_OWNER }} -n ${{ env.REPOSITORY_NAME }} -u ${{ env.REPOSITORY_URLNAME }} -d "Awesome ${{ env.REPOSITORY_NAME }} created by ${{ env.REPOSITORY_OWNER }}" - - - uses: stefanzweifel/git-auto-commit-action@v5 - with: - commit_message: "✅ Ready to clone and code." - # commit_options: '--amend --no-edit' - push_options: --force diff --git a/HISTORY.md b/HISTORY.md index d3ee8c0..7f585ee 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -2,10 +2,31 @@ Changelog ========= +(unreleased) +------------ +- During tests, path_schemas to point to tmp folder. [sdementen] +- Use setup.cfg instead of .flake8. [sdementen] +- Add blank in some gnucash book name for testing purposes. [sdementen] +- Mypy fixes. [sdementen] +- Add make CLI implemented in python. [sdementen] +- Fix flake8 issues. [sdementen] +- Fix linter issues. [sdementen] +- Adapt linter to use 140 charts per line. [sdementen] +- Remove the rename_project.yml CI action. [sdementen] +- Add sqlacodegen_v2 as dependency. [sdementen] +- Update setup.py with maturity and description. [sdementen] +- Add some simple tests. [sdementen] +- Add example in README.md. [sdementen] +- Add first working code. [sdementen] +- Add data folder with sample of gnucash sqlite books. [sdementen] +- Release: version 0.1.1 🚀 [sdementen] + + 0.1.1 (2024-06-24) ------------------ -- Release: version 0.1.1 🚀 [sdementen] -- Clean up repo generated from template. [sdementen] +- Release: version 0.1.1 🚀 [Sébastien de Menten] +- Clean up repo generated from template. [Sébastien de Menten] +- ✅ Ready to clone and code. [sdementen] - Initial commit. [sdementen] diff --git a/Makefile b/Makefile index 4e10c5e..176174b 100644 --- a/Makefile +++ b/Makefile @@ -33,8 +33,8 @@ fmt: ## Format code using black & isort. .PHONY: lint lint: ## Run pep8, black, mypy linters. $(ENV_PREFIX)flake8 piecash2/ - $(ENV_PREFIX)black -l 79 --check piecash2/ - $(ENV_PREFIX)black -l 79 --check tests/ + $(ENV_PREFIX)black -l 139 --check piecash2/ + $(ENV_PREFIX)black -l 139 --check tests/ $(ENV_PREFIX)mypy --ignore-missing-imports piecash2/ .PHONY: test diff --git a/README.md b/README.md index 3881733..532895f 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![codecov](https://codecov.io/gh/sdementen/piecash2/branch/main/graph/badge.svg?token=piecash2_token_here)](https://codecov.io/gh/sdementen/piecash2) [![CI](https://github.com/sdementen/piecash2/actions/workflows/main.yml/badge.svg)](https://github.com/sdementen/piecash2/actions/workflows/main.yml) -piecash using sqlalchemy 2 +A python library to work with [GnuCash](https://www.gnucash.org/) books, a successor of the [piecash](https://github.com/sdementen/piecash) library, built on top of SQLAlchemy 2. ## Install it from PyPI @@ -15,17 +15,17 @@ pip install piecash2 ## Usage ```py -from piecash2 import BaseClass -from piecash2 import base_function +from piecash2 import open_book -BaseClass().base_method() -base_function() -``` +# open the gnucash book (sqlite3 file) +Session = open_book("mybook.gnucash") +# retrieve the module +piecash = Session.module -```bash -$ python -m piecash2 -#or -$ piecash2 +with Session() as session: + # query all accounts in the + for account in session.query(piecash.Account).all(): + print(account.name) ``` ## Development diff --git a/data/README.md b/data/README.md new file mode 100644 index 0000000..f7dfbf5 --- /dev/null +++ b/data/README.md @@ -0,0 +1,11 @@ +# Sample gnucash books + +This folder holds a set of gnucash books used for testing, examples and debugging. + + +## Default empty books + +The following gnucash books can be used to introspect the SQL schema: +- reference/default_2_6_21_basic.gnucash: empty gnucash books created with gnucash 2.6.21 and no specific options +- reference/default_2_6_21_full_options.gnucash: empty gnucash books created with gnucash 2.6.21 with multiple options enabled (trading accounts, ...) + diff --git a/data/book_prices.gnucash b/data/book_prices.gnucash new file mode 100644 index 0000000..0d93c28 Binary files /dev/null and b/data/book_prices.gnucash differ diff --git a/data/book_schtx.gnucash b/data/book_schtx.gnucash new file mode 100644 index 0000000..f8fc602 Binary files /dev/null and b/data/book_schtx.gnucash differ diff --git a/data/complex_sample.gnucash b/data/complex_sample.gnucash new file mode 100644 index 0000000..3214b57 Binary files /dev/null and b/data/complex_sample.gnucash differ diff --git a/data/default_book.gnucash b/data/default_book.gnucash new file mode 100644 index 0000000..0c549a3 Binary files /dev/null and b/data/default_book.gnucash differ diff --git a/data/empty_book.gnucash b/data/empty_book.gnucash new file mode 100644 index 0000000..75f8a72 Binary files /dev/null and b/data/empty_book.gnucash differ diff --git a/data/ghost_kvp_scheduled_transaction.gnucash b/data/ghost_kvp_scheduled_transaction.gnucash new file mode 100644 index 0000000..3e3af00 Binary files /dev/null and b/data/ghost_kvp_scheduled_transaction.gnucash differ diff --git a/data/investment.gnucash b/data/investment.gnucash new file mode 100644 index 0000000..5427a0a Binary files /dev/null and b/data/investment.gnucash differ diff --git a/data/invoices.gnucash b/data/invoices.gnucash new file mode 100644 index 0000000..97bf90f Binary files /dev/null and b/data/invoices.gnucash differ diff --git a/data/reference/2_6/default_2_6_21_basic.gnucash b/data/reference/2_6/default_2_6_21_basic.gnucash new file mode 100644 index 0000000..127b330 Binary files /dev/null and b/data/reference/2_6/default_2_6_21_basic.gnucash differ diff --git a/data/reference/2_6/default_2_6_21_full_options.gnucash b/data/reference/2_6/default_2_6_21_full_options.gnucash new file mode 100644 index 0000000..951beed Binary files /dev/null and b/data/reference/2_6/default_2_6_21_full_options.gnucash differ diff --git a/data/reference/3_0/default_3_0_0_basic.gnucash b/data/reference/3_0/default_3_0_0_basic.gnucash new file mode 100644 index 0000000..f2829c3 Binary files /dev/null and b/data/reference/3_0/default_3_0_0_basic.gnucash differ diff --git a/data/reference/3_0/default_3_0_0_full_options.gnucash b/data/reference/3_0/default_3_0_0_full_options.gnucash new file mode 100644 index 0000000..700ca4b Binary files /dev/null and b/data/reference/3_0/default_3_0_0_full_options.gnucash differ diff --git a/data/simple_sample.272.gnucash b/data/simple_sample.272.gnucash new file mode 100644 index 0000000..05c5eb5 Binary files /dev/null and b/data/simple_sample.272.gnucash differ diff --git a/data/simple_sample.gnucash b/data/simple_sample.gnucash new file mode 100644 index 0000000..a7bbb73 Binary files /dev/null and b/data/simple_sample.gnucash differ diff --git a/data/test book.gnucash b/data/test book.gnucash new file mode 100644 index 0000000..b44b5d6 Binary files /dev/null and b/data/test book.gnucash differ diff --git a/make.bat b/make.bat new file mode 100644 index 0000000..9778dd5 --- /dev/null +++ b/make.bat @@ -0,0 +1 @@ +@py.exe make.py %* \ No newline at end of file diff --git a/make.py b/make.py new file mode 100644 index 0000000..6e622a8 --- /dev/null +++ b/make.py @@ -0,0 +1,104 @@ +import os +import shutil +from pathlib import Path + +import typer + +os.environ["PYTHONIOENCODING"] = "utf-8" + +app = typer.Typer() + +HERE = Path(__file__).parent + + +@app.command() +def release(tag: str): + print(f"WARNING: This operation will create version {tag=} and push to github") + typer.confirm("Do you want to continue?", abort=True) + Path("piecash2/VERSION").write_text(tag) + os.system("gitchangelog > HISTORY.md") + os.startfile("HISTORY.md") + typer.confirm("Did you update the changelog?", abort=True) + os.system("git add piecash2/VERSION HISTORY.md") + os.system(f'git commit -m "release: version {tag}') + print(f"creating git tag : {tag}") + os.system(f"git tag {tag}") + os.system("git push -u origin HEAD --tags") + print("Github Actions will detect the new tag and release the new version.") + + +@app.command() +def lint(): + """lint: ## Run pep8, black, mypy linters.""" + os.system("flake8 piecash2/") + os.system("black -l 140 --check piecash2/") + os.system("black -l 140 --check tests/") + os.system("mypy --ignore-missing-imports piecash2/") + + +@app.command() +def fmt(): + """fmt: ## Format code using black & isort.""" + os.system("isort piecash2/") + os.system("black -l 140 piecash2/") + os.system("black -l 140 tests/") + + +@app.command() +def docs(): + """fmt: ## Format code using black & isort.""" + os.system("mkdocs build") + os.startfile(Path(__file__).parent / "site" / "index.html") + + +@app.command() +def clean(): + """clean: ## Clean up unused files.""" + patterns = [ + "**/*.pyc", + "**/__pycache__", + "**/Thumbs.db", + "**/*~", + ".cache", + ".pytest_cache", + ".mypy_cache", + ".tox", + "build", + "dist", + "*.egg-info", + "htmlcov", + "docs/_build", + ] + + for fp in patterns: + for f in HERE.glob(fp): + if f.is_file(): + f.unlink() + else: + shutil.rmtree(f) + + +@app.command() +def test(): + """test: ## Run tests and generate coverage report.""" + os.system("pytest -v --cov-config .coveragerc --cov=piecash2 -l --tb=short --maxfail=1 tests/") + os.system("coverage xml") + os.system("coverage html") + os.startfile(HERE / "htmlcov" / "index.html") + + +@app.command() +def schema(): + """schema: ## Generate the schema from the sqlite database using sqlacodegen.""" + import piecash2.schema.generation.schema_generation as schema_generation + import piecash2.schema.generated as generated + + schema_generation.path_schemas = Path(generated.__file__).parent + + print(f"Generating schemas in {schema_generation.path_schemas}") + for book in (HERE / "data").glob("*.gnucash"): + schema_generation.generate_schema(book, schema_generation.get_schema_name(book)) + + +if __name__ == "__main__": + app() diff --git a/piecash2/VERSION b/piecash2/VERSION index 17e51c3..8294c18 100644 --- a/piecash2/VERSION +++ b/piecash2/VERSION @@ -1 +1 @@ -0.1.1 +0.1.2 \ No newline at end of file diff --git a/piecash2/__init__.py b/piecash2/__init__.py index e69de29..783cb4e 100644 --- a/piecash2/__init__.py +++ b/piecash2/__init__.py @@ -0,0 +1,5 @@ +from piecash2.schema.generation.schema_generation import add_book_module_in_path + +from .core.book import open_book + +__ALL__ = [open_book, add_book_module_in_path] diff --git a/piecash2/base.py b/piecash2/base.py deleted file mode 100644 index c6cbf0d..0000000 --- a/piecash2/base.py +++ /dev/null @@ -1,17 +0,0 @@ -""" -piecash2 base module. - -This is the principal module of the piecash2 project. -here you put your main classes and objects. - -Be creative! do whatever you want! - -If you want to replace this with a Flask application run: - - $ make init - -and then choose `flask` as template. -""" - -# example constant variable -NAME = "piecash2" diff --git a/piecash2/core/__init__.py b/piecash2/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/piecash2/core/book.py b/piecash2/core/book.py new file mode 100644 index 0000000..2cc1ef1 --- /dev/null +++ b/piecash2/core/book.py @@ -0,0 +1,33 @@ +import sqlite3 +from pathlib import Path + +import sqlalchemy +import sqlalchemy.orm + +from piecash2.schema.generation.schema_generation import import_gnucash + + +def open_book(book, regenerate_schema=False): + if isinstance(book, str): + book = Path(book) + + book_posix = book.as_posix() + + # make a backup of the DB in memory + db_memory_name = f":memgeco_{abs(hash(book_posix))}:" + with sqlite3.connect(book_posix) as source: + sqliteconn = f"file:{db_memory_name}?mode=memory&cache=shared" + dest = sqlite3.connect(sqliteconn, uri=True) + source.backup(dest) + + engine = sqlalchemy.create_engine(f"sqlite:///{db_memory_name}", echo=False, creator=lambda: sqlite3.connect(sqliteconn, uri=True)) + + Session = sqlalchemy.orm.sessionmaker(bind=engine, autoflush=True, autocommit=False) + + # must execute some query otherwise future call to the Session raise error + with Session() as s: + s.execute(sqlalchemy.text("")) + + Session.module = import_gnucash(book, regenerate_schema=regenerate_schema) + + return Session diff --git a/piecash2/schema/__init__.py b/piecash2/schema/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/piecash2/schema/generated/__init__.py b/piecash2/schema/generated/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/piecash2/schema/generation/__init__.py b/piecash2/schema/generation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/piecash2/schema/generation/sa_schema_post.py b/piecash2/schema/generation/sa_schema_post.py new file mode 100644 index 0000000..e95943d --- /dev/null +++ b/piecash2/schema/generation/sa_schema_post.py @@ -0,0 +1,4 @@ +# mypy: ignore-errors +# list all glasses with a guid property (that can be linked to in Slot/Recurrence through obj_guid) +gl = globals() +name2kls = {n: gl[n] for n in klswithguid_names} # noqa: F821 diff --git a/piecash2/schema/generation/sa_schema_pre.py b/piecash2/schema/generation/sa_schema_pre.py new file mode 100644 index 0000000..82663b9 --- /dev/null +++ b/piecash2/schema/generation/sa_schema_pre.py @@ -0,0 +1,331 @@ +from decimal import Decimal +from enum import Enum +from fractions import Fraction +from functools import lru_cache + +import sqlalchemy +from sqlalchemy import select, text, union_all +from sqlalchemy.ext.declarative import declarative_base as declarative_base_ +from sqlalchemy.ext.hybrid import Comparator, hybrid_property +from sqlalchemy.orm import DeclarativeMeta as DeclarativeMeta_, Session + +from piecash2 import utils + +# todo: reduce the list to classes which can really contains slots/recurrences to speed up detection of object class +klswithguid_names = [ + "Account", + "Billterm", + "Book", + "Budget", + "Commodity", + "Customer", + "Employee", + "Entry", + "Invoice", + "Job", + "Lot", + "Order", + "Price", + "Schedxaction", + "Slot", + "Split", + "Taxtable", + "Transaction", + "Vendor", +] + + +class SlotValueType(Enum): + """Enum to define the type of owner""" + + INVALID = -1 + INT64 = 1 + DOUBLE = 2 + NUMERIC = 3 + STRING = 4 + GUID = 5 + TIME64 = 6 + PLACEHOLDER_DONT_USE = 7 + GLIST = 8 + FRAME = 9 + GDATE = 10 + + +class OwnerType(Enum): + """Enum to define the type of owner""" + + NONE = 0 + UNDEFINED = 1 + CUSTOMER = 2 + JOB = 3 + VENDOR = 4 + EMPLOYEE = 5 + + @classmethod + def from_cls(cls, obj): + return cls[obj.__class__.__name__.upper()] if obj is not None else cls.NONE + + @property + def cls_name(self): + return self.attr_name.capitalize() + + @property + def attr_name(self): + return self.name.lower() + + @classmethod + def object_classes(cls) -> list["OwnerType"]: + return [cls for cls in cls if cls not in {cls.NONE, cls.UNDEFINED}] + + +class GuidComparator(Comparator): + """Take a guid and compare it to another object's guid""" + + def __eq__(self, other): + # Define custom comparison for equality + return self.__clause_element__() == other.guid + + def __ne__(self, other): + # Define custom comparison for inequality + return self.__clause_element__() != other.guid + + +class DeclarativeMeta(DeclarativeMeta_): + use_decimal = False + + def __new__(cls, name, bases, attrs): + """ + The different objects use different naming conventions: + - tablename = underscore plural + - name = CamelCase singular + - fields = underscore singular + """ + + tablename = attrs.get("__tablename__") + + if tablename: + assert tablename == utils.underscore(utils.pluralize(name)), f"{tablename} <> {utils.underscore(utils.pluralize(name))}" + + self_id = attrs.get("guid", attrs.get("id")) + + for k, v in list(attrs.items()): + if name == "Slot" and k == "obj_guid": + # replace name of column obj_guid to guid + attrs["guid"] = sqlalchemy.orm.synonym("guid_val") + + elif name in {"Slot", "Recurrence"} and k == "guid_val": + attrs_to_expire = [] + # define the n relationships related to guid + for kls_other in klswithguid_names: + kwargs = dict( + argument=kls_other, + backref="slots", + primaryjoin=f"foreign({name}.obj_guid) == remote({kls_other}.guid)", + viewonly=True, + ) + attr_name = f"object_val_{kls_other.lower()}" + attrs[attr_name] = sqlalchemy.orm.relationship(**kwargs) + attrs_to_expire.append(attr_name) + + # define the object property + def object_getter(self): + session = Session.object_session(self) + + # detect which type of class is the object related to the obj_guid + qry_object_type_attr = union_all( + *( + select(text(f"'{name.lower()}'")).where(kls.guid == self.obj_guid) + for name, kls in name2kls.items() # noqa: F821 + ), + ).limit(1) + object_type_attr = session.execute(qry_object_type_attr).scalar() + + # return the object related to the obj_guid + return getattr(self, f"object_val_{object_type_attr}") + + def object_comparator(cls): + return GuidComparator(cls.obj_guid) + + def object_setter(self, object): + object_old = self.object + self.obj_guid = object.guid + # expire the relationships related to the attr + session = Session.object_session(self) + session.expire(self, attrs_to_expire) # expire link Slot -> Object + session.expire(object_old, ["slots"]) # expire backrefs on old object + session.expire(object, ["slots"]) # expire backrefs on new object + + attrs["object"] = object_getter + attrs["object"] = hybrid_property(object_getter).setter(object_setter).comparator(object_comparator) + + elif name == "Slot" and k == "slot_type": + # handle slot_type as enum via slot_type_enum + # todo: rename slot_type to _slot_type and slot_type_enum to slot_type + # todo: redefine slot_type to be an Enum type but using the same backed as original + @hybrid_property + def getter(self): + return SlotValueType(self.slot_type) + + @getter.setter + def setter(self, value): + self.slot_type = value.value + + @getter.expression + def expression(cls): + return cls.slot_type + + attrs["slot_type_enum"] = getter + + # handle fractions/decimals as numerators/denominators + elif k.endswith("_denom") and k[:-6] + "_num" in attrs: + fraction_base = k[:-6] + if cls.use_decimal: + attrs[f"{fraction_base}"] = hybrid_property( + lambda self: Decimal(getattr(self, fraction_base + "_num")) / Decimal(getattr(self, fraction_base + "_denom")) + ).setter( + lambda self, value: setattr(self, fraction_base + "_num", (ratio := value.as_integer_ratio())[0]) + or setattr(self, fraction_base + "_denom", ratio[1]) + ) + else: + attrs[f"{fraction_base}"] = hybrid_property( + lambda self: Fraction(getattr(self, fraction_base + "_num"), getattr(self, fraction_base + "_denom")) + ).setter( + lambda self, value: setattr(self, fraction_base + "_num", value.numerator) + or setattr(self, fraction_base + "_denom", value.denominator) + ) + + # handle owner/billto _guid/_type combo + elif k.endswith("_guid") and k[:-5] + "_type" in attrs: + # define the n relationships related to OwnerTypes + # and the appropriate python getter/setter and SQL comparator + attr = k[:-5] + assert attr in {"billto", "owner"}, attr + + for typ in OwnerType: + if typ in {OwnerType.NONE, OwnerType.UNDEFINED}: + continue + kls_other = typ.cls_name + kwargs = dict( + argument=kls_other, + backref=None, + primaryjoin=f"(foreign({name}.{attr}_guid) == remote({kls_other}.guid)) " + f"& ({name}.{attr}_type == {typ.value})", + viewonly=True, + ) + attrs[f"{attr}_{typ.attr_name}"] = sqlalchemy.orm.relationship(**kwargs) + + def attr_getter(self, attr=attr): + typ = OwnerType(getattr(self, f"{attr}_type") or 0) + + if typ in OwnerType.object_classes(): + return getattr(self, f"{attr}_{typ.attr_name}") + else: + return None + + def attr_comparator(cls, attr=attr): + return GuidComparator(getattr(cls, f"{attr}_guid")) + + def attr_setter( + self, value, attr=attr, attrs_to_expire=tuple(f"{attr}_{kls.attr_name}" for kls in OwnerType.object_classes()) + ): + typ = OwnerType.from_cls(value) + setattr(self, f"{attr}_guid", value.guid) + setattr(self, f"{attr}_type", typ.value) + # expire the relationships related to the attr + Session.object_session(self).expire(self, attrs_to_expire) + + attrs[f"{attr}"] = hybrid_property(attr_getter).setter(attr_setter).comparator(attr_comparator) + + # create relationships through xxx_guid + elif k.endswith("_guid"): + obj_name = k[:-5] + + kls_self = name + kls_other = utils.camelize(obj_name, uppercase_first_letter=True) + backref = utils.pluralize(utils.underscore(kls_self)) + + if kls_other == "Parent" and kls_self == "Account": + backref = sqlalchemy.orm.backref("parent", remote_side=[self_id]) + kls_other = kls_self + obj_name = "children" + if kls_other in {"RootAccount", "RootTemplate"}: + kls_other = "Account" + backref = None + if kls_other == "Currency": + kls_other = "Commodity" + backref = None + if kls_other == "TemplateAct": + kls_other = "Account" + if kls_other == "Tx": + kls_other = "Transaction" + + if obj_name == "ccard": # credit car account + kls_other = "Account" + if f"{obj_name}_type" in attrs: # x_guid to be used with x_type in Entries/Invoicing/... + print(tablename, obj_name) + for kls_other in cls.classes_with_guid: + kls_other = kls_other.__name__ + kwargs = dict( + argument=kls_other, + backref=None, + primaryjoin=f"foreign({kls_self}.{k}) == {kls_other}.guid", + viewonly=True, + ) + attrs[f"{obj_name}_{kls_other}"] = sqlalchemy.orm.relationship(**kwargs) + + continue + if obj_name == "obj" and kls_self in {"Recurrence", "Slot"}: + # done via ObjGUIDMixin + continue + + kwargs = dict( + argument=kls_other, + backref=backref, + primaryjoin=f"foreign({kls_self}.{k}) == {kls_other}.guid", + ) + + attrs[obj_name] = sqlalchemy.orm.relationship(**kwargs) + + self = super().__new__(cls, name, bases, attrs) + + return self + + +class DeclarativeBase: + def __hash__(self): + return hash((self.__class__.__name__, self.id)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and ((self.__class__.__name__, self.id)) == ((other.__class__.__name__, other.id)) + + def __repr__(self): + return f"{self.__class__.__name__}<{self._human_id()}>[{self.uid}]" + + def _human_id(self): + if hasattr(self, "name"): + return self.name + elif hasattr(self, "table_version"): + return self.table_version + return "" + + @property + def uid(self): + """Property to return the id if no Column with id already exists""" + try: + return self.guid + except AttributeError: + return self.id + + @property + def id(self): + """Property to return the id if no Column with id already exists""" + try: + return self.guid + except AttributeError: + return self.table_name + + +@lru_cache +def declarative_base(*args, **kwargs): + base = declarative_base_(*args, **kwargs, metaclass=DeclarativeMeta, cls=DeclarativeBase) + return base diff --git a/piecash2/schema/generation/schema_generation.py b/piecash2/schema/generation/schema_generation.py new file mode 100644 index 0000000..c4dd0b0 --- /dev/null +++ b/piecash2/schema/generation/schema_generation.py @@ -0,0 +1,133 @@ +"""Generate and import the SA schema from a gnucash sqlite file.""" + +import importlib.util +import sqlite3 +import sys +import textwrap +from datetime import datetime +from pathlib import Path + +# folder with code to insert into the schema generated by sqlacodegen +code_templates = Path(__file__).parent + +# folder in HOME to store the generated schemas +path_schemas = Path.home() / ".piecash2" / "schemas" +path_schemas.mkdir(exist_ok=True, parents=True) + + +def import_module_from_path(path: Path): + """Import a module from a path.""" + module_name = path.stem + spec = importlib.util.spec_from_file_location(module_name, path) + module = importlib.util.module_from_spec(spec) # type: ignore[arg-type] + sys.modules[module_name] = module + spec.loader.exec_module(module) # type: ignore[union-attr] + return module + + +def import_gnucash(book, regenerate_schema=False): + """Import the gnucash schema from the sqlite database as a module. If the schema does not exist, generate it.""" + modulename = get_schema_name(book) + + if not modulename.exists() or regenerate_schema: + generate_schema(book, schema_file=modulename) + + return import_module_from_path(modulename) + + +def generate_schema(db, schema_file: Path): + """Generate the schema from the sqlite database using sqlacodegen and copy the common files to the db folder.""" + + sys.argv = [ + "sqlacodegen_v2", + "--outfile", + str(schema_file), + f"sqlite:///{db.as_posix()}", + "--option", + "use_inflect", + ] + import sqlacodegen_v2.cli + + sqlacodegen_v2.cli.main() + + # insert before/after the generated schema the common schema + schema_file_text = ( + textwrap.dedent( + f""" + # -*- coding: utf-8 -*- + '''This file has been generated by piecash2.schema.generation.generate_schema on {datetime.now().isoformat()}. + ''' + """ + ) + + (code_templates / "sa_schema_pre.py").read_text() + + schema_file.read_text() + .replace( + "from sqlalchemy.ext.declarative import declarative_base as declarative_base_", + "from sqlalchemy.orm import declarative_base as declarative_base_", + ) + .replace( + "from sqlalchemy.orm import Mapped, declarative_base, mapped_column", + "from sqlalchemy.orm import mapped_column", + ) + .replace("from sqlalchemy.orm.base import Mapped", "") + + (code_templates / "sa_schema_post.py").read_text() + + "" + ) + + try: + # isort the imports if isort is available + import isort + + schema_file_text = isort.code(schema_file_text, float_to_top=True) + except ImportError: + pass + + try: + # reformat with black if black is available + import black + + schema_file_text = black.format_str(schema_file_text, mode=black.FileMode(line_length=140)) + except ImportError: + pass + + schema_file.write_text(schema_file_text) + + return schema_file_text + + +def get_version(db: Path): + """Return the table version of a given book file.""" + with sqlite3.connect(db) as conn: + c = conn.cursor() + c.execute("SELECT table_version FROM versions WHERE table_name=='Gnucash'") + (gnucash_version,) = c.fetchone() + + return gnucash_version + + +def get_schema_name(book: Path): + """Return the schema file name for a given book file.""" + v = get_version(book) + return path_schemas / f"book_schema_sqlite_{v}.py" + + +def add_book_module_in_path(book): + module_folder = str(get_schema_name(book).parent) + if module_folder not in sys.path: + sys.path.append(module_folder) + + +def unload_module(mod): + if mod in sys.modules: + del sys.modules[mod] + + +def remove_book_module_in_path(book): + module_path = get_schema_name(book) + module_folder = str(module_path.parent) + module_name = module_path.stem + + unload_module(module_name) + + if module_folder in sys.path: + sys.path.remove(module_folder) diff --git a/piecash2/utils.py b/piecash2/utils.py new file mode 100644 index 0000000..2cca0b9 --- /dev/null +++ b/piecash2/utils.py @@ -0,0 +1,30 @@ +import re + +import inflect + +inflect.def_classical["names"] = False +engine = inflect.engine() + + +def camelize(string, uppercase_first_letter=True): + """ + Convert strings to CamelCase. + """ + if uppercase_first_letter: + return re.sub(r"(?:^|_|-)(.)", lambda m: m.group(1).upper(), string) + else: + return string[0].lower() + camelize(string)[1:] + + +def underscore(word): + """ + Make an underscored, lowercase form from the expression in the string. + """ + word = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1_\2", word) + word = re.sub(r"([a-z\d])([A-Z])", r"\1_\2", word) + word = word.replace("-", "_") + return word.lower() + + +def pluralize(word): + return engine.plural_noun(word) diff --git a/requirements-test.txt b/requirements-test.txt index 11ba2ba..a5963ae 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -8,3 +8,4 @@ pytest-cov mypy gitchangelog mkdocs +typer diff --git a/requirements.txt b/requirements.txt index b05f2a6..6e09bd4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1 @@ -# This template is a low-dependency template. -# By default there is no requirements added here. -# Add the requirements you need to this file. -# or run `make init` to create this file automatically based on the template. -# You can also run `make switch-to-poetry` to use the poetry package manager. +sqlacodegen_v2 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..3354cbb --- /dev/null +++ b/setup.cfg @@ -0,0 +1,10 @@ +[flake8] +max-line-length = 140 + +[coverage:run] +omit = + sa_schema_post.py + sa_schema_pre.py + +[tool.isort] +profile = "black" diff --git a/setup.py b/setup.py index 9564ae9..c08c61b 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,3 @@ -"""Python setup.py for piecash2 package""" import io import os from setuptools import find_packages, setup @@ -22,25 +21,20 @@ def read(*paths, **kwargs): def read_requirements(path): - return [ - line.strip() - for line in read(path).split("\n") - if not line.startswith(('"', "#", "-", "git+")) - ] + return [line.strip() for line in read(path).split("\n") if not line.startswith(('"', "#", "-", "git+"))] setup( name="piecash2", version=read("piecash2", "VERSION"), - description="Awesome piecash2 created by sdementen", + description="piecash2, a python library to work with [GnuCash](https://www.gnucash.org/) books", url="https://github.com/sdementen/piecash2/", long_description=read("README.md"), long_description_content_type="text/markdown", author="sdementen", + maturity="Development Status :: 3 - Alpha", packages=find_packages(exclude=["tests", ".github"]), install_requires=read_requirements("requirements.txt"), - entry_points={ - "console_scripts": ["piecash2 = piecash2.__main__:main"] - }, + entry_points={"console_scripts": ["piecash2 = piecash2.__main__:main"]}, extras_require={"test": read_requirements("requirements-test.txt")}, ) diff --git a/tests/conftest.py b/tests/conftest.py index 1cbb7b1..037038f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,10 @@ import sys +from pathlib import Path + import pytest +HERE = Path(__file__).parent + # each test runs on cwd to its temp dir @pytest.fixture(autouse=True) @@ -12,3 +16,20 @@ def go_to_tmpdir(request): # Chdir only for the duration of the test. with tmpdir.as_cwd(): yield + + +@pytest.fixture(autouse=True) +def change_schema_path(request): + import piecash2.schema.generation.schema_generation as schema_generation + + # schema_generation.path_schemas = Path(request.getfixturevalue("tmpdir")) / "schemas" + # schema_generation.path_schemas.mkdir(exist_ok=True, parents=True) + + import piecash2.schema.generated as generated + + schema_generation.path_schemas = Path(generated.__file__).parent + + for book in (HERE.parent / "data").glob("*.gnucash"): + schema_generation.generate_schema(book, schema_generation.get_schema_name(book)) + + yield diff --git a/tests/test_base.py b/tests/test_base.py deleted file mode 100644 index c3fda5a..0000000 --- a/tests/test_base.py +++ /dev/null @@ -1,5 +0,0 @@ -from piecash2.base import NAME - - -def test_base(): - assert NAME == "piecash2" diff --git a/tests/test_core.py b/tests/test_core.py new file mode 100644 index 0000000..78eff82 --- /dev/null +++ b/tests/test_core.py @@ -0,0 +1,34 @@ +import pytest + +from piecash2 import open_book +from piecash2.schema.generation import schema_generation +from tests.test_schemas import path_data + + +def test_simple_query(): + book = path_data / "test book.gnucash" + + Session = open_book(book) + piecash = Session.module + print(piecash.__file__) + + with Session() as s: + for name, attr in piecash.__dict__.items(): + # print(name, type(attr)) + if isinstance(attr, type) and issubclass(attr, piecash.Base) and attr is not piecash.Base: + for obj in s.query(attr).all(): + print(obj) + assert len(list(s.query(attr).all())) >= 0 + + +@pytest.mark.parametrize( + "book", + [ + path_data / "test book.gnucash", + str(path_data / "test book.gnucash"), + ], +) +def test_open_book(book): + Session = open_book(book) + piecash = Session.module + assert piecash.__name__ == schema_generation.get_schema_name(book).stem diff --git a/tests/test_schemas.py b/tests/test_schemas.py new file mode 100644 index 0000000..311092b --- /dev/null +++ b/tests/test_schemas.py @@ -0,0 +1,78 @@ +import importlib +from pathlib import Path + +import pytest + +from piecash2.schema.generation import schema_generation + +path_data = Path(__file__).parent.parent / "data" + + +def test_get_schema_name(): + book = path_data / "test book.gnucash" + assert schema_generation.get_schema_name(book).is_relative_to(schema_generation.path_schemas) + assert schema_generation.get_schema_name(book).stem == "book_schema_sqlite_2060400" + + +@pytest.fixture +def change_schema_path(request): + import piecash2.schema.generation.schema_generation as schema_generation + + ops = schema_generation.path_schemas + + tmp_path = Path(request.getfixturevalue("tmpdir")) / "schemas" + tmp_path.mkdir(exist_ok=True, parents=True) + schema_generation.path_schemas = tmp_path + + yield + + schema_generation.path_schemas = ops + + +@pytest.mark.parametrize("book", list(path_data.glob("*.gnucash"))) +def test_import_dynamic(book): + schema_generation.import_gnucash(book) + + +@pytest.mark.parametrize("book", list(path_data.glob("*.gnucash"))) +def test_import_explicit(book): + module_name = schema_generation.get_schema_name(book).stem + importlib.import_module(f"piecash2.schema.generated.{module_name}") + + +@pytest.mark.parametrize("book", [path_data / "test book.gnucash"]) +def test_import_regenerate(book): + schema_generation.import_gnucash(book, regenerate_schema=True) + + +@pytest.mark.parametrize("book", [path_data / "test book.gnucash"]) +def test_add_book_module_in_path(book, change_schema_path): + importlib.invalidate_caches() + + if True: + path_module = schema_generation.get_schema_name(book) + module_name = path_module.stem + assert module_name.startswith("book_schema_sqlite_") + + # clean up existing file and unload module + if path_module.exists(): + path_module.unlink() + schema_generation.unload_module(module_name) + + # generate the schema + schema_generation.generate_schema(book, schema_file=path_module) + + # if True: + with pytest.raises(ImportError): + importlib.import_module(module_name) + + schema_generation.add_book_module_in_path(book) + + # import the module + importlib.import_module(module_name) + schema_generation.unload_module(module_name) + + schema_generation.remove_book_module_in_path(book) + + with pytest.raises(ImportError): + importlib.import_module(module_name) diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..0d8126f --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,27 @@ +import pytest + + +def test_camelize(): + from piecash2.utils import camelize + + assert camelize("hello_world") == "HelloWorld" + assert camelize("hello_world", uppercase_first_letter=False) == "helloWorld" + assert camelize("hello-world") == "HelloWorld" + assert camelize("hello-world", uppercase_first_letter=False) == "helloWorld" + + +def test_underscore(): + from piecash2.utils import underscore + + assert underscore("HelloWorld") == "hello_world" + assert underscore("helloWorld") == "hello_world" + assert underscore("hello-world") == "hello_world" + assert underscore("helloWorld") == "hello_world" + + +def test_pluralize(): + from piecash2.utils import pluralize + + assert pluralize("word") == "words" + assert pluralize("commodity") == "commodities" + assert pluralize("Commodity") == "Commodities"