From 294321eaec992a810207c58e14dd68e1c7529543 Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Sun, 18 Dec 2022 14:39:56 +0100 Subject: [PATCH 01/16] setup: increase flask-sqlalchemy version --- setup.cfg | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 849eb96..3c278b7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,7 +4,7 @@ # Copyright (C) 2015-2022 CERN. # Copyright (C) 2021 Northwestern University. # Copyright (C) 2022 RERO. -# Copyright (C) 2022 Graz University of Technology. +# Copyright (C) 2022-2023 Graz University of Technology. # # Invenio is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. @@ -32,7 +32,7 @@ install_requires = # due to incompatibility on the 1.11 release alembic>=1.10.0,<1.11.0 Flask-Alembic>=2.0.1 - Flask-SQLAlchemy>=2.1,<3.0.0 + Flask-SQLAlchemy>=3.0,<4.0.0 invenio-base>=1.2.10 SQLAlchemy-Continuum>=1.3.12 SQLAlchemy-Utils>=0.33.1,<0.39 @@ -44,7 +44,6 @@ tests = cryptography>=2.1.4 pytest-invenio>=1.4.5 Sphinx>=4.5.0 -# Left here for backward compatibility mysql = pymysql>=0.10.1 postgresql = From 07acd2e315c63e5e8d23fa99d2281afdcca5671d Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Sun, 18 Dec 2022 14:32:50 +0100 Subject: [PATCH 02/16] fix: tests LocalProxy * it is not working anymore with localproxy and it seams not necessary --- tests/test_db.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/test_db.py b/tests/test_db.py index 0b6db25..15363b0 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -335,8 +335,6 @@ def test_entry_points(db, app): def test_local_proxy(app, db): """Test local proxy filter.""" - from werkzeug.local import LocalProxy - InvenioDB(app, db=db) with app.app_context(): @@ -350,10 +348,10 @@ def test_local_proxy(app, db): ) result = db.engine.execute( query, - a=LocalProxy(lambda: "world"), - x=LocalProxy(lambda: 1), - y=LocalProxy(lambda: "2"), - z=LocalProxy(lambda: None), + a="world", + x=1, + y="2", + z=None, ).fetchone() assert result == (True, True, True, True) From 7cb181b0ba2b21858a901388838c6694a15ccfb9 Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Sun, 18 Dec 2022 14:38:25 +0100 Subject: [PATCH 03/16] change: remove click 3 compatibility --- invenio_db/cli.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/invenio_db/cli.py b/invenio_db/cli.py index 0660a23..dc6560c 100644 --- a/invenio_db/cli.py +++ b/invenio_db/cli.py @@ -8,10 +8,7 @@ """Click command-line interface for database management.""" -import sys - import click -from click import _termui_impl from flask import current_app from flask.cli import with_appcontext from sqlalchemy_utils.functions import create_database, database_exists, drop_database @@ -21,11 +18,6 @@ _db = LocalProxy(lambda: current_app.extensions["sqlalchemy"].db) -# Fix Python 3 compatibility issue in click -if sys.version_info > (3,): - _termui_impl.long = int # pragma: no cover - - def abort_if_false(ctx, param, value): """Abort command is value is False.""" if not value: From 1785799a648026f25acf63133137e6c167715291 Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Sun, 18 Dec 2022 14:44:49 +0100 Subject: [PATCH 04/16] change: add proxy file * use the proxy file instead of creating the _db variable in both files * this removes a DeprecationWarning from flask-sqlalchemy, which states that the support of the use '.db' will be removed --- invenio_db/cli.py | 26 ++++++++++++-------------- invenio_db/proxies.py | 15 +++++++++++++++ invenio_db/utils.py | 12 +++++------- 3 files changed, 32 insertions(+), 21 deletions(-) create mode 100644 invenio_db/proxies.py diff --git a/invenio_db/cli.py b/invenio_db/cli.py index dc6560c..86fa493 100644 --- a/invenio_db/cli.py +++ b/invenio_db/cli.py @@ -9,14 +9,12 @@ """Click command-line interface for database management.""" import click -from flask import current_app from flask.cli import with_appcontext from sqlalchemy_utils.functions import create_database, database_exists, drop_database -from werkzeug.local import LocalProxy +from .proxies import current_db from .utils import create_alembic_version_table, drop_alembic_version_table -_db = LocalProxy(lambda: current_app.extensions["sqlalchemy"].db) def abort_if_false(ctx, param, value): """Abort command is value is False.""" @@ -47,11 +45,11 @@ def db(): def create(verbose): """Create tables.""" click.secho("Creating all tables!", fg="yellow", bold=True) - with click.progressbar(_db.metadata.sorted_tables) as bar: + with click.progressbar(current_db.metadata.sorted_tables) as bar: for table in bar: if verbose: click.echo(" Creating table {0}".format(table)) - table.create(bind=_db.engine, checkfirst=True) + table.create(bind=current_db.engine, checkfirst=True) create_alembic_version_table() click.secho("Created all tables!", fg="green") @@ -69,11 +67,11 @@ def create(verbose): def drop(verbose): """Drop tables.""" click.secho("Dropping all tables!", fg="red", bold=True) - with click.progressbar(reversed(_db.metadata.sorted_tables)) as bar: + with click.progressbar(reversed(current_db.metadata.sorted_tables)) as bar: for table in bar: if verbose: click.echo(" Dropping table {0}".format(table)) - table.drop(bind=_db.engine, checkfirst=True) + table.drop(bind=current_db.engine, checkfirst=True) drop_alembic_version_table() click.secho("Dropped all tables!", fg="green") @@ -82,9 +80,9 @@ def drop(verbose): @with_appcontext def init(): """Create database.""" - displayed_database = render_url(_db.engine.url) + displayed_database = render_url(current_db.engine.url) click.secho(f"Creating database {displayed_database}", fg="green") - database_url = str(_db.engine.url) + database_url = str(current_db.engine.url) if not database_exists(database_url): create_database(database_url) @@ -100,12 +98,12 @@ def init(): @with_appcontext def destroy(): """Drop database.""" - displayed_database = render_url(_db.engine.url) + displayed_database = render_url(current_db.engine.url) click.secho(f"Destroying database {displayed_database}", fg="red", bold=True) - if _db.engine.name == "sqlite": + if current_db.engine.name == "sqlite": try: - drop_database(_db.engine.url) - except FileNotFoundError as e: + drop_database(current_db.engine.url) + except FileNotFoundError: click.secho("Sqlite database has not been initialised", fg="red", bold=True) else: - drop_database(_db.engine.url) + drop_database(current_db.engine.url) diff --git a/invenio_db/proxies.py b/invenio_db/proxies.py new file mode 100644 index 0000000..395ce72 --- /dev/null +++ b/invenio_db/proxies.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +# +# This file is part of Invenio. +# Copyright (C) 2022 Graz University of Technology. +# +# Invenio is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. + +"""Helper proxy to the state object.""" + + +from flask import current_app +from werkzeug.local import LocalProxy + +current_db = LocalProxy(lambda: current_app.extensions["sqlalchemy"]) diff --git a/invenio_db/utils.py b/invenio_db/utils.py index 7690d31..a1bee6f 100644 --- a/invenio_db/utils.py +++ b/invenio_db/utils.py @@ -11,12 +11,10 @@ from flask import current_app from sqlalchemy import inspect -from werkzeug.local import LocalProxy +from .proxies import current_db from .shared import db -_db = LocalProxy(lambda: current_app.extensions["sqlalchemy"].db) - def rebuild_encrypted_properties(old_key, model, properties): """Rebuild model's EncryptedType properties when the SECRET_KEY is changed. @@ -73,11 +71,11 @@ def create_alembic_version_table(): def drop_alembic_version_table(): """Drop alembic_version table.""" - if has_table(_db.engine, "alembic_version"): - alembic_version = _db.Table( - "alembic_version", _db.metadata, autoload_with=_db.engine + if has_table(current_db.engine, "alembic_version"): + alembic_version = current_db.Table( + "alembic_version", current_db.metadata, autoload_with=current_db.engine ) - alembic_version.drop(bind=_db.engine) + alembic_version.drop(bind=current_db.engine) def versioning_model_classname(manager, model): From 9bb8784704bedf78b69993418d67eaec5d01f203 Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Sun, 18 Dec 2022 22:46:09 +0100 Subject: [PATCH 05/16] fix: WeakKeyDictionary * the error indicates that db.init_app is not called, but it is and it works in other contexts. It may that the `db = invenio_db = shared.db` line in conftest.py is not working as expected anymore. The only found solution is this. The drawback is, if that is the only solution it has to be done also in the other packages which are using this function. With the default value it is backward compatible but as the tests indicates it may not work without adding the db parameter to the function call --- invenio_db/utils.py | 5 +++-- tests/test_utils.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/invenio_db/utils.py b/invenio_db/utils.py index a1bee6f..ef8925c 100644 --- a/invenio_db/utils.py +++ b/invenio_db/utils.py @@ -2,6 +2,7 @@ # # This file is part of Invenio. # Copyright (C) 2017-2018 CERN. +# Copyright (C) 2022 Graz University of Technology. # # Invenio is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. @@ -13,10 +14,10 @@ from sqlalchemy import inspect from .proxies import current_db -from .shared import db +from .shared import db as _db -def rebuild_encrypted_properties(old_key, model, properties): +def rebuild_encrypted_properties(old_key, model, properties, db=_db): """Rebuild model's EncryptedType properties when the SECRET_KEY is changed. :param old_key: old SECRET_KEY. diff --git a/tests/test_utils.py b/tests/test_utils.py index 18e444b..aa34f9c 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -50,13 +50,13 @@ class Demo(db.Model): with pytest.raises(ValueError): db.session.query(Demo).all() with pytest.raises(AttributeError): - rebuild_encrypted_properties(old_secret_key, Demo, ["nonexistent"]) + rebuild_encrypted_properties(old_secret_key, Demo, ["nonexistent"], db) assert app.secret_key == new_secret_key with app.app_context(): with pytest.raises(ValueError): db.session.query(Demo).all() - rebuild_encrypted_properties(old_secret_key, Demo, ["et"]) + rebuild_encrypted_properties(old_secret_key, Demo, ["et"], db) d1_after = db.session.query(Demo).first() assert d1_after.et == "something" From 8244aef22166e23c1c6e71a6d9ead65e80d6fc51 Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Sat, 25 Feb 2023 20:38:05 +0100 Subject: [PATCH 06/16] fix: remove warning and black problems * added D202, pydocstyle and black do not habe the same opinion * remove warning by using StringEncryptedType instead of EncryptedType. This removes a warning and there is further a notice in the code that the base type of EncryptedType will change in the future and it is better to replace it with StringEncryptedType --- setup.cfg | 2 +- tests/test_utils.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 3c278b7..de4de34 100644 --- a/setup.cfg +++ b/setup.cfg @@ -68,7 +68,7 @@ all_files = 1 universal = 1 [pydocstyle] -add_ignore = D401 +add_ignore = D401, D202 [isort] profile=black diff --git a/tests/test_utils.py b/tests/test_utils.py index aa34f9c..7e32f79 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -11,7 +11,7 @@ import pytest import sqlalchemy as sa from sqlalchemy_continuum import remove_versioning -from sqlalchemy_utils.types import EncryptedType +from sqlalchemy_utils.types import StringEncryptedType from invenio_db import InvenioDB from invenio_db.utils import ( @@ -33,7 +33,8 @@ class Demo(db.Model): __tablename__ = "demo" pk = db.Column(sa.Integer, primary_key=True) et = db.Column( - EncryptedType(type_in=db.Unicode, key=_secret_key), nullable=False + StringEncryptedType(type_in=db.Unicode, key=_secret_key), + nullable=False, ) InvenioDB(app, entry_point_group=False, db=db) From eb010d3e62b167d18db35209f144edee9e7e9efe Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Sat, 25 Feb 2023 21:28:44 +0100 Subject: [PATCH 07/16] fix: VARCHAR needs length on mysql --- tests/test_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 7e32f79..07810b9 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,6 +2,7 @@ # # This file is part of Invenio. # Copyright (C) 2017-2018 CERN. +# Copyright (C) 2023 Graz University of Technology. # # Invenio is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. @@ -33,7 +34,7 @@ class Demo(db.Model): __tablename__ = "demo" pk = db.Column(sa.Integer, primary_key=True) et = db.Column( - StringEncryptedType(type_in=db.Unicode, key=_secret_key), + StringEncryptedType(length=255, type_in=db.Unicode, key=_secret_key), nullable=False, ) From 6492e056c49aa7cff323c3051bced5226f0f1505 Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Sun, 12 Mar 2023 13:27:16 +0100 Subject: [PATCH 08/16] setup: remove upper pins --- setup.cfg | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/setup.cfg b/setup.cfg index de4de34..117436a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,14 +29,13 @@ packages = find: python_requires = >=3.7 zip_safe = False install_requires = - # due to incompatibility on the 1.11 release - alembic>=1.10.0,<1.11.0 + alembic>=1.10.0 Flask-Alembic>=2.0.1 - Flask-SQLAlchemy>=3.0,<4.0.0 + Flask-SQLAlchemy>=3.0 invenio-base>=1.2.10 SQLAlchemy-Continuum>=1.3.12 - SQLAlchemy-Utils>=0.33.1,<0.39 - SQLAlchemy[asyncio]>=1.2.18,<1.5.0 + SQLAlchemy-Utils>=0.33.1 + SQLAlchemy[asyncio]>=1.2.18 [options.extras_require] tests = From 0ba97c1c1ca1149764fd13be77f764601384fee3 Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Sun, 12 Mar 2023 13:28:30 +0100 Subject: [PATCH 09/16] refactor: move MockEntryPoint to independent file * this change makes it possible to import the MockEntryPoints from an independend file. this class is used in two tests but defined in one of them. placing it into a separated file makes it independent. --- tests/mocks.py | 51 ++++++++++++++++++++++++++++++++++++++++ tests/test_db.py | 41 +------------------------------- tests/test_versioning.py | 2 +- 3 files changed, 53 insertions(+), 41 deletions(-) create mode 100644 tests/mocks.py diff --git a/tests/mocks.py b/tests/mocks.py new file mode 100644 index 0000000..067f7ac --- /dev/null +++ b/tests/mocks.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# +# This file is part of Invenio. +# Copyright (C) 2023 Graz University of Technology. +# +# Invenio is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. + +"""Test database integration layer.""" + + +from importlib_metadata import EntryPoint +from werkzeug.utils import import_string + + +class MockEntryPoint(EntryPoint): + """Mocking of entrypoint.""" + + def load(self): + """Mock load entry point.""" + if self.name == "importfail": + raise ImportError() + else: + return import_string(self.name) + + +def _mock_entry_points(name): + def fn(group): + data = { + "invenio_db.models": [ + MockEntryPoint(name="demo.child", value="demo.child", group="test"), + MockEntryPoint(name="demo.parent", value="demo.parent", group="test"), + ], + "invenio_db.models_a": [ + MockEntryPoint( + name="demo.versioned_a", value="demo.versioned_a", group="test" + ), + ], + "invenio_db.models_b": [ + MockEntryPoint( + name="demo.versioned_b", value="demo.versioned_b", group="test" + ), + ], + } + if group: + return data.get(group, []) + if name: + return {name: data.get(name)} + return data + + return fn diff --git a/tests/test_db.py b/tests/test_db.py index 15363b0..04d0dae 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -14,56 +14,17 @@ import pytest import sqlalchemy as sa from flask import Flask -from importlib_metadata import EntryPoint +from mocks import _mock_entry_points from sqlalchemy import inspect from sqlalchemy.exc import IntegrityError from sqlalchemy_continuum import VersioningManager, remove_versioning from sqlalchemy_utils.functions import create_database, drop_database -from werkzeug.utils import import_string from invenio_db import InvenioDB, shared from invenio_db.cli import db as db_cmd from invenio_db.utils import drop_alembic_version_table, has_table -class MockEntryPoint(EntryPoint): - """Mocking of entrypoint.""" - - def load(self): - """Mock load entry point.""" - if self.name == "importfail": - raise ImportError() - else: - return import_string(self.name) - - -def _mock_entry_points(name): - def fn(group): - data = { - "invenio_db.models": [ - MockEntryPoint(name="demo.child", value="demo.child", group="test"), - MockEntryPoint(name="demo.parent", value="demo.parent", group="test"), - ], - "invenio_db.models_a": [ - MockEntryPoint( - name="demo.versioned_a", value="demo.versioned_a", group="test" - ), - ], - "invenio_db.models_b": [ - MockEntryPoint( - name="demo.versioned_b", value="demo.versioned_b", group="test" - ), - ], - } - if group: - return data.get(group, []) - if name: - return {name: data.get(name)} - return data - - return fn - - def test_init(db, app): """Test extension initialization.""" diff --git a/tests/test_versioning.py b/tests/test_versioning.py index 8232da0..b804b5e 100644 --- a/tests/test_versioning.py +++ b/tests/test_versioning.py @@ -12,8 +12,8 @@ from unittest.mock import patch import pytest +from mocks import _mock_entry_points from sqlalchemy_continuum import VersioningManager, remove_versioning -from test_db import _mock_entry_points from invenio_db import InvenioDB From e382de9456aeb3fb2b318e39970e0c69c3d0a1bd Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Thu, 16 Nov 2023 09:48:37 +0100 Subject: [PATCH 10/16] cli: password is per default hided * this leads to the problem that the program couldn't connect to the database --- invenio_db/cli.py | 14 ++++++-------- tests/test_db.py | 4 ++-- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/invenio_db/cli.py b/invenio_db/cli.py index 86fa493..18f288f 100644 --- a/invenio_db/cli.py +++ b/invenio_db/cli.py @@ -2,6 +2,7 @@ # # This file is part of Invenio. # Copyright (C) 2015-2018 CERN. +# Copyright (C) 2024 Graz University of Technology. # # Invenio is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. @@ -24,11 +25,7 @@ def abort_if_false(ctx, param, value): def render_url(url): """Render the URL for CLI output.""" - try: - return url.render_as_string(hide_password=True) - except AttributeError: - # SQLAlchemy <1.4 - return url.__to_string__(hide_password=True) + return url.render_as_string(hide_password=True) # @@ -82,7 +79,7 @@ def init(): """Create database.""" displayed_database = render_url(current_db.engine.url) click.secho(f"Creating database {displayed_database}", fg="green") - database_url = str(current_db.engine.url) + database_url = current_db.engine.url.render_as_string(hide_password=False) if not database_exists(database_url): create_database(database_url) @@ -100,10 +97,11 @@ def destroy(): """Drop database.""" displayed_database = render_url(current_db.engine.url) click.secho(f"Destroying database {displayed_database}", fg="red", bold=True) + plain_url = current_db.engine.url.render_as_string(hide_password=False) if current_db.engine.name == "sqlite": try: - drop_database(current_db.engine.url) + drop_database(plain_url) except FileNotFoundError: click.secho("Sqlite database has not been initialised", fg="red", bold=True) else: - drop_database(current_db.engine.url) + drop_database(plain_url) diff --git a/tests/test_db.py b/tests/test_db.py index 04d0dae..92a76d7 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -363,6 +363,6 @@ def test_db_create_alembic_upgrade(app, db): assert len(inspect(db.engine).get_table_names()) == 0 finally: - drop_database(str(db.engine.url)) + drop_database(str(db.engine.url.render_as_string(hide_password=False))) remove_versioning(manager=ext.versioning_manager) - create_database(str(db.engine.url)) + create_database(str(db.engine.url.render_as_string(hide_password=False))) From 3846460f25bfb0aaf8539fa1715328fd3d1f1191 Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Sun, 4 Aug 2024 23:38:05 +0200 Subject: [PATCH 11/16] setup: increase min sqlalchemy dependency * apply new correct return values to cli calls * refactor code * remove unused code --- .../dbdbc1b19cf2_create_transaction_table.py | 2 +- invenio_db/ext.py | 20 +++---- setup.cfg | 5 +- tests/conftest.py | 4 +- tests/test_db.py | 59 +++++++++++++++---- tests/test_versioning.py | 2 +- 6 files changed, 64 insertions(+), 28 deletions(-) diff --git a/invenio_db/alembic/dbdbc1b19cf2_create_transaction_table.py b/invenio_db/alembic/dbdbc1b19cf2_create_transaction_table.py index cf58a1c..8f83cb6 100644 --- a/invenio_db/alembic/dbdbc1b19cf2_create_transaction_table.py +++ b/invenio_db/alembic/dbdbc1b19cf2_create_transaction_table.py @@ -23,9 +23,9 @@ def upgrade(): """Update database.""" op.create_table( "transaction", - sa.Column("issued_at", sa.DateTime(), nullable=True), sa.Column("id", sa.BigInteger(), nullable=False), sa.Column("remote_addr", sa.String(length=50), nullable=True), + sa.Column("issued_at", sa.DateTime(), nullable=True), ) op.create_primary_key("pk_transaction", "transaction", ["id"]) if op._proxy.migration_context.dialect.supports_sequences: diff --git a/invenio_db/ext.py b/invenio_db/ext.py index 3e7f9c2..fda1a0f 100644 --- a/invenio_db/ext.py +++ b/invenio_db/ext.py @@ -36,19 +36,17 @@ def init_app(self, app, **kwargs): """Initialize application object.""" self.init_db(app, **kwargs) - script_location = str(importlib_resources.files("invenio_db") / "alembic") - version_locations = [ - ( - base_entry.name, - str( - importlib_resources.files(base_entry.module) - / os.path.join(base_entry.attr) - ), - ) - for base_entry in importlib_metadata.entry_points( - group="invenio_db.alembic" + def pathify(base_entry): + return str( + importlib_resources.files(base_entry.module) + / os.path.join(base_entry.attr) ) + + entry_points = importlib_metadata.entry_points(group="invenio_db.alembic") + version_locations = [ + (base_entry.name, pathify(base_entry)) for base_entry in entry_points ] + script_location = str(importlib_resources.files("invenio_db") / "alembic") app.config.setdefault( "ALEMBIC", { diff --git a/setup.cfg b/setup.cfg index 117436a..519649c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,11 +35,12 @@ install_requires = invenio-base>=1.2.10 SQLAlchemy-Continuum>=1.3.12 SQLAlchemy-Utils>=0.33.1 - SQLAlchemy[asyncio]>=1.2.18 + SQLAlchemy[asyncio]>=2.0.0 [options.extras_require] tests = - pytest-black>=0.3.0 + six>=1.0.0 + pytest-black-ng>=0.4.0 cryptography>=2.1.4 pytest-invenio>=1.4.5 Sphinx>=4.5.0 diff --git a/tests/conftest.py b/tests/conftest.py index 83d0024..6a2ec02 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,8 +17,8 @@ from invenio_db.utils import alembic_test_context -@pytest.fixture() -def db(): +@pytest.fixture(name="db") +def fixture_db(): """Database fixture with session sharing.""" import invenio_db from invenio_db import shared diff --git a/tests/test_db.py b/tests/test_db.py index 92a76d7..90afd1b 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -3,6 +3,7 @@ # This file is part of Invenio. # Copyright (C) 2015-2018 CERN. # Copyright (C) 2022 RERO. +# Copyright (C) 2024 Graz University of Technology. # # Invenio is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. @@ -20,8 +21,9 @@ from sqlalchemy_continuum import VersioningManager, remove_versioning from sqlalchemy_utils.functions import create_database, drop_database -from invenio_db import InvenioDB, shared +from invenio_db import InvenioDB from invenio_db.cli import db as db_cmd +from invenio_db.shared import NAMING_CONVENTION, MetaData, SQLAlchemy from invenio_db.utils import drop_alembic_version_table, has_table @@ -81,9 +83,8 @@ def test_alembic(db, app): def test_naming_convention(db, app): """Test naming convention.""" - from sqlalchemy_continuum import remove_versioning - ext = InvenioDB(app, entry_point_group=False, db=db) + InvenioDB(app, entry_point_group=False, db=db) cfg = dict( DB_VERSIONING=True, DB_VERSIONING_USER_MODEL=None, @@ -119,8 +120,8 @@ class Slave(base): return Master, Slave - source_db = shared.SQLAlchemy( - metadata=shared.MetaData( + source_db = SQLAlchemy( + metadata=MetaData( naming_convention={ "ix": "source_ix_%(table_name)s_%(column_0_label)s", "uq": "source_uq_%(table_name)s_%(column_0_name)s", @@ -158,9 +159,7 @@ class Slave(base): remove_versioning(manager=source_ext.versioning_manager) - target_db = shared.SQLAlchemy( - metadata=shared.MetaData(naming_convention=shared.NAMING_CONVENTION) - ) + target_db = SQLAlchemy(metadata=MetaData(naming_convention=NAMING_CONVENTION)) target_app = Flask("target_app") target_app.config.update(**cfg) @@ -181,7 +180,7 @@ class Slave(base): target_constraints = set( [ cns.name - for model in source_models + for model in target_models for cns in list(model.__table__.constraints) + list(model.__table__.indexes) ] @@ -263,6 +262,9 @@ def test_entry_points(db, app): result = runner.invoke(db_cmd, []) assert result.exit_code == 0 + result = runner.invoke(db_cmd, ["init"]) + assert result.exit_code == 0 + result = runner.invoke(db_cmd, ["destroy", "--yes-i-know"]) assert result.exit_code == 0 @@ -272,6 +274,21 @@ def test_entry_points(db, app): result = runner.invoke(db_cmd, ["create", "-v"]) assert result.exit_code == 0 + result = runner.invoke(db_cmd, ["destroy", "--yes-i-know"]) + assert result.exit_code == 0 + + result = runner.invoke(db_cmd, ["init"]) + assert result.exit_code == 0 + + result = runner.invoke(db_cmd, ["create", "-v"]) + assert result.exit_code == 1 + + result = runner.invoke(db_cmd, ["create", "-v"]) + assert result.exit_code == 1 + + result = runner.invoke(db_cmd, ["drop", "-v", "--yes-i-know"]) + assert result.exit_code == 0 + result = runner.invoke(db_cmd, ["drop"]) assert result.exit_code == 1 @@ -282,10 +299,19 @@ def test_entry_points(db, app): assert result.exit_code == 1 result = runner.invoke(db_cmd, ["drop", "--yes-i-know", "create"]) + assert result.exit_code == 1 + + result = runner.invoke(db_cmd, ["destroy", "--yes-i-know"]) assert result.exit_code == 0 - result = runner.invoke(db_cmd, ["destroy"]) - assert result.exit_code == 1 + result = runner.invoke(db_cmd, ["init"]) + assert result.exit_code == 0 + + result = runner.invoke(db_cmd, ["destroy", "--yes-i-know"]) + assert result.exit_code == 0 + + result = runner.invoke(db_cmd, ["init"]) + assert result.exit_code == 0 result = runner.invoke(db_cmd, ["destroy", "--yes-i-know"]) assert result.exit_code == 0 @@ -293,6 +319,15 @@ def test_entry_points(db, app): result = runner.invoke(db_cmd, ["init"]) assert result.exit_code == 0 + result = runner.invoke(db_cmd, ["drop", "-v", "--yes-i-know"]) + assert result.exit_code == 1 + + result = runner.invoke(db_cmd, ["create", "-v"]) + assert result.exit_code == 1 + + result = runner.invoke(db_cmd, ["drop", "-v", "--yes-i-know"]) + assert result.exit_code == 0 + def test_local_proxy(app, db): """Test local proxy filter.""" @@ -339,11 +374,13 @@ def test_db_create_alembic_upgrade(app, db): assert result.exit_code == 0 assert has_table(db.engine, "transaction") assert ext.alembic.migration_context._has_version_table() + # Note that compare_metadata does not detect additional sequences # and constraints. # TODO fix failing test on mysql if db.engine.name != "mysql": assert not ext.alembic.compare_metadata() + ext.alembic.upgrade() assert has_table(db.engine, "transaction") diff --git a/tests/test_versioning.py b/tests/test_versioning.py index b804b5e..aabdf56 100644 --- a/tests/test_versioning.py +++ b/tests/test_versioning.py @@ -3,6 +3,7 @@ # This file is part of Invenio. # Copyright (C) 2015-2018 CERN. # Copyright (C) 2022 RERO. +# Copyright (C) 2024 Graz University of Technology. # # Invenio is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. @@ -45,7 +46,6 @@ class EarlyClass(db.Model): db.drop_all() db.create_all() - before = len(db.metadata.tables) ec = EarlyClass() ec.pk = 1 db.session.add(ec) From a4dad4309b64c5f2ce02aeed2031b57a996beefa Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Sun, 4 Aug 2024 23:32:45 +0200 Subject: [PATCH 12/16] ci: change to reusable workflows --- .github/workflows/tests.yml | 66 ++++++++----------------------------- 1 file changed, 13 insertions(+), 53 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b36afa5..5a0e288 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,7 +2,7 @@ # # This file is part of Invenio. # Copyright (C) 2020-2024 CERN. -# Copyright (C) 2022 Graz University of Technology. +# Copyright (C) 2022-2024 Graz University of Technology. # # Invenio is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. @@ -11,65 +11,25 @@ name: CI on: push: - branches: master + branches: + - master pull_request: - branches: master + branches: + - master schedule: # * is a special character in YAML so you have to quote this string - - cron: '0 3 * * 6' + - cron: "0 3 * * 6" workflow_dispatch: inputs: reason: - description: 'Reason' + description: "Reason" required: false - default: 'Manual trigger' - + default: "Manual trigger" + jobs: Tests: - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - python-version: [3.9, 3.12] - db-service: [postgresql11, postgresql14, mysql8, sqlite] - - include: - - db-service: postgresql11 - EXTRAS: "tests,postgresql" - - - db-service: postgresql14 - EXTRAS: "tests,postgresql" - - - db-service: mysql8 - EXTRAS: "tests,mysql" - - - db-service: sqlite - EXTRAS: "tests" - - env: - DB: ${{ matrix.db-service }} - EXTRAS: ${{ matrix.EXTRAS }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: pip - cache-dependency-path: setup.cfg - - - name: Pre-install - uses: ./.github/actions/pre-install - if: ${{ hashFiles('.github/actions/pre-install/action.yml') != '' }} - - - name: Install dependencies - run: | - pip install ".[$EXTRAS]" - pip freeze - docker version + uses: inveniosoftware/workflows/.github/workflows/tests-python.yml@master + with: + extras: "tests,postgresql" + search-service: '[""]' - - name: Run tests - run: ./run-tests.sh From 7fbb0421ca9bf4bb0af4def8860f0863650fb9e0 Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Tue, 1 Oct 2024 22:29:57 +0200 Subject: [PATCH 13/16] setup: increase flask-alembic * flask-alembic >= 3.0 needs flask >= 3.0 and flask-sqlalchemy >= 3.0 * added some explanation comments * apply db.session.query over Model.query syntax --- setup.cfg | 4 ++-- tests/test_db.py | 5 +++-- tests/test_versioning.py | 2 ++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index 519649c..523679d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,7 +4,7 @@ # Copyright (C) 2015-2022 CERN. # Copyright (C) 2021 Northwestern University. # Copyright (C) 2022 RERO. -# Copyright (C) 2022-2023 Graz University of Technology. +# Copyright (C) 2022-2024 Graz University of Technology. # # Invenio is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. @@ -30,7 +30,7 @@ python_requires = >=3.7 zip_safe = False install_requires = alembic>=1.10.0 - Flask-Alembic>=2.0.1 + Flask-Alembic>=3.0.0 Flask-SQLAlchemy>=3.0 invenio-base>=1.2.10 SQLAlchemy-Continuum>=1.3.12 diff --git a/tests/test_db.py b/tests/test_db.py index 90afd1b..a0f2385 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -62,8 +62,8 @@ class Demo2(db.Model): db.session.rollback() with app.app_context(): - Demo2.query.delete() - Demo.query.delete() + db.session.query(Demo2).delete() + db.session.query(Demo).delete() db.session.commit() db.drop_all() @@ -366,6 +366,7 @@ def test_db_create_alembic_upgrade(app, db): try: if db.engine.name == "sqlite": raise pytest.skip("Upgrades are not supported on SQLite.") + db.drop_all() runner = app.test_cli_runner() # Check that 'db create' creates the same schema as diff --git a/tests/test_versioning.py b/tests/test_versioning.py index aabdf56..b268f04 100644 --- a/tests/test_versioning.py +++ b/tests/test_versioning.py @@ -33,6 +33,8 @@ def test_disabled_versioning_with_custom_table(db, app, versioning, tables): """Test SQLAlchemy-Continuum table loading.""" app.config["DB_VERSIONING"] = versioning + # this class has to be defined here, because the the db has to be the db + # from the fixture. using it "from invenio_db import db" is not working class EarlyClass(db.Model): __versioned__ = {} From 0fe66b07a6b1b98dd29abec3624f3b6a4acbcf44 Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Sat, 9 Nov 2024 22:41:50 +0100 Subject: [PATCH 14/16] fix: select of BinaryExpressions * sqlalchemy.exc.ArgumentError: Column expression, FROM clause, or other columns clause element expected, got [, ]. Did you mean to say select(, ) don't give the array to the select but give the elements to the select method * AttributeError: 'Engine' object has no attribute 'execute' -> change to session * TypeError: scoped_session.execute() got an unexpected keyword argument 'a' -> change to dict --- tests/test_db.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/test_db.py b/tests/test_db.py index a0f2385..540df4c 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -335,19 +335,21 @@ def test_local_proxy(app, db): with app.app_context(): query = db.select( - [ + *[ db.literal("hello") != db.bindparam("a"), db.literal(0) <= db.bindparam("x"), db.literal("2") == db.bindparam("y"), db.literal(None).is_(db.bindparam("z")), ] ) - result = db.engine.execute( + result = db.session.execute( query, - a="world", - x=1, - y="2", - z=None, + { + "a": "world", + "x": 1, + "y": "2", + "z": None, + }, ).fetchone() assert result == (True, True, True, True) From 2d220f1fb54814381c706d298f996da79ede6321 Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Tue, 12 Nov 2024 23:12:33 +0100 Subject: [PATCH 15/16] uow: possible solution for the rollback problem * the rollback should only rollback whats in that savepoint. using session.begin_nested creates a subtransaction which can then rolledback --- invenio_db/uow.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/invenio_db/uow.py b/invenio_db/uow.py index 4b80fd4..7e585ea 100644 --- a/invenio_db/uow.py +++ b/invenio_db/uow.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # # Copyright (C) 2024 CERN. +# Copyright (C) 2024 Graz University of Technology. # # Invenio is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. @@ -167,6 +168,7 @@ def __init__(self, session=None): def __enter__(self): """Entering the context.""" + self.session.begin_nested() return self def __exit__(self, exc_type, exc_value, traceback): From db26fa3b5bee4a21fe6c18504eb057a727b327bc Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Tue, 19 Nov 2024 11:25:34 +0100 Subject: [PATCH 16/16] release: v2.0.0 --- CHANGES.rst | 18 ++++++++++++++++++ invenio_db/__init__.py | 3 ++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 72c6698..82c65b6 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,7 @@ .. This file is part of Invenio. Copyright (C) 2015-2024 CERN. + Copyright (C) 2024 Graz University of Technology. Invenio is free software; you can redistribute it and/or modify it under the terms of the MIT License; see LICENSE file for more details. @@ -8,6 +9,23 @@ Changes ======= +Version v2.0.0 (released 2024-11-19) + +- uow: possible solution for the rollback problem +- fix: select of BinaryExpressions +- setup: increase flask-sqlalchemy +- ci: change to reusable workflows +- setup: increase min sqlalchemy dependency +- cli: password is per default hided +- refactor: move MockEntryPoint to independent file +- setup: remove upper pins +- fix: remove warning and black problems +- fix: WeakKeyDictionary +- change: add proxy file +- change: remove click 3 compatibility +- fix: tests LocalProxy +- setup: increase flask-sqlalchemy version + Version v1.3.1 (released 2024-11-14) - uow: improve decorator to create uow if it's None diff --git a/invenio_db/__init__.py b/invenio_db/__init__.py index 2511f09..0269ab5 100644 --- a/invenio_db/__init__.py +++ b/invenio_db/__init__.py @@ -2,6 +2,7 @@ # # This file is part of Invenio. # Copyright (C) 2015-2024 CERN. +# Copyright (C) 2024 Graz University of Technology. # # Invenio is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. @@ -91,7 +92,7 @@ class User(db.Model): from .ext import InvenioDB from .shared import db -__version__ = "1.3.1" +__version__ = "2.0.0" __all__ = ( "__version__",