diff --git a/README.rst b/README.rst index c1baf4b..0609d68 100644 --- a/README.rst +++ b/README.rst @@ -115,7 +115,7 @@ code: .. code-block:: python - from sqlalchemy_mate import EngineCreator + from sqlalchemy_mate.api import EngineCreator ec = EngineCreator.from_json( json_file="path-to-json-file", @@ -149,7 +149,7 @@ You can put lots of database connection info in a ``.db.json`` file in your ``$H .. code-block:: python - from sqlalchemy_mate import EngineCreator + from sqlalchemy_mate.api import EngineCreator ec = EngineCreator.from_home_db_json(identifier="db1") engine = ec.create_postgresql_psycopg2() @@ -179,7 +179,7 @@ This is similar to ``from_json``, but the json file is stored on AWS S3. .. code-block:: python - from sqlalchemy_mate import EngineCreator + from sqlalchemy_mate.api import EngineCreator ec = EngineCreator.from_s3_json( bucket_name="my-bucket", key="db.json", json_path="identifier1", @@ -203,7 +203,7 @@ You can put your credentials in Environment Variable. For example: .. code-block:: python - from sqlalchemy_mate import EngineCreator + from sqlalchemy_mate.api import EngineCreator # read from DB_DEV_USERNAME, DB_DEV_PASSWORD, ... ec = EngineCreator.from_env(prefix="DB_DEV") engine = ec.create_redshift() @@ -239,7 +239,7 @@ With sql expression: .. code-block:: python - from sqlalchemy_mate import inserting + from sqlalchemy_mate.api import inserting engine = create_engine(...) t_users = Table( "users", metadata, @@ -256,7 +256,7 @@ With ORM: .. code-block:: python - from sqlalchemy_mate import ExtendedBase + from sqlalchemy_mate.api import ExtendedBase Base = declarative_base() class User(Base, ExtendedBase): # inherit from ExtendedBase ... @@ -274,7 +274,7 @@ Automatically update value by primary key. .. code-block:: python # in SQL expression - from sqlalchemy_mate import updating + from sqlalchemy_mate.api import updating data = [{"id": 1, "name": "Alice}, {"id": 2, "name": "Bob"}, ...] updating.update_all(engine, table, data) diff --git a/docs/source/01-Core-API/index.rst b/docs/source/01-Core-API/index.rst index 16a7a92..4ee0c1c 100644 --- a/docs/source/01-Core-API/index.rst +++ b/docs/source/01-Core-API/index.rst @@ -44,7 +44,7 @@ We want to insert 3 random user data into the database and do some basic query: .. code-block:: python - import sqlalchemy_mate as sam + import sqlalchemy_mate.api as sam # do bulk insert sam.inserting.smart_insert(engine, t_users, user_data_list) diff --git a/docs/source/02-ORM-API/index.rst b/docs/source/02-ORM-API/index.rst index 958f8e3..cc421ed 100644 --- a/docs/source/02-ORM-API/index.rst +++ b/docs/source/02-ORM-API/index.rst @@ -12,7 +12,7 @@ Extended Declarative Base import sqlalchemy as sa import sqlalchemy.orm as orm - import sqlalchemy_mate as sam + import sqlalchemy_mate.api as sam Base = orm.declarative_base() diff --git a/docs/source/03-Other-Helpers/index.rst b/docs/source/03-Other-Helpers/index.rst index 53f26b2..4b0f1ac 100644 --- a/docs/source/03-Other-Helpers/index.rst +++ b/docs/source/03-Other-Helpers/index.rst @@ -12,7 +12,7 @@ User Friendly Engine Creator .. code-block:: python - import sqlalchemy_mate as sam + import sqlalchemy_mate.api as sam # An Postgres DB example # First, you use EngineCreator class to create the db connection specs @@ -48,7 +48,7 @@ First let's insert some sample data: import sqlalchemy as sa from sqlalchemy.orm import declarative_base - import sqlalchemy_mate as sam + import sqlalchemy_mate.api as sam Base = declarative_base() @@ -81,7 +81,7 @@ First let's insert some sample data: .. code-block:: python - import sqlalchemy_mate as sam + import sqlalchemy_mate.api as sam # from ORM class print(sam.pt.from_everything(User, engine)) diff --git a/docs/source/05-Patterns/Status-Tracker/index.ipynb b/docs/source/05-Patterns/Status-Tracker/index.ipynb index 394d3d2..7f97373 100644 --- a/docs/source/05-Patterns/Status-Tracker/index.ipynb +++ b/docs/source/05-Patterns/Status-Tracker/index.ipynb @@ -252,7 +252,7 @@ "\n", "The ``Job.start()`` class method is a magic context manager that does a lot of things.\n", "\n", - "1. It try to obtain lock before the job begin. Once we have obtained the lock, other work won't be able to update this items (they will see that it is locked).\n", + "1. Try to obtain lock before the job begin. Once we have obtained the lock, other work won't be able to update this row (they will see that it is locked).\n", "2. Any raised exception will be captured by the context manager, and it will set the status as ``failed``, add retry count, log the error (and save the error information to DB), and release the lock.\n", "3. If the job has been failed too many times, it will set the status as ``ignored``.\n", "4. If everything goes well, it will set status as ``succeeded`` and apply updates." diff --git a/examples/e1_core_api.py b/examples/e1_core_api.py index f6bed84..0c9e8ac 100644 --- a/examples/e1_core_api.py +++ b/examples/e1_core_api.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import sqlalchemy as sa -import sqlalchemy_mate as sam +import sqlalchemy_mate.api as sam metadata = sa.MetaData() diff --git a/examples/e2_orm_api.py b/examples/e2_orm_api.py index f345215..8caa133 100644 --- a/examples/e2_orm_api.py +++ b/examples/e2_orm_api.py @@ -2,7 +2,7 @@ import sqlalchemy as sa from sqlalchemy.orm import declarative_base, Session -import sqlalchemy_mate as sam +import sqlalchemy_mate.api as sam Base = declarative_base() diff --git a/examples/e31_engine_creator.py b/examples/e31_engine_creator.py index 050f313..09bc075 100644 --- a/examples/e31_engine_creator.py +++ b/examples/e31_engine_creator.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -import sqlalchemy_mate as sam +import sqlalchemy_mate.api as sam engine_sqlite = sam.EngineCreator().create_sqlite(path="/tmp/db.sqlite") sam.test_connection(engine_sqlite, timeout=1) diff --git a/examples/e32_pretty_table.py b/examples/e32_pretty_table.py index 0316ab0..29784b1 100644 --- a/examples/e32_pretty_table.py +++ b/examples/e32_pretty_table.py @@ -2,7 +2,7 @@ import sqlalchemy as sa from sqlalchemy.orm import declarative_base -import sqlalchemy_mate as sam +import sqlalchemy_mate.api as sam Base = declarative_base() diff --git a/release-history.rst b/release-history.rst index 1ef1b3d..1557a1d 100644 --- a/release-history.rst +++ b/release-history.rst @@ -15,12 +15,23 @@ Backlog (TODO) **Miscellaneous** -2.0.0.0 (TODO) +2.0.0.1 (TODO) +------------------------------------------------------------------------------ +**💥Breaking Change** + +- Rework the public API import. Now you have to use ``import sqlalchemy_mate.api as sm`` to access the public API. ``from sqlalchemy_mate import ...`` is no longer working. + +**Features and Improvements** + +- Add status tracker pattern. + + +2.0.0.0 (2024-05-15) ------------------------------------------------------------------------------ **💥Breaking Change** - From ``sqlalchemy_mate>=2.0.0.0``, it only support ``sqlalchemy>=2.0.0`` and only compatible with sqlalchemy 2.X API. Everything marked as ``no longer supported`` or ``no longer accepted`` in `SQLAlchemy 2.0 - Major Migration Guide `_ document will no longer be supported from this version. -- Drop Python3.7 support. +- Drop Python3.7 support. Now it only support 3.8+. **Features and Improvements** diff --git a/sqlalchemy_mate/patterns/status_tracker/impl.py b/sqlalchemy_mate/patterns/status_tracker/impl.py index 87b0764..fe22bf8 100644 --- a/sqlalchemy_mate/patterns/status_tracker/impl.py +++ b/sqlalchemy_mate/patterns/status_tracker/impl.py @@ -1,13 +1,11 @@ # -*- coding: utf-8 -*- - import typing as T -import enum import uuid import traceback import dataclasses from contextlib import contextmanager -from datetime import datetime, timezone +from datetime import datetime import sqlalchemy as sa import sqlalchemy.orm as orm @@ -17,15 +15,31 @@ class JobLockedError(Exception): + """ + Raised when try to start a locked job. + """ + pass class JobIgnoredError(Exception): + """ + Raised when try to start a ignored (failed too many times) job. + """ + pass class JobMixin: """ + The sqlalchemy ORM data model mixin class that brings in status tracking + related features. Core API includes: + + - :meth:`JobMixin.create` + - :meth:`JobMixin.create_and_save` + - :meth:`JobMixin.start` + - :meth:`JobMixin.query_by_status` + See: https://docs.sqlalchemy.org/en/20/orm/declarative_mixins.html **锁机制** @@ -66,6 +80,18 @@ def create( data: T.Optional[dict] = None, **kwargs, ): + """ + Create an in-memory instance of the job object. This method won't write + the job to database. This is useful for initializing many new jobs in batch. + + Usage example:: + + with orm.Session(engine) as ses: + for job_id in job_id_list: + job = Job.create(id=job_id, status=10) + ses.add(job) + ses.commit() + """ utc_now = datetime.utcnow() return cls( id=id, @@ -103,6 +129,14 @@ def create_and_save( data: T.Optional[dict] = None, **kwargs, ): + """ + Create an instance of the job object and write the job to database. + + Usage example:: + + with orm.Session(engine) as ses: + Job.create_and_save(ses, id="job-1", status=10) + """ if isinstance(engine_or_session, sa.Engine): with orm.Session(engine_or_session) as ses: return cls._create_and_save( @@ -224,6 +258,36 @@ def start( debug: bool = False, ) -> T.ContextManager[T.Tuple["T_JOB", "Updates"]]: """ + This is the most important API. A context manager that does a lot of things: + + 1. Try to obtain lock before the job begin. Once we have obtained the lock, + other work won't be able to update this row (they will see that it is locked). + 2. Any raised exception will be captured by the context manager, and it will + set the status as failed, add retry count, log the error + (and save the error information to DB), and release the lock. + 3. If the job has been failed too many times, it will set the status as ``ignored``. + 4. If everything goes well, it will set status as ``succeeded`` and apply updates. + + Usage example:: + + with Job.start( + engine=engine, + id="job-1", + in_process_status=20, + failed_status=30, + success_status=40, + ignore_status=50, + expire=900, # concurrency lock will expire in 15 minutes, + max_retry=3, + debug=True, + ) as (job, updates): + # do your job logic here + ... + # you can use ``updates.set(...)`` method to specify + # what you would like to update at the end of the job + # if the job succeeded. + updates.set(key="data", value={"version": 1}) + :param engine: SQLAlchemy engine. A life-cycle of a job has to be done in a new session. """ @@ -333,6 +397,15 @@ def query_by_status( limit: int = 10, older_task_first: bool = True, ) -> T.List["T_JOB"]: + """ + Query job by status. + + :param engine_or_session: + :param status: desired status code + :param limit: number of jobs to return + :param older_task_first: if True, then return older task + (older update_at time) first. + """ if isinstance(engine_or_session, sa.Engine): with orm.Session(engine_or_session) as ses: job_list = cls._query_by_status( @@ -364,9 +437,19 @@ def query_by_status( @dataclasses.dataclass class Updates: + """ + A helper class that hold the key value you want to update at the end of the + job if the job succeeded. + """ + values: dict = dataclasses.field(default_factory=dict) def set(self, key: str, value: T.Any): + """ + Use this method to set "to-update" data. Note that you should not + update some columns like "id", "status", "update_at" yourself, + it will be updated by the :meth:`JobMixin.start` context manager. + """ if key in disallowed_cols: # pragma: no cover raise KeyError(f"You should NOT set {key!r} column yourself!") self.values[key] = value