From 51b7216ac8431149faa934aa1418eb9d172a8581 Mon Sep 17 00:00:00 2001 From: Mikhail <87110754+mixx3@users.noreply.github.com> Date: Sun, 19 May 2024 22:57:37 +0300 Subject: [PATCH] Fix test1 (#43) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Изменения пофиксил миграции + добавил тест на ступенчатую миграцию --- ...20240422_1624_4892e78eb989_add_raw_html.py | 89 ------------------- ...010_a80b250420e4_schema_integrity_fixes.py | 35 ++------ ...507_0246_6f659c404b5f_add_container_log.py | 12 +++ ...6_45fc3ad3a4db_add_container_log_in_ods.py | 20 +++++ ...0240507_0742_1100c470c547_dm_infra_logs.py | 16 ++++ .../20240509_1220_d459997cd681_github_stg.py | 12 +++ profcomff_definitions/ODS/timetable.py | 13 +++ profcomff_definitions/base.py | 3 +- tests/conftest.py | 53 ++++++++--- tests/database.py | 13 --- tests/tests.py | 31 ------- 11 files changed, 127 insertions(+), 170 deletions(-) delete mode 100644 migrations/versions/20240422_1624_4892e78eb989_add_raw_html.py delete mode 100644 tests/database.py delete mode 100644 tests/tests.py diff --git a/migrations/versions/20240422_1624_4892e78eb989_add_raw_html.py b/migrations/versions/20240422_1624_4892e78eb989_add_raw_html.py deleted file mode 100644 index 60d763e..0000000 --- a/migrations/versions/20240422_1624_4892e78eb989_add_raw_html.py +++ /dev/null @@ -1,89 +0,0 @@ -"""add_raw_html - -Revision ID: 4892e78eb989 -Revises: 1e868db5c6ea -Create Date: 2024-04-22 16:24:05.795444 - -""" - -import os - -from alembic import op - - -revision = '4892e78eb989' -down_revision = '1e868db5c6ea' -branch_labels = None -depends_on = None - - -def upgrade(): - op.execute( - """ - CREATE TABLE IF NOT EXISTS "STG_TIMETABLE".raw_html (url varchar(256) NULL, raw_html text NULL); - CREATE TABLE IF NOT EXISTS "STG_TIMETABLE".raw_html_old (url varchar(256) NULL, raw_html text NULL); - """ - ) # this table is produced in dwh-pipelines - op.grant_on_table( - "test_dwh_stg_timetable_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_timetable_read", - ['SELECT'], - '"STG_TIMETABLE".raw_html', - ) - op.grant_on_table( - "test_dwh_stg_timetable_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_timetable_write", - ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'], - '"STG_TIMETABLE".raw_html', - ) - op.grant_on_table( - "test_dwh_stg_timetable_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_timetable_all", - ['ALL'], - '"STG_TIMETABLE".raw_html', - ) - op.grant_on_table( - "test_dwh_stg_timetable_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_timetable_read", - ['SELECT'], - '"STG_TIMETABLE".raw_html_old', - ) - op.grant_on_table( - "test_dwh_stg_timetable_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_timetable_write", - ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'], - '"STG_TIMETABLE".raw_html_old', - ) - op.grant_on_table( - "test_dwh_stg_timetable_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_timetable_all", - ['ALL'], - '"STG_TIMETABLE".raw_html_old', - ) - - -def downgrade(): - op.revoke_on_table( - "test_dwh_stg_timetable_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_timetable_all", - ['ALL'], - '"STG_TIMETABLE".raw_html_old', - ) - op.revoke_on_table( - "test_dwh_stg_timetable_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_timetable_write", - ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'], - '"STG_TIMETABLE".raw_html_old', - ) - op.revoke_on_table( - "test_dwh_stg_timetable_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_timetable_read", - ['SELECT'], - '"STG_TIMETABLE".raw_html_old', - ) - op.revoke_on_table( - "test_dwh_stg_timetable_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_timetable_all", - ['ALL'], - '"STG_TIMETABLE".raw_html', - ) - op.revoke_on_table( - "test_dwh_stg_timetable_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_timetable_write", - ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'], - '"STG_TIMETABLE".raw_html', - ) - op.revoke_on_table( - "test_dwh_stg_timetable_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_timetable_read", - ['SELECT'], - '"STG_TIMETABLE".raw_html', - ) diff --git a/migrations/versions/20240505_0010_a80b250420e4_schema_integrity_fixes.py b/migrations/versions/20240505_0010_a80b250420e4_schema_integrity_fixes.py index 25052af..26eb01a 100644 --- a/migrations/versions/20240505_0010_a80b250420e4_schema_integrity_fixes.py +++ b/migrations/versions/20240505_0010_a80b250420e4_schema_integrity_fixes.py @@ -15,14 +15,12 @@ # revision identifiers, used by Alembic. revision = 'a80b250420e4' -down_revision = '4892e78eb989' +down_revision = '1e868db5c6ea' branch_labels = None depends_on = None def upgrade(): - op.drop_table('raw_html_old', schema='STG_TIMETABLE') - op.drop_table('raw_html', schema='STG_TIMETABLE') op.drop_table('vk_groups', schema='STG_SOCIAL') op.create_table_schema("STG_RASPHYSMSU") op.create_table_schema("STG_ACHIEVEMENT") @@ -714,38 +712,18 @@ def downgrade(): sa.PrimaryKeyConstraint('id', name='vk_groups_pkey'), schema='STG_SOCIAL', ) - op.create_table( - 'raw_html', - sa.Column('url', sa.VARCHAR(length=256), autoincrement=False, nullable=True), - sa.Column('raw_html', sa.TEXT(), autoincrement=False, nullable=True), - schema='STG_TIMETABLE', - ) - op.create_table( - 'raw_html_old', - sa.Column('url', sa.VARCHAR(length=256), autoincrement=False, nullable=True), - sa.Column('raw_html', sa.TEXT(), autoincrement=False, nullable=True), - schema='STG_TIMETABLE', - ) op.grant_on_table( - "test_test_dwh_stg_social_all" if os.getenv("ENVIRONMENT") != "production" else "prod_test_dwh_stg_social_all", + "test_dwh_stg_social_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_social_all", ['ALL'], '"STG_SOCIAL".vk_groups', ) op.grant_on_table( - ( - "test_test_dwh_stg_social_write" - if os.getenv("ENVIRONMENT") != "production" - else "prod_test_dwh_stg_social_write" - ), + ("test_dwh_stg_social_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_social_write"), ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'], '"STG_SOCIAL".vk_groups', ) op.grant_on_table( - ( - "test_test_dwh_stg_social_read" - if os.getenv("ENVIRONMENT") != "production" - else "prod_test_dwh_stg_social_read" - ), + ("test_dwh_stg_social_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_social_read"), ['SELECT'], '"STG_SOCIAL".vk_groups', ) @@ -863,6 +841,7 @@ def downgrade(): 'credentials', 'token', existing_type=sa.String(), + postgresql_using="token::json", type_=postgresql.JSON(astext_type=sa.Text()), nullable=False, schema='STG_TIMETABLE', @@ -871,6 +850,7 @@ def downgrade(): 'credentials', 'scope', existing_type=sa.String(), + postgresql_using="scope::json", type_=postgresql.JSON(astext_type=sa.Text()), nullable=False, schema='STG_TIMETABLE', @@ -913,6 +893,7 @@ def downgrade(): 'webhook_storage', 'message', existing_type=sa.String(), + postgresql_using="message::json", type_=postgresql.JSON(astext_type=sa.Text()), nullable=False, schema='STG_SOCIAL', @@ -944,6 +925,7 @@ def downgrade(): 'receiver', 'receiver_body', existing_type=sa.String(), + postgresql_using="receiver_body::json", type_=postgresql.JSON(astext_type=sa.Text()), nullable=False, schema='STG_PINGER', @@ -965,6 +947,7 @@ def downgrade(): 'alert', 'data', existing_type=sa.String(), + postgresql_using="data::json", type_=postgresql.JSON(astext_type=sa.Text()), existing_nullable=True, schema='STG_PINGER', diff --git a/migrations/versions/20240507_0246_6f659c404b5f_add_container_log.py b/migrations/versions/20240507_0246_6f659c404b5f_add_container_log.py index c6e3dd8..5983bd9 100644 --- a/migrations/versions/20240507_0246_6f659c404b5f_add_container_log.py +++ b/migrations/versions/20240507_0246_6f659c404b5f_add_container_log.py @@ -69,6 +69,18 @@ def upgrade(): def downgrade(): + op.revoke_on_schema( + "test_dwh_stg_infra_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_infra_all", + "STG_INFRA", + ) + op.revoke_on_schema( + ("test_dwh_stg_infra_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_infra_write"), + "STG_INFRA", + ) + op.revoke_on_schema( + ("test_dwh_stg_infra_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_infra_read"), + "STG_INFRA", + ) op.drop_table('container_log', schema='STG_INFRA') op.delete_group("test_dwh_stg_infra_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_infra_all") op.delete_group( diff --git a/migrations/versions/20240507_0436_45fc3ad3a4db_add_container_log_in_ods.py b/migrations/versions/20240507_0436_45fc3ad3a4db_add_container_log_in_ods.py index 649982c..3066f40 100644 --- a/migrations/versions/20240507_0436_45fc3ad3a4db_add_container_log_in_ods.py +++ b/migrations/versions/20240507_0436_45fc3ad3a4db_add_container_log_in_ods.py @@ -77,6 +77,26 @@ def upgrade(): def downgrade(): + op.revoke_on_schema( + "test_dwh_ods_infra_logs_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_infra_logs_all", + "ODS_INFRA_LOGS", + ) + op.revoke_on_schema( + ( + "test_dwh_ods_infra_logs_write" + if os.getenv("ENVIRONMENT") != "production" + else "prod_dwh_ods_infra_logs_write" + ), + "ODS_INFRA_LOGS", + ) + op.revoke_on_schema( + ( + "test_dwh_ods_infra_logs_read" + if os.getenv("ENVIRONMENT") != "production" + else "prod_dwh_ods_infra_logs_read" + ), + "ODS_INFRA_LOGS", + ) op.drop_table('container_log', schema='ODS_INFRA_LOGS') op.delete_group( "test_dwh_ods_infra_logs_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_infra_logs_all" diff --git a/migrations/versions/20240507_0742_1100c470c547_dm_infra_logs.py b/migrations/versions/20240507_0742_1100c470c547_dm_infra_logs.py index 4cdfabd..d55e668 100644 --- a/migrations/versions/20240507_0742_1100c470c547_dm_infra_logs.py +++ b/migrations/versions/20240507_0742_1100c470c547_dm_infra_logs.py @@ -102,6 +102,22 @@ def upgrade(): def downgrade(): + op.revoke_on_schema( + "test_dwh_dm_infra_logs_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dm_infra_logs_all", + "DM_INFRA_LOGS", + ) + op.revoke_on_schema( + ( + "test_dwh_dm_infra_logs_write" + if os.getenv("ENVIRONMENT") != "production" + else "prod_dwh_dm_infra_logs_write" + ), + "DM_INFRA_LOGS", + ) + op.revoke_on_schema( + ("test_dwh_dm_infra_logs_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dm_infra_logs_read"), + "DM_INFRA_LOGS", + ) op.drop_table('container_log_cube', schema='DM_INFRA_LOGS') op.delete_group( "test_dwh_dm_infra_logs_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dm_infra_logs_all" diff --git a/migrations/versions/20240509_1220_d459997cd681_github_stg.py b/migrations/versions/20240509_1220_d459997cd681_github_stg.py index 0f0c931..9231083 100644 --- a/migrations/versions/20240509_1220_d459997cd681_github_stg.py +++ b/migrations/versions/20240509_1220_d459997cd681_github_stg.py @@ -537,6 +537,18 @@ def upgrade(): def downgrade(): + op.revoke_on_schema( + "test_dwh_stg_github_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_github_all", + "STG_GITHUB", + ) + op.revoke_on_schema( + ("test_dwh_stg_github_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_github_write"), + "STG_GITHUB", + ) + op.revoke_on_schema( + ("test_dwh_stg_github_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_github_read"), + "STG_GITHUB", + ) op.drop_table('profcomff_team_repo', schema='STG_GITHUB') op.drop_table('profcomff_team_member', schema='STG_GITHUB') op.drop_table('profcomff_team', schema='STG_GITHUB') diff --git a/profcomff_definitions/ODS/timetable.py b/profcomff_definitions/ODS/timetable.py index e69de29..894231c 100644 --- a/profcomff_definitions/ODS/timetable.py +++ b/profcomff_definitions/ODS/timetable.py @@ -0,0 +1,13 @@ +from sqlalchemy import String +from sqlalchemy.orm import Mapped, mapped_column + +from profcomff_definitions.base import Base + + +class OdsTimetableAct(Base): + event_text: Mapped[str | None] = mapped_column(String, nullable=True, index=True) + time_interval_text: Mapped[str | None] = mapped_column(String, nullable=True) + group_text: Mapped[str | None] = mapped_column(String, nullable=True) + __mapper_args__ = { + "primary_key": [event_text, time_interval_text, group_text] + } # Used only to correctly map ORM object to sql table diff --git a/profcomff_definitions/base.py b/profcomff_definitions/base.py index dd754e0..82344e8 100644 --- a/profcomff_definitions/base.py +++ b/profcomff_definitions/base.py @@ -1,6 +1,7 @@ import re -from sqlalchemy.ext.declarative import as_declarative, declared_attr +from sqlalchemy.ext.declarative import declared_attr +from sqlalchemy.orm import as_declarative from migrations.schema.schemas import add_table_schema_to_model diff --git a/tests/conftest.py b/tests/conftest.py index 97bd1db..e62071c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,28 +3,61 @@ from typing import Generator import pytest -from alembic import command from alembic.config import Config from sqlalchemy import create_engine +from alembic.command import downgrade, upgrade, revision +from alembic.config import Config +from alembic.script import Script, ScriptDirectory from sqlalchemy.engine import Engine REPO_ROOT = Path(os.path.abspath(os.path.dirname(__file__))).parent.resolve() -@pytest.fixture(scope='session') -def migration() -> Generator[None, None, None]: +@pytest.fixture +def alembic_config(): alembic_cfg = Config() alembic_cfg.set_main_option('script_location', str(REPO_ROOT / "migrations")) - alembic_cfg.set_main_option('sqlalchemy.url', "postgresql://postgres:postgres@localhost:5432/postgres") - command.upgrade(alembic_cfg, 'head') - command.revision(alembic_cfg, autogenerate=True, message="tests") - command.upgrade(alembic_cfg, 'head') - yield - command.downgrade(alembic_cfg, 'head-1') + alembic_cfg.set_main_option('sqlalchemy.url', os.getenv("DB_DSN") or "postgresql://postgres:postgres@localhost:5432/postgres") # db for migration tests + return alembic_cfg + + +@pytest.fixture +def revisions(alembic_config: Config) -> list[Script]: + revisions_dir = ScriptDirectory.from_config(alembic_config) + revisions = list(revisions_dir.walk_revisions("base", "heads")) + revisions.reverse() + return revisions + + +def test_migrations_stairway(alembic_config: Config, revisions: list[Script]) -> None: + for revision in revisions: + down_revision = revision.down_revision or "-1" + if isinstance(down_revision, tuple): + down_revision = down_revision[0] + upgrade(alembic_config, revision.revision) + downgrade(alembic_config, down_revision) + upgrade(alembic_config, revision.revision) + + +### @mixx3 these tests is obsolete, TODO write generation tests for lib +# @pytest.fixture +# def generator_alembic_config(): +# alembic_cfg = Config(str(REPO_ROOT / "generation_test_alembic.ini")) +# alembic_cfg.set_main_option('sqlalchemy.url', os.getenv("DB_DSN") or "postgresql://postgres:postgres@localhost:5432/postgres") # db for migration tests +# return alembic_cfg + + +# @pytest.fixture +# def test_do_generate_migration(generator_alembic_config: Config) -> Generator[None, None, None]: +# upgrade(generator_alembic_config, 'head') +# revision(generator_alembic_config, autogenerate=True, message="tests") +# upgrade(generator_alembic_config, 'head') +# yield +# downgrade(generator_alembic_config, 'head-1') @pytest.fixture() def engine() -> Generator[Engine, None, None]: - engine = create_engine("postgresql://postgres:postgres@localhost:5432/postgres") + engine = create_engine(os.getenv("DB_DSN") or "postgresql://postgres:postgres@localhost:5432/postgres") yield engine diff --git a/tests/database.py b/tests/database.py deleted file mode 100644 index fac1229..0000000 --- a/tests/database.py +++ /dev/null @@ -1,13 +0,0 @@ -import logging - -from sqlalchemy import Integer -from sqlalchemy.orm import Mapped, mapped_column - -from profcomff_definitions.base import Base - - -logger = logging.getLogger(__name__) - - -class Test(Base): - id: Mapped[int] = mapped_column(Integer, primary_key=True) diff --git a/tests/tests.py b/tests/tests.py deleted file mode 100644 index fdef78e..0000000 --- a/tests/tests.py +++ /dev/null @@ -1,31 +0,0 @@ -from sqlalchemy import text - - -def test_schema_creation(engine, migration): - with engine.connect() as conn: - query = text("select schema_name from information_schema.schemata") - result = [sch[0] for sch in conn.execute(query)] - - assert 'TESTS_DATABASE' in result - - -def test_group_creation(engine, migration): - with engine.connect() as conn: - query = text("SELECT * FROM pg_group") - result = set(obj[0] for obj in conn.execute(query)) - check = {'test_dwh_tests_database_read', 'test_dwh_tests_database_write', 'test_dwh_tests_database_all'} - - assert check.issubset(result) - - -def test_table_rights(engine, migration): - scopes = [ - {'SELECT'}, - {'SELECT', 'INSERT', 'UPDATE', 'DELETE', 'TRUNCATE'}, - {'SELECT', 'UPDATE', 'TRIGGER', 'DELETE', 'TRUNCATE', 'INSERT', 'REFERENCES'}, - ] - groups = ['test_dwh_tests_database_read', 'test_dwh_tests_database_write', 'test_dwh_tests_database_all'] - with engine.connect() as conn: - for i in range(len(groups)): - query = text(f"SELECT privilege_type FROM information_schema.role_table_grants WHERE grantee='{groups[i]}'") - assert scopes[i] == set([right[0] for right in conn.execute(query)])