Fix test1

migrations/versions/20240505_0010_a80b250420e4_schema_integrity_fixes.py

@@ -15,14 +15,12 @@
 
 # revision identifiers, used by Alembic.
 revision = 'a80b250420e4'
-down_revision = '4892e78eb989'
+down_revision = '1e868db5c6ea'
 branch_labels = None
 depends_on = None
 
 
 def upgrade():
-    op.drop_table('raw_html_old', schema='STG_TIMETABLE')
-    op.drop_table('raw_html', schema='STG_TIMETABLE')
     op.drop_table('vk_groups', schema='STG_SOCIAL')
     op.create_table_schema("STG_RASPHYSMSU")
     op.create_table_schema("STG_ACHIEVEMENT")
@@ -714,38 +712,18 @@ def downgrade():
         sa.PrimaryKeyConstraint('id', name='vk_groups_pkey'),
         schema='STG_SOCIAL',
     )
-    op.create_table(
-        'raw_html',
-        sa.Column('url', sa.VARCHAR(length=256), autoincrement=False, nullable=True),
-        sa.Column('raw_html', sa.TEXT(), autoincrement=False, nullable=True),
-        schema='STG_TIMETABLE',
-    )
-    op.create_table(
-        'raw_html_old',
-        sa.Column('url', sa.VARCHAR(length=256), autoincrement=False, nullable=True),
-        sa.Column('raw_html', sa.TEXT(), autoincrement=False, nullable=True),
-        schema='STG_TIMETABLE',
-    )
     op.grant_on_table(
-        "test_test_dwh_stg_social_all" if os.getenv("ENVIRONMENT") != "production" else "prod_test_dwh_stg_social_all",
+        "test_dwh_stg_social_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_social_all",
         ['ALL'],
         '"STG_SOCIAL".vk_groups',
     )
     op.grant_on_table(
-        (
-            "test_test_dwh_stg_social_write"
-            if os.getenv("ENVIRONMENT") != "production"
-            else "prod_test_dwh_stg_social_write"
-        ),
+        ("test_dwh_stg_social_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_social_write"),
         ['SELECT', 'UPDATE', 'DELETE', 'TRUNCATE', 'INSERT'],
         '"STG_SOCIAL".vk_groups',
     )
     op.grant_on_table(
-        (
-            "test_test_dwh_stg_social_read"
-            if os.getenv("ENVIRONMENT") != "production"
-            else "prod_test_dwh_stg_social_read"
-        ),
+        ("test_dwh_stg_social_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_social_read"),
         ['SELECT'],
         '"STG_SOCIAL".vk_groups',
     )
@@ -863,6 +841,7 @@ def downgrade():
         'credentials',
         'token',
         existing_type=sa.String(),
+        postgresql_using="token::json",
         type_=postgresql.JSON(astext_type=sa.Text()),
         nullable=False,
         schema='STG_TIMETABLE',
@@ -871,6 +850,7 @@ def downgrade():
         'credentials',
         'scope',
         existing_type=sa.String(),
+        postgresql_using="scope::json",
         type_=postgresql.JSON(astext_type=sa.Text()),
         nullable=False,
         schema='STG_TIMETABLE',
@@ -913,6 +893,7 @@ def downgrade():
         'webhook_storage',
         'message',
         existing_type=sa.String(),
+        postgresql_using="message::json",
         type_=postgresql.JSON(astext_type=sa.Text()),
         nullable=False,
         schema='STG_SOCIAL',
@@ -944,6 +925,7 @@ def downgrade():
         'receiver',
         'receiver_body',
         existing_type=sa.String(),
+        postgresql_using="receiver_body::json",
         type_=postgresql.JSON(astext_type=sa.Text()),
         nullable=False,
         schema='STG_PINGER',
@@ -965,6 +947,7 @@ def downgrade():
         'alert',
         'data',
         existing_type=sa.String(),
+        postgresql_using="data::json",
         type_=postgresql.JSON(astext_type=sa.Text()),
         existing_nullable=True,
         schema='STG_PINGER',

migrations/versions/20240507_0246_6f659c404b5f_add_container_log.py

@@ -69,6 +69,18 @@ def upgrade():
 
 
 def downgrade():
+    op.revoke_on_schema(
+        "test_dwh_stg_infra_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_infra_all",
+        "STG_INFRA",
+    )
+    op.revoke_on_schema(
+        ("test_dwh_stg_infra_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_infra_write"),
+        "STG_INFRA",
+    )
+    op.revoke_on_schema(
+        ("test_dwh_stg_infra_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_infra_read"),
+        "STG_INFRA",
+    )
     op.drop_table('container_log', schema='STG_INFRA')
     op.delete_group("test_dwh_stg_infra_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_infra_all")
     op.delete_group(

migrations/versions/20240507_0436_45fc3ad3a4db_add_container_log_in_ods.py

@@ -77,6 +77,26 @@ def upgrade():
 
 
 def downgrade():
+    op.revoke_on_schema(
+        "test_dwh_ods_infra_logs_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_infra_logs_all",
+        "ODS_INFRA_LOGS",
+    )
+    op.revoke_on_schema(
+        (
+            "test_dwh_ods_infra_logs_write"
+            if os.getenv("ENVIRONMENT") != "production"
+            else "prod_dwh_ods_infra_logs_write"
+        ),
+        "ODS_INFRA_LOGS",
+    )
+    op.revoke_on_schema(
+        (
+            "test_dwh_ods_infra_logs_read"
+            if os.getenv("ENVIRONMENT") != "production"
+            else "prod_dwh_ods_infra_logs_read"
+        ),
+        "ODS_INFRA_LOGS",
+    )
     op.drop_table('container_log', schema='ODS_INFRA_LOGS')
     op.delete_group(
         "test_dwh_ods_infra_logs_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_ods_infra_logs_all"

migrations/versions/20240507_0742_1100c470c547_dm_infra_logs.py

@@ -102,6 +102,22 @@ def upgrade():
 
 
 def downgrade():
+    op.revoke_on_schema(
+        "test_dwh_dm_infra_logs_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dm_infra_logs_all",
+        "DM_INFRA_LOGS",
+    )
+    op.revoke_on_schema(
+        (
+            "test_dwh_dm_infra_logs_write"
+            if os.getenv("ENVIRONMENT") != "production"
+            else "prod_dwh_dm_infra_logs_write"
+        ),
+        "DM_INFRA_LOGS",
+    )
+    op.revoke_on_schema(
+        ("test_dwh_dm_infra_logs_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dm_infra_logs_read"),
+        "DM_INFRA_LOGS",
+    )
     op.drop_table('container_log_cube', schema='DM_INFRA_LOGS')
     op.delete_group(
         "test_dwh_dm_infra_logs_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_dm_infra_logs_all"

migrations/versions/20240509_1220_d459997cd681_github_stg.py

@@ -537,6 +537,18 @@ def upgrade():
 
 
 def downgrade():
+    op.revoke_on_schema(
+        "test_dwh_stg_github_all" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_github_all",
+        "STG_GITHUB",
+    )
+    op.revoke_on_schema(
+        ("test_dwh_stg_github_write" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_github_write"),
+        "STG_GITHUB",
+    )
+    op.revoke_on_schema(
+        ("test_dwh_stg_github_read" if os.getenv("ENVIRONMENT") != "production" else "prod_dwh_stg_github_read"),
+        "STG_GITHUB",
+    )
     op.drop_table('profcomff_team_repo', schema='STG_GITHUB')
     op.drop_table('profcomff_team_member', schema='STG_GITHUB')
     op.drop_table('profcomff_team', schema='STG_GITHUB')

profcomff_definitions/ODS/timetable.py

@@ -0,0 +1,13 @@
+from sqlalchemy import String
+from sqlalchemy.orm import Mapped, mapped_column
+
+from profcomff_definitions.base import Base
+
+
+class OdsTimetableAct(Base):
+    event_text: Mapped[str | None] = mapped_column(String, nullable=True, index=True)
+    time_interval_text: Mapped[str | None] = mapped_column(String, nullable=True)
+    group_text: Mapped[str | None] = mapped_column(String, nullable=True)
+    __mapper_args__ = {
+        "primary_key": [event_text, time_interval_text, group_text]
+    }  # Used only to correctly map ORM object to sql table

profcomff_definitions/base.py

@@ -1,6 +1,7 @@
 import re
 
-from sqlalchemy.ext.declarative import as_declarative, declared_attr
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import as_declarative
 
 from migrations.schema.schemas import add_table_schema_to_model
 

tests/conftest.py

@@ -1,30 +1,63 @@
 import os
 from pathlib import Path
 from typing import Generator
 
 import pytest
-from alembic import command
 from alembic.config import Config
 from sqlalchemy import create_engine
+from alembic.command import downgrade, upgrade, revision
+from alembic.config import Config
+from alembic.script import Script, ScriptDirectory
 from sqlalchemy.engine import Engine
 
 
 REPO_ROOT = Path(os.path.abspath(os.path.dirname(__file__))).parent.resolve()
 
 
-@pytest.fixture(scope='session')
-def migration() -> Generator[None, None, None]:
+@pytest.fixture
+def alembic_config():
     alembic_cfg = Config()
     alembic_cfg.set_main_option('script_location', str(REPO_ROOT / "migrations"))
-    alembic_cfg.set_main_option('sqlalchemy.url', "postgresql://postgres:postgres@localhost:5432/postgres")
-    command.upgrade(alembic_cfg, 'head')
-    command.revision(alembic_cfg, autogenerate=True, message="tests")
-    command.upgrade(alembic_cfg, 'head')
-    yield
-    command.downgrade(alembic_cfg, 'head-1')
+    alembic_cfg.set_main_option('sqlalchemy.url', os.getenv("DB_DSN") or "postgresql://postgres:postgres@localhost:5432/postgres")  # db for migration tests
+    return alembic_cfg
+
+
+@pytest.fixture
+def revisions(alembic_config: Config) -> list[Script]:
+    revisions_dir = ScriptDirectory.from_config(alembic_config)
+    revisions = list(revisions_dir.walk_revisions("base", "heads"))
+    revisions.reverse()
+    return revisions
+
+
+def test_migrations_stairway(alembic_config: Config, revisions: list[Script]) -> None:
+    for revision in revisions:
+        down_revision = revision.down_revision or "-1"
+        if isinstance(down_revision, tuple):
+            down_revision = down_revision[0]
+        upgrade(alembic_config, revision.revision)
+        downgrade(alembic_config, down_revision)
+        upgrade(alembic_config, revision.revision)
+
+
+### @mixx3 these tests is obsolete, TODO write generation tests for lib
+# @pytest.fixture
+# def generator_alembic_config():
+#     alembic_cfg = Config(str(REPO_ROOT / "generation_test_alembic.ini"))
+#     alembic_cfg.set_main_option('sqlalchemy.url', os.getenv("DB_DSN") or "postgresql://postgres:postgres@localhost:5432/postgres")  # db for migration tests
+#     return alembic_cfg
+
+
+# @pytest.fixture
+# def test_do_generate_migration(generator_alembic_config: Config) -> Generator[None, None, None]:
+#     upgrade(generator_alembic_config, 'head')
+#     revision(generator_alembic_config, autogenerate=True, message="tests")
+#     upgrade(generator_alembic_config, 'head')
+#     yield
+#     downgrade(generator_alembic_config, 'head-1')
 
 
 @pytest.fixture()
 def engine() -> Generator[Engine, None, None]:
-    engine = create_engine("postgresql://postgres:postgres@localhost:5432/postgres")
+    engine = create_engine(os.getenv("DB_DSN") or "postgresql://postgres:postgres@localhost:5432/postgres")
     yield engine