From 4a2359753064f3e57c917f09db0a953312f4f51a Mon Sep 17 00:00:00 2001 From: James Bursa Date: Wed, 17 Apr 2024 15:33:19 -0400 Subject: [PATCH 01/15] [Issue #1697] add more foreign table models and update setup_foreign_tables.py --- api/src/constants/schema.py | 1 + api/src/data_migration/copy_oracle_data.py | 10 +- .../data_migration/setup_foreign_tables.py | 74 +++------- api/src/db/foreign/__init__.py | 7 + api/src/db/foreign/base.py | 34 +++++ api/src/db/foreign/opportunity.py | 31 +++++ .../db/migrations/setup_local_postgres_db.py | 3 +- api/tests/conftest.py | 5 + .../data_migration/test_copy_oracle_data.py | 34 ++--- .../test_setup_foreign_tables.py | 127 ++++++++++-------- 10 files changed, 191 insertions(+), 135 deletions(-) create mode 100644 api/src/db/foreign/__init__.py create mode 100644 api/src/db/foreign/base.py create mode 100644 api/src/db/foreign/opportunity.py diff --git a/api/src/constants/schema.py b/api/src/constants/schema.py index 21fb26d80..f233b9674 100644 --- a/api/src/constants/schema.py +++ b/api/src/constants/schema.py @@ -3,3 +3,4 @@ class Schemas(StrEnum): API = "api" + FOREIGN = "grants" diff --git a/api/src/data_migration/copy_oracle_data.py b/api/src/data_migration/copy_oracle_data.py index 2d597cb3c..65a5d07d6 100644 --- a/api/src/data_migration/copy_oracle_data.py +++ b/api/src/data_migration/copy_oracle_data.py @@ -41,7 +41,7 @@ class SqlCommands: last_upd_date, creator_id, created_date - from {}.foreign_topportunity + from {}.topportunity where is_draft = 'N' """ @@ -55,7 +55,7 @@ def copy_oracle_data(db_session: db.Session) -> None: try: with db_session.begin(): - _run_copy_commands(db_session, Schemas.API) + _run_copy_commands(db_session, Schemas.API, Schemas.FOREIGN) except Exception: logger.exception("Failed to run copy-oracle-data command") raise @@ -63,11 +63,13 @@ def copy_oracle_data(db_session: db.Session) -> None: logger.info("Successfully ran copy-oracle-data") -def _run_copy_commands(db_session: db.Session, api_schema: str) -> None: +def _run_copy_commands(db_session: db.Session, api_schema: str, foreign_schema: str) -> None: logger.info("Running copy commands for TOPPORTUNITY") db_session.execute(text(SqlCommands.OPPORTUNITY_DELETE_QUERY.format(api_schema))) - db_session.execute(text(SqlCommands.OPPORTUNITY_INSERT_QUERY.format(api_schema, api_schema))) + db_session.execute( + text(SqlCommands.OPPORTUNITY_INSERT_QUERY.format(api_schema, foreign_schema)) + ) count = db_session.scalar( text(f"SELECT count(*) from {api_schema}.transfer_topportunity") # nosec ) diff --git a/api/src/data_migration/setup_foreign_tables.py b/api/src/data_migration/setup_foreign_tables.py index 76304977f..b6ec603ce 100644 --- a/api/src/data_migration/setup_foreign_tables.py +++ b/api/src/data_migration/setup_foreign_tables.py @@ -1,11 +1,13 @@ import logging -from dataclasses import dataclass +import re +import sqlalchemy from pydantic import Field from sqlalchemy import text import src.adapters.db as db import src.adapters.db.flask_db as flask_db +import src.db.foreign from src.constants.schema import Schemas from src.data_migration.data_migration_blueprint import data_migration_blueprint from src.util.env_config import PydanticBaseEnvConfig @@ -15,38 +17,7 @@ class ForeignTableConfig(PydanticBaseEnvConfig): is_local_foreign_table: bool = Field(False) - schema_name: str = Field(Schemas.API) - - -@dataclass -class Column: - column_name: str - postgres_type: str - - is_nullable: bool = True - is_primary_key: bool = False - - -OPPORTUNITY_COLUMNS: list[Column] = [ - Column("OPPORTUNITY_ID", "numeric(20)", is_nullable=False, is_primary_key=True), - Column("OPPNUMBER", "character varying (40)"), - Column("REVISION_NUMBER", "numeric(20)"), - Column("OPPTITLE", "character varying (255)"), - Column("OWNINGAGENCY", "character varying (255)"), - Column("PUBLISHERUID", "character varying (255)"), - Column("LISTED", "CHAR(1)"), - Column("OPPCATEGORY", "CHAR(1)"), - Column("INITIAL_OPPORTUNITY_ID", "numeric(20)"), - Column("MODIFIED_COMMENTS", "character varying (2000)"), - Column("CREATED_DATE", "DATE"), - Column("LAST_UPD_DATE", "DATE"), - Column("CREATOR_ID", "character varying (50)"), - Column("LAST_UPD_ID", "character varying (50)"), - Column("FLAG_2006", "CHAR(1)"), - Column("CATEGORY_EXPLANATION", "character varying (255)"), - Column("PUBLISHER_PROFILE_ID", "numeric(20)"), - Column("IS_DRAFT", "character varying (1)"), -] + schema_name: str = Field(Schemas.FOREIGN) @data_migration_blueprint.cli.command( @@ -64,7 +35,7 @@ def setup_foreign_tables(db_session: db.Session) -> None: logger.info("Successfully ran setup-foreign-tables") -def build_sql(table_name: str, columns: list[Column], is_local: bool, schema_name: str) -> str: +def build_sql(table: sqlalchemy.schema.Table, is_local: bool, schema_name: str) -> str: """ Build the SQL for creating a possibly foreign data table. If running with is_local, it instead creates a regular table. @@ -86,18 +57,15 @@ def build_sql(table_name: str, columns: list[Column], is_local: bool, schema_nam """ column_sql_parts = [] - for column in columns: - column_sql = f"{column.column_name} {column.postgres_type}" + for column in table.columns: + column_sql = str(sqlalchemy.schema.CreateColumn(column)) # Primary keys are defined as constraints in a regular table # and as options in a foreign data table - if column.is_primary_key and is_local: - column_sql += f" CONSTRAINT {table_name}_pkey PRIMARY KEY" - elif column.is_primary_key and not is_local: - column_sql += " OPTIONS (key 'true')" - - if not column.is_nullable: - column_sql += " NOT NULL" + if column.primary_key and is_local: + column_sql += f" CONSTRAINT {table.name}_pkey PRIMARY KEY" + elif column.primary_key and not is_local: + column_sql = re.sub(r"^(.*?)( NOT NULL)?$", r"\1 OPTIONS (key 'true')\2", column_sql) column_sql_parts.append(column_sql) @@ -106,24 +74,16 @@ def build_sql(table_name: str, columns: list[Column], is_local: bool, schema_nam # Don't make a foreign table if running locally create_table_command = "CREATE TABLE IF NOT EXISTS" - create_command_suffix = ( - f" SERVER grants OPTIONS (schema 'EGRANTSADMIN', table '{table_name}')" # noqa: B907 - ) + create_command_suffix = f" SERVER grants OPTIONS (schema 'EGRANTSADMIN', table '{table.name.upper()}')" # noqa: B907 if is_local: # We don't want the config at the end if we're running locally so unset it create_command_suffix = "" - return f"{create_table_command} {schema_name}.foreign_{table_name.lower()} ({','.join(column_sql_parts)}){create_command_suffix}" + return f"{create_table_command} {schema_name}.{table.name} ({','.join(column_sql_parts)}){create_command_suffix}" def _run_create_table_commands(db_session: db.Session, config: ForeignTableConfig) -> None: - db_session.execute( - text( - build_sql( - "TOPPORTUNITY", - OPPORTUNITY_COLUMNS, - config.is_local_foreign_table, - config.schema_name, - ) - ) - ) + for table in src.db.foreign.metadata.tables.values(): + sql = build_sql(table, config.is_local_foreign_table, config.schema_name) + logger.info("create table", extra={"table": table.name, "sql": sql}) + db_session.execute(text(sql)) diff --git a/api/src/db/foreign/__init__.py b/api/src/db/foreign/__init__.py new file mode 100644 index 000000000..99c158814 --- /dev/null +++ b/api/src/db/foreign/__init__.py @@ -0,0 +1,7 @@ +# +# SQLAlchemy models for foreign tables. +# + +from . import base, opportunity + +metadata = base.metadata diff --git a/api/src/db/foreign/base.py b/api/src/db/foreign/base.py new file mode 100644 index 000000000..24f870f15 --- /dev/null +++ b/api/src/db/foreign/base.py @@ -0,0 +1,34 @@ +# +# SQLAlchemy base model and metadata for foreign tables. +# +# These are defined in a separate sqlalchemy.MetaData object as they are managed differently: +# - The tables are actually foreign tables connected to the source Oracle database in most environments. +# - System components should not access them, except transform related components. +# - Migrations are not used to manage creation and changes as the tables are actually defined in a different system. +# + +from typing import Any, Iterable + +import sqlalchemy + +from src.constants.schema import Schemas + +metadata = sqlalchemy.MetaData() + + +class Base(sqlalchemy.orm.DeclarativeBase): + metadata = metadata + + __table_args__ = {"schema": Schemas.FOREIGN} + + def _dict(self) -> dict: + return {c.key: getattr(self, c.key) for c in sqlalchemy.inspect(self).mapper.column_attrs} + + def __repr__(self) -> str: + return f"<{type(self).__name__}({self._dict()!r})" + + def __rich_repr__(self) -> Iterable[tuple[str, Any]]: + """Rich repr for interactive console. + See https://rich.readthedocs.io/en/latest/pretty.html#rich-repr-protocol + """ + return self._dict().items() diff --git a/api/src/db/foreign/opportunity.py b/api/src/db/foreign/opportunity.py new file mode 100644 index 000000000..b94ad16d2 --- /dev/null +++ b/api/src/db/foreign/opportunity.py @@ -0,0 +1,31 @@ +# +# SQLAlchemy models for foreign tables. +# + +import sqlalchemy +from sqlalchemy.orm import mapped_column + +from . import base + + +class Topportunity(base.Base): + __tablename__ = "topportunity" + + opportunity_id = mapped_column(sqlalchemy.NUMERIC(20), nullable=False, primary_key=True) + oppnumber = mapped_column(sqlalchemy.VARCHAR(40)) + revision_number = mapped_column(sqlalchemy.NUMERIC(20)) + opptitle = mapped_column(sqlalchemy.VARCHAR(255)) + owningagency = mapped_column(sqlalchemy.VARCHAR(255)) + publisheruid = mapped_column(sqlalchemy.VARCHAR(255)) + listed = mapped_column(sqlalchemy.CHAR(1)) + oppcategory = mapped_column(sqlalchemy.CHAR(1)) + initial_opportunity_id = mapped_column(sqlalchemy.NUMERIC(20)) + modified_comments = mapped_column(sqlalchemy.VARCHAR(2000)) + created_date = mapped_column(sqlalchemy.DATE) + last_upd_date = mapped_column(sqlalchemy.DATE) + creator_id = mapped_column(sqlalchemy.VARCHAR(50)) + last_upd_id = mapped_column(sqlalchemy.VARCHAR(50)) + flag_2006 = mapped_column(sqlalchemy.CHAR(1)) + category_explanation = mapped_column(sqlalchemy.VARCHAR(255)) + publisher_profile_id = mapped_column(sqlalchemy.NUMERIC(20)) + is_draft = mapped_column(sqlalchemy.VARCHAR(1)) diff --git a/api/src/db/migrations/setup_local_postgres_db.py b/api/src/db/migrations/setup_local_postgres_db.py index 3a13d4431..27aa16a9c 100644 --- a/api/src/db/migrations/setup_local_postgres_db.py +++ b/api/src/db/migrations/setup_local_postgres_db.py @@ -1,5 +1,6 @@ import logging +import sqlalchemy from sqlalchemy import text import src.adapters.db as db @@ -24,4 +25,4 @@ def setup_local_postgres_db() -> None: def _create_schema(conn: db.Connection, schema_name: str) -> None: logger.info("Creating schema %s if it does not already exist", schema_name) - conn.execute(text(f"CREATE SCHEMA IF NOT EXISTS {schema_name}")) + conn.execute(sqlalchemy.schema.CreateSchema(schema_name, if_not_exists=True)) diff --git a/api/tests/conftest.py b/api/tests/conftest.py index 20815aa86..856023dbc 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -136,6 +136,11 @@ def test_api_schema(db_schema_prefix): return f"{db_schema_prefix}{Schemas.API}" +@pytest.fixture +def test_foreign_schema(db_schema_prefix): + return f"{db_schema_prefix}{Schemas.FOREIGN}" + + #################### # Test App & Client #################### diff --git a/api/tests/src/data_migration/test_copy_oracle_data.py b/api/tests/src/data_migration/test_copy_oracle_data.py index 51c92a254..20de8cea0 100644 --- a/api/tests/src/data_migration/test_copy_oracle_data.py +++ b/api/tests/src/data_migration/test_copy_oracle_data.py @@ -10,16 +10,16 @@ @pytest.fixture(autouse=True) -def setup_foreign_tables(db_session, test_api_schema): +def setup_foreign_tables(db_session, test_foreign_schema): _run_create_table_commands( - db_session, ForeignTableConfig(is_local_foreign_table=True, schema_name=test_api_schema) + db_session, ForeignTableConfig(is_local_foreign_table=True, schema_name=test_foreign_schema) ) @pytest.fixture(autouse=True, scope="function") -def truncate_tables(db_session, test_api_schema): +def truncate_tables(db_session, test_api_schema, test_foreign_schema): # Automatically delete all the data in the relevant tables before tests - db_session.execute(text(f"TRUNCATE TABLE {test_api_schema}.foreign_topportunity")) + db_session.execute(text(f"TRUNCATE TABLE {test_foreign_schema}.topportunity")) db_session.execute(text(f"TRUNCATE TABLE {test_api_schema}.transfer_topportunity")) @@ -37,7 +37,7 @@ def convert_value_for_insert(value) -> str: raise Exception("Type not configured for conversion") -def build_foreign_opportunity(db_session, opp_params: dict, api_schema: str): +def build_foreign_opportunity(db_session, opp_params: dict, table_schema: str): opp = ForeignTopportunityFactory.build(**opp_params) columns = opp.keys() @@ -45,41 +45,43 @@ def build_foreign_opportunity(db_session, opp_params: dict, api_schema: str): db_session.execute( text( - f"INSERT INTO {api_schema}.foreign_topportunity ({','.join(columns)}) VALUES ({','.join(values)})" # nosec + f"INSERT INTO {table_schema}.topportunity ({','.join(columns)}) VALUES ({','.join(values)})" # nosec ) ) return opp -def test_copy_oracle_data_foreign_empty(db_session, enable_factory_create, test_api_schema): +def test_copy_oracle_data_foreign_empty( + db_session, enable_factory_create, test_api_schema, test_foreign_schema +): TransferTopportunityFactory.create_batch(size=5) # The foreign table is empty, so this just truncates the transfer table assert db_session.query(TransferTopportunity).count() == 5 - _run_copy_commands(db_session, test_api_schema) + _run_copy_commands(db_session, test_api_schema, test_foreign_schema) assert db_session.query(TransferTopportunity).count() == 0 -def test_copy_oracle_data(db_session, enable_factory_create, test_api_schema): +def test_copy_oracle_data(db_session, enable_factory_create, test_api_schema, test_foreign_schema): print(db_session.__class__.__name__) # Create some records initially in the table that we'll wipe TransferTopportunityFactory.create_batch(size=3) foreign_records = [ - build_foreign_opportunity(db_session, {}, test_api_schema), - build_foreign_opportunity(db_session, {}, test_api_schema), - build_foreign_opportunity(db_session, {}, test_api_schema), + build_foreign_opportunity(db_session, {}, test_foreign_schema), + build_foreign_opportunity(db_session, {}, test_foreign_schema), + build_foreign_opportunity(db_session, {}, test_foreign_schema), build_foreign_opportunity( - db_session, {"oppnumber": "ABC-123-454-321-CBA"}, test_api_schema + db_session, {"oppnumber": "ABC-123-454-321-CBA"}, test_foreign_schema ), - build_foreign_opportunity(db_session, {"opportunity_id": 100}, test_api_schema), + build_foreign_opportunity(db_session, {"opportunity_id": 100}, test_foreign_schema), ] # The copy script won't fetch anything with is_draft not equaling "N" so add one - build_foreign_opportunity(db_session, {"is_draft": "Y"}, test_api_schema) + build_foreign_opportunity(db_session, {"is_draft": "Y"}, test_foreign_schema) - _run_copy_commands(db_session, test_api_schema) + _run_copy_commands(db_session, test_api_schema, test_foreign_schema) copied_opportunities = db_session.query(TransferTopportunity).all() diff --git a/api/tests/src/data_migration/test_setup_foreign_tables.py b/api/tests/src/data_migration/test_setup_foreign_tables.py index 2f235e966..d4d84aea5 100644 --- a/api/tests/src/data_migration/test_setup_foreign_tables.py +++ b/api/tests/src/data_migration/test_setup_foreign_tables.py @@ -1,80 +1,93 @@ import pytest +import sqlalchemy -from src.data_migration.setup_foreign_tables import OPPORTUNITY_COLUMNS, Column, build_sql +import src.db.foreign +from src.data_migration.setup_foreign_tables import build_sql EXPECTED_LOCAL_OPPORTUNITY_SQL = ( - "CREATE TABLE IF NOT EXISTS {}.foreign_topportunity " - "(OPPORTUNITY_ID numeric(20) CONSTRAINT TOPPORTUNITY_pkey PRIMARY KEY NOT NULL," - "OPPNUMBER character varying (40)," - "REVISION_NUMBER numeric(20)," - "OPPTITLE character varying (255)," - "OWNINGAGENCY character varying (255)," - "PUBLISHERUID character varying (255)," - "LISTED CHAR(1)," - "OPPCATEGORY CHAR(1)," - "INITIAL_OPPORTUNITY_ID numeric(20)," - "MODIFIED_COMMENTS character varying (2000)," - "CREATED_DATE DATE," - "LAST_UPD_DATE DATE," - "CREATOR_ID character varying (50)," - "LAST_UPD_ID character varying (50)," - "FLAG_2006 CHAR(1)," - "CATEGORY_EXPLANATION character varying (255)," - "PUBLISHER_PROFILE_ID numeric(20)," - "IS_DRAFT character varying (1))" + "CREATE TABLE IF NOT EXISTS {}.topportunity " + "(opportunity_id NUMERIC(20) NOT NULL CONSTRAINT topportunity_pkey PRIMARY KEY," + "oppnumber VARCHAR(40)," + "revision_number NUMERIC(20)," + "opptitle VARCHAR(255)," + "owningagency VARCHAR(255)," + "publisheruid VARCHAR(255)," + "listed CHAR(1)," + "oppcategory CHAR(1)," + "initial_opportunity_id NUMERIC(20)," + "modified_comments VARCHAR(2000)," + "created_date DATE," + "last_upd_date DATE," + "creator_id VARCHAR(50)," + "last_upd_id VARCHAR(50)," + "flag_2006 CHAR(1)," + "category_explanation VARCHAR(255)," + "publisher_profile_id NUMERIC(20)," + "is_draft VARCHAR(1))" ) EXPECTED_NONLOCAL_OPPORTUNITY_SQL = ( - "CREATE FOREIGN TABLE IF NOT EXISTS {}.foreign_topportunity " - "(OPPORTUNITY_ID numeric(20) OPTIONS (key 'true') NOT NULL," - "OPPNUMBER character varying (40)," - "REVISION_NUMBER numeric(20)," - "OPPTITLE character varying (255)," - "OWNINGAGENCY character varying (255)," - "PUBLISHERUID character varying (255)," - "LISTED CHAR(1)," - "OPPCATEGORY CHAR(1)," - "INITIAL_OPPORTUNITY_ID numeric(20)," - "MODIFIED_COMMENTS character varying (2000)," - "CREATED_DATE DATE," - "LAST_UPD_DATE DATE," - "CREATOR_ID character varying (50)," - "LAST_UPD_ID character varying (50)," - "FLAG_2006 CHAR(1)," - "CATEGORY_EXPLANATION character varying (255)," - "PUBLISHER_PROFILE_ID numeric(20)," - "IS_DRAFT character varying (1))" + "CREATE FOREIGN TABLE IF NOT EXISTS {}.topportunity " + "(opportunity_id NUMERIC(20) OPTIONS (key 'true') NOT NULL," + "oppnumber VARCHAR(40)," + "revision_number NUMERIC(20)," + "opptitle VARCHAR(255)," + "owningagency VARCHAR(255)," + "publisheruid VARCHAR(255)," + "listed CHAR(1)," + "oppcategory CHAR(1)," + "initial_opportunity_id NUMERIC(20)," + "modified_comments VARCHAR(2000)," + "created_date DATE," + "last_upd_date DATE," + "creator_id VARCHAR(50)," + "last_upd_id VARCHAR(50)," + "flag_2006 CHAR(1)," + "category_explanation VARCHAR(255)," + "publisher_profile_id NUMERIC(20)," + "is_draft VARCHAR(1))" " SERVER grants OPTIONS (schema 'EGRANTSADMIN', table 'TOPPORTUNITY')" ) -TEST_COLUMNS = [ - Column("ID", "integer", is_nullable=False, is_primary_key=True), - Column("DESCRIPTION", "text"), -] +TEST_METADATA = sqlalchemy.MetaData() +TEST_TABLE = sqlalchemy.Table( + "test_table", + TEST_METADATA, + sqlalchemy.Column("id", sqlalchemy.Integer, nullable=False, primary_key=True), + sqlalchemy.Column("description", sqlalchemy.Text), +) EXPECTED_LOCAL_TEST_SQL = ( - "CREATE TABLE IF NOT EXISTS {}.foreign_test_table " - "(ID integer CONSTRAINT TEST_TABLE_pkey PRIMARY KEY NOT NULL," - "DESCRIPTION text)" + "CREATE TABLE IF NOT EXISTS {}.test_table " + "(id INTEGER NOT NULL CONSTRAINT test_table_pkey PRIMARY KEY," + "description TEXT)" ) EXPECTED_NONLOCAL_TEST_SQL = ( - "CREATE FOREIGN TABLE IF NOT EXISTS {}.foreign_test_table " - "(ID integer OPTIONS (key 'true') NOT NULL," - "DESCRIPTION text)" + "CREATE FOREIGN TABLE IF NOT EXISTS {}.test_table " + "(id INTEGER OPTIONS (key 'true') NOT NULL," + "description TEXT)" " SERVER grants OPTIONS (schema 'EGRANTSADMIN', table 'TEST_TABLE')" ) @pytest.mark.parametrize( - "table_name,columns,is_local,expected_sql", + "table,is_local,expected_sql", [ - ("TEST_TABLE", TEST_COLUMNS, True, EXPECTED_LOCAL_TEST_SQL), - ("TEST_TABLE", TEST_COLUMNS, False, EXPECTED_NONLOCAL_TEST_SQL), - ("TOPPORTUNITY", OPPORTUNITY_COLUMNS, True, EXPECTED_LOCAL_OPPORTUNITY_SQL), - ("TOPPORTUNITY", OPPORTUNITY_COLUMNS, False, EXPECTED_NONLOCAL_OPPORTUNITY_SQL), + (TEST_TABLE, True, EXPECTED_LOCAL_TEST_SQL), + (TEST_TABLE, False, EXPECTED_NONLOCAL_TEST_SQL), + ( + src.db.foreign.metadata.tables["grants.topportunity"], + True, + EXPECTED_LOCAL_OPPORTUNITY_SQL, + ), + ( + src.db.foreign.metadata.tables["grants.topportunity"], + False, + EXPECTED_NONLOCAL_OPPORTUNITY_SQL, + ), ], ) -def test_build_sql(table_name, columns, is_local, expected_sql, test_api_schema): - sql = build_sql(table_name, columns, is_local, test_api_schema) +def test_build_sql(table, is_local, expected_sql, test_foreign_schema): + sql = build_sql(table, is_local, test_foreign_schema) - assert sql == expected_sql.format(test_api_schema) + assert sql == expected_sql.format(test_foreign_schema) From 43da4b8ab10a8e896f2fc4b37c6fb81fabf4f586 Mon Sep 17 00:00:00 2001 From: James Bursa Date: Thu, 18 Apr 2024 10:33:55 -0400 Subject: [PATCH 02/15] Depend on SQLAlchemy to do more of the generating for us --- .../data_migration/setup_foreign_tables.py | 50 +++----- api/src/db/foreign/dialect.py | 42 +++++++ .../test_setup_foreign_tables.py | 117 ++++++++++-------- 3 files changed, 119 insertions(+), 90 deletions(-) create mode 100644 api/src/db/foreign/dialect.py diff --git a/api/src/data_migration/setup_foreign_tables.py b/api/src/data_migration/setup_foreign_tables.py index b6ec603ce..3cef180a3 100644 --- a/api/src/data_migration/setup_foreign_tables.py +++ b/api/src/data_migration/setup_foreign_tables.py @@ -1,13 +1,12 @@ import logging -import re import sqlalchemy from pydantic import Field -from sqlalchemy import text import src.adapters.db as db import src.adapters.db.flask_db as flask_db import src.db.foreign +import src.db.foreign.dialect from src.constants.schema import Schemas from src.data_migration.data_migration_blueprint import data_migration_blueprint from src.util.env_config import PydanticBaseEnvConfig @@ -40,50 +39,31 @@ def build_sql(table: sqlalchemy.schema.Table, is_local: bool, schema_name: str) Build the SQL for creating a possibly foreign data table. If running with is_local, it instead creates a regular table. - Assume you have a table with two columns, an "ID" primary key column, and a "description" text column, - you would call this as:: - - build_sql("EXAMPLE_TABLE", [Column("ID", "integer", is_nullable=False, is_primary_key=True), Column("DESCRIPTION", "text")], is_local) - - Depending on whether the is_local bool is true or false would give two different outputs. - is_local is True:: - CREATE TABLE IF NOT EXISTS foreign_example_table (ID integer CONSTRAINT EXAMPLE_TABLE_pkey PRIMARY KEY NOT NULL,DESCRIPTION text) + CREATE TABLE IF NOT EXISTS foreign_example_table + (ID integer CONSTRAINT EXAMPLE_TABLE_pkey PRIMARY KEY NOT NULL,DESCRIPTION text) is_local is False:: - CREATE FOREIGN TABLE IF NOT EXISTS foreign_example_table (ID integer OPTIONS (key 'true') NOT NULL,DESCRIPTION text) SERVER grants OPTIONS (schema 'EGRANTSADMIN', table 'EXAMPLE_TABLE') + CREATE FOREIGN TABLE IF NOT EXISTS foreign_example_table + (ID integer OPTIONS (key 'true') NOT NULL,DESCRIPTION text) + SERVER grants OPTIONS (schema 'EGRANTSADMIN', table 'EXAMPLE_TABLE') """ - column_sql_parts = [] - for column in table.columns: - column_sql = str(sqlalchemy.schema.CreateColumn(column)) - - # Primary keys are defined as constraints in a regular table - # and as options in a foreign data table - if column.primary_key and is_local: - column_sql += f" CONSTRAINT {table.name}_pkey PRIMARY KEY" - elif column.primary_key and not is_local: - column_sql = re.sub(r"^(.*?)( NOT NULL)?$", r"\1 OPTIONS (key 'true')\2", column_sql) - - column_sql_parts.append(column_sql) - - create_table_command = "CREATE FOREIGN TABLE IF NOT EXISTS" - if is_local: - # Don't make a foreign table if running locally - create_table_command = "CREATE TABLE IF NOT EXISTS" - - create_command_suffix = f" SERVER grants OPTIONS (schema 'EGRANTSADMIN', table '{table.name.upper()}')" # noqa: B907 + create_table = sqlalchemy.schema.CreateTable(table, if_not_exists=True) if is_local: - # We don't want the config at the end if we're running locally so unset it - create_command_suffix = "" - - return f"{create_table_command} {schema_name}.{table.name} ({','.join(column_sql_parts)}){create_command_suffix}" + compiler = create_table.compile(schema_translate_map={"grants": schema_name}) + else: + compiler = create_table.compile( + dialect=src.db.foreign.dialect.ForeignTableDialect(), + schema_translate_map={"grants": schema_name}, + ) + return str(compiler).strip() def _run_create_table_commands(db_session: db.Session, config: ForeignTableConfig) -> None: for table in src.db.foreign.metadata.tables.values(): sql = build_sql(table, config.is_local_foreign_table, config.schema_name) logger.info("create table", extra={"table": table.name, "sql": sql}) - db_session.execute(text(sql)) + db_session.execute(sqlalchemy.text(sql)) diff --git a/api/src/db/foreign/dialect.py b/api/src/db/foreign/dialect.py new file mode 100644 index 000000000..ea388e480 --- /dev/null +++ b/api/src/db/foreign/dialect.py @@ -0,0 +1,42 @@ +# +# Support for generating SQL for "CREATE FOREIGN TABLE". +# + +import re + +import sqlalchemy + + +class ForeignTableDDLCompiler(sqlalchemy.sql.compiler.DDLCompiler): + """SQLAlchemy compiler for creating foreign tables.""" + + def create_table_constraints(self, _table, _include_foreign_key_constraints=None, **kw): + return "" # Don't generate any constraints. + + def visit_create_table(self, create, **kw): + table = create.element + table._prefixes = ("FOREIGN",) # Add "FOREIGN" before "TABLE". + sql = super().visit_create_table(create, **kw) + table._prefixes = () + return sql + + def post_create_table(self, table): + # Add foreign options at the end. + return f" SERVER grants OPTIONS (schema 'EGRANTSADMIN', table '{table.name.upper()}')" + + def visit_create_column(self, create, first_pk=False, **kw): + column = create.element + sql = super().visit_create_column(create, first_pk, **kw) + if sql and column.primary_key: + # Add "OPTIONS ..." to primary key column. + sql = re.sub(r"^(.*?)( NOT NULL)?$", r"\1 OPTIONS (key 'true')\2", sql) + return sql + + +class ForeignTableDialect(sqlalchemy.dialects.postgresql.dialect): + """SQLAlchemy dialect for creating foreign tables. + + See https://docs.sqlalchemy.org/en/20/dialects/ + """ + + ddl_compiler = ForeignTableDDLCompiler diff --git a/api/tests/src/data_migration/test_setup_foreign_tables.py b/api/tests/src/data_migration/test_setup_foreign_tables.py index d4d84aea5..7a5753283 100644 --- a/api/tests/src/data_migration/test_setup_foreign_tables.py +++ b/api/tests/src/data_migration/test_setup_foreign_tables.py @@ -1,53 +1,57 @@ +import re + import pytest import sqlalchemy import src.db.foreign from src.data_migration.setup_foreign_tables import build_sql -EXPECTED_LOCAL_OPPORTUNITY_SQL = ( - "CREATE TABLE IF NOT EXISTS {}.topportunity " - "(opportunity_id NUMERIC(20) NOT NULL CONSTRAINT topportunity_pkey PRIMARY KEY," - "oppnumber VARCHAR(40)," - "revision_number NUMERIC(20)," - "opptitle VARCHAR(255)," - "owningagency VARCHAR(255)," - "publisheruid VARCHAR(255)," - "listed CHAR(1)," - "oppcategory CHAR(1)," - "initial_opportunity_id NUMERIC(20)," - "modified_comments VARCHAR(2000)," - "created_date DATE," - "last_upd_date DATE," - "creator_id VARCHAR(50)," - "last_upd_id VARCHAR(50)," - "flag_2006 CHAR(1)," - "category_explanation VARCHAR(255)," - "publisher_profile_id NUMERIC(20)," - "is_draft VARCHAR(1))" -) +EXPECTED_LOCAL_OPPORTUNITY_SQL = [ + "CREATE TABLE IF NOT EXISTS __[SCHEMA_grants].topportunity (", + "opportunity_id NUMERIC(20) NOT NULL,", + "oppnumber VARCHAR(40),", + "revision_number NUMERIC(20),", + "opptitle VARCHAR(255),", + "owningagency VARCHAR(255),", + "publisheruid VARCHAR(255),", + "listed CHAR(1),", + "oppcategory CHAR(1),", + "initial_opportunity_id NUMERIC(20),", + "modified_comments VARCHAR(2000),", + "created_date DATE,", + "last_upd_date DATE,", + "creator_id VARCHAR(50),", + "last_upd_id VARCHAR(50),", + "flag_2006 CHAR(1),", + "category_explanation VARCHAR(255),", + "publisher_profile_id NUMERIC(20),", + "is_draft VARCHAR(1),", + "PRIMARY KEY (opportunity_id)", + ")", +] -EXPECTED_NONLOCAL_OPPORTUNITY_SQL = ( - "CREATE FOREIGN TABLE IF NOT EXISTS {}.topportunity " - "(opportunity_id NUMERIC(20) OPTIONS (key 'true') NOT NULL," - "oppnumber VARCHAR(40)," - "revision_number NUMERIC(20)," - "opptitle VARCHAR(255)," - "owningagency VARCHAR(255)," - "publisheruid VARCHAR(255)," - "listed CHAR(1)," - "oppcategory CHAR(1)," - "initial_opportunity_id NUMERIC(20)," - "modified_comments VARCHAR(2000)," - "created_date DATE," - "last_upd_date DATE," - "creator_id VARCHAR(50)," - "last_upd_id VARCHAR(50)," - "flag_2006 CHAR(1)," - "category_explanation VARCHAR(255)," - "publisher_profile_id NUMERIC(20)," - "is_draft VARCHAR(1))" - " SERVER grants OPTIONS (schema 'EGRANTSADMIN', table 'TOPPORTUNITY')" -) +EXPECTED_NONLOCAL_OPPORTUNITY_SQL = [ + "CREATE FOREIGN TABLE IF NOT EXISTS __[SCHEMA_grants].topportunity (", + "opportunity_id NUMERIC(20) OPTIONS (key 'true') NOT NULL,", + "oppnumber VARCHAR(40),", + "revision_number NUMERIC(20),", + "opptitle VARCHAR(255),", + "owningagency VARCHAR(255),", + "publisheruid VARCHAR(255),", + "listed CHAR(1),", + "oppcategory CHAR(1),", + "initial_opportunity_id NUMERIC(20),", + "modified_comments VARCHAR(2000),", + "created_date DATE,", + "last_upd_date DATE,", + "creator_id VARCHAR(50),", + "last_upd_id VARCHAR(50),", + "flag_2006 CHAR(1),", + "category_explanation VARCHAR(255),", + "publisher_profile_id NUMERIC(20),", + "is_draft VARCHAR(1)", + ") SERVER grants OPTIONS (schema 'EGRANTSADMIN', table 'TOPPORTUNITY')", +] TEST_METADATA = sqlalchemy.MetaData() @@ -56,18 +60,21 @@ TEST_METADATA, sqlalchemy.Column("id", sqlalchemy.Integer, nullable=False, primary_key=True), sqlalchemy.Column("description", sqlalchemy.Text), + schema="schema1", ) -EXPECTED_LOCAL_TEST_SQL = ( - "CREATE TABLE IF NOT EXISTS {}.test_table " - "(id INTEGER NOT NULL CONSTRAINT test_table_pkey PRIMARY KEY," - "description TEXT)" -) -EXPECTED_NONLOCAL_TEST_SQL = ( - "CREATE FOREIGN TABLE IF NOT EXISTS {}.test_table " - "(id INTEGER OPTIONS (key 'true') NOT NULL," - "description TEXT)" - " SERVER grants OPTIONS (schema 'EGRANTSADMIN', table 'TEST_TABLE')" -) +EXPECTED_LOCAL_TEST_SQL = [ + "CREATE TABLE IF NOT EXISTS __[SCHEMA_schema1].test_table (", + "id INTEGER NOT NULL,", + "description TEXT,", + "PRIMARY KEY (id)", + ")", +] +EXPECTED_NONLOCAL_TEST_SQL = [ + "CREATE FOREIGN TABLE IF NOT EXISTS __[SCHEMA_schema1].test_table (", + "id INTEGER OPTIONS (key 'true') NOT NULL,", + "description TEXT", + ") SERVER grants OPTIONS (schema 'EGRANTSADMIN', table 'TEST_TABLE')", +] @pytest.mark.parametrize( @@ -90,4 +97,4 @@ def test_build_sql(table, is_local, expected_sql, test_foreign_schema): sql = build_sql(table, is_local, test_foreign_schema) - assert sql == expected_sql.format(test_foreign_schema) + assert re.split(r"\s*\n\s*", sql) == expected_sql From 0c6b2feb2e6a0de1efa4cd41b8f983d7e75f9663 Mon Sep 17 00:00:00 2001 From: James Bursa Date: Thu, 18 Apr 2024 13:39:31 -0400 Subject: [PATCH 03/15] Convert to basic column types: `integer`, `text`, etc. --- api/src/db/foreign/opportunity.py | 40 ++++++++++++++++--------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/api/src/db/foreign/opportunity.py b/api/src/db/foreign/opportunity.py index b94ad16d2..e43700b82 100644 --- a/api/src/db/foreign/opportunity.py +++ b/api/src/db/foreign/opportunity.py @@ -2,8 +2,10 @@ # SQLAlchemy models for foreign tables. # +import datetime + import sqlalchemy -from sqlalchemy.orm import mapped_column +from sqlalchemy.orm import Mapped, mapped_column from . import base @@ -11,21 +13,21 @@ class Topportunity(base.Base): __tablename__ = "topportunity" - opportunity_id = mapped_column(sqlalchemy.NUMERIC(20), nullable=False, primary_key=True) - oppnumber = mapped_column(sqlalchemy.VARCHAR(40)) - revision_number = mapped_column(sqlalchemy.NUMERIC(20)) - opptitle = mapped_column(sqlalchemy.VARCHAR(255)) - owningagency = mapped_column(sqlalchemy.VARCHAR(255)) - publisheruid = mapped_column(sqlalchemy.VARCHAR(255)) - listed = mapped_column(sqlalchemy.CHAR(1)) - oppcategory = mapped_column(sqlalchemy.CHAR(1)) - initial_opportunity_id = mapped_column(sqlalchemy.NUMERIC(20)) - modified_comments = mapped_column(sqlalchemy.VARCHAR(2000)) - created_date = mapped_column(sqlalchemy.DATE) - last_upd_date = mapped_column(sqlalchemy.DATE) - creator_id = mapped_column(sqlalchemy.VARCHAR(50)) - last_upd_id = mapped_column(sqlalchemy.VARCHAR(50)) - flag_2006 = mapped_column(sqlalchemy.CHAR(1)) - category_explanation = mapped_column(sqlalchemy.VARCHAR(255)) - publisher_profile_id = mapped_column(sqlalchemy.NUMERIC(20)) - is_draft = mapped_column(sqlalchemy.VARCHAR(1)) + opportunity_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + oppnumber: Mapped[str | None] + revision_number: Mapped[int] = mapped_column(sqlalchemy.BigInteger) + opptitle: Mapped[str | None] + owningagency: Mapped[str | None] + publisheruid: Mapped[str | None] + listed: Mapped[str | None] + oppcategory: Mapped[str | None] + initial_opportunity_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger) + modified_comments: Mapped[str | None] + created_date: Mapped[datetime.datetime] = mapped_column(sqlalchemy.TIMESTAMP(timezone=True)) + last_upd_date: Mapped[datetime.datetime] = mapped_column(sqlalchemy.TIMESTAMP(timezone=True)) + creator_id: Mapped[str | None] + last_upd_id: Mapped[str | None] + flag_2006: Mapped[str | None] + category_explanation: Mapped[str | None] + publisher_profile_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger) + is_draft: Mapped[str | None] From 76a9a4b5dda95fc4b6b18b9edcf85d6c0de533ec Mon Sep 17 00:00:00 2001 From: James Bursa Date: Thu, 18 Apr 2024 14:09:10 -0400 Subject: [PATCH 04/15] Full set of foreign tables --- api/src/db/foreign/__init__.py | 2 +- api/src/db/foreign/forecast.py | 224 ++++++++++++++++++++++++++++++ api/src/db/foreign/opportunity.py | 20 +++ api/src/db/foreign/synopsis.py | 218 +++++++++++++++++++++++++++++ 4 files changed, 463 insertions(+), 1 deletion(-) create mode 100644 api/src/db/foreign/forecast.py create mode 100644 api/src/db/foreign/synopsis.py diff --git a/api/src/db/foreign/__init__.py b/api/src/db/foreign/__init__.py index 99c158814..d1f28b3fd 100644 --- a/api/src/db/foreign/__init__.py +++ b/api/src/db/foreign/__init__.py @@ -2,6 +2,6 @@ # SQLAlchemy models for foreign tables. # -from . import base, opportunity +from . import base, forecast, opportunity, synopsis metadata = base.metadata diff --git a/api/src/db/foreign/forecast.py b/api/src/db/foreign/forecast.py new file mode 100644 index 000000000..26afcfc4e --- /dev/null +++ b/api/src/db/foreign/forecast.py @@ -0,0 +1,224 @@ +# +# SQLAlchemy models for foreign tables. +# + +import datetime + +import sqlalchemy +from sqlalchemy.orm import Mapped, mapped_column + +from . import base + + +class Tforecast(base.Base): + __tablename__ = "tforecast" + + version_nbr: Mapped[int] = mapped_column(sqlalchemy.BigInteger) + sendmail: Mapped[str | None] + publisher_profile_id: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger) + publisheruid: Mapped[str | None] + posting_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + oth_cat_fa_desc: Mapped[str | None] + opportunity_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + number_of_awards: Mapped[str | None] + modification_comments: Mapped[str | None] + last_upd_id: Mapped[str | None] + last_upd_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + forecast_desc: Mapped[str | None] + fiscal_year: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger) + fd_link_url: Mapped[str | None] + fd_link_desc: Mapped[str | None] + est_synopsis_posting_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + est_project_start_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + est_funding: Mapped[str | None] + est_award_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + est_appl_response_date_desc: Mapped[str | None] + est_appl_response_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + creator_id: Mapped[str | None] + create_ts: Mapped[datetime.datetime] = mapped_column(sqlalchemy.TIMESTAMP(timezone=True)) + created_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + cost_sharing: Mapped[str | None] + award_floor: Mapped[str | None] + award_ceiling: Mapped[str | None] + archive_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + applicant_elig_desc: Mapped[str | None] + agency_code: Mapped[str | None] + ac_phone: Mapped[str | None] + ac_name: Mapped[str | None] + ac_email_desc: Mapped[str | None] + ac_email_addr: Mapped[str | None] + + +class TforecastHist(base.Base): + __tablename__ = "tforecast_hist" + + created_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + cost_sharing: Mapped[str | None] + award_floor: Mapped[str | None] + award_ceiling: Mapped[str | None] + archive_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + applicant_elig_desc: Mapped[str | None] + agency_code: Mapped[str | None] + ac_phone: Mapped[str | None] + ac_name: Mapped[str | None] + ac_email_desc: Mapped[str | None] + ac_email_addr: Mapped[str | None] + action_type: Mapped[str | None] + action_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + version_nbr: Mapped[int] = mapped_column(sqlalchemy.BigInteger) + sendmail: Mapped[str | None] + revision_number: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + publisher_profile_id: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger) + publisheruid: Mapped[str | None] + posting_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + oth_cat_fa_desc: Mapped[str | None] + opportunity_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + number_of_awards: Mapped[str | None] + modification_comments: Mapped[str | None] + last_upd_id: Mapped[str | None] + last_upd_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + forecast_desc: Mapped[str | None] + fiscal_year: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger) + fd_link_url: Mapped[str | None] + fd_link_desc: Mapped[str | None] + est_synopsis_posting_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + est_project_start_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + est_funding: Mapped[str | None] + est_award_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + est_appl_response_date_desc: Mapped[str | None] + est_appl_response_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + creator_id: Mapped[str | None] + create_ts: Mapped[datetime.datetime] = mapped_column(sqlalchemy.TIMESTAMP(timezone=True)) + + +class TapplicanttypesForecast(base.Base): + __tablename__ = "tapplicanttypes_forecast" + + opportunity_id: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + last_upd_id: Mapped[str | None] + last_upd_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + creator_id: Mapped[str | None] + created_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + at_id: Mapped[str | None] = mapped_column(primary_key=True) + at_frcst_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + + +class TapplicanttypesForecastHist(base.Base): + __tablename__ = "tapplicanttypes_forecast_hist" + + revision_number: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + opportunity_id: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + last_upd_id: Mapped[str | None] + last_upd_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + creator_id: Mapped[str | None] + created_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + at_id: Mapped[str | None] = mapped_column(primary_key=True) + at_frcst_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + + +class TfundactcatForecast(base.Base): + __tablename__ = "tfundactcat_forecast" + + opportunity_id: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + last_upd_id: Mapped[str | None] + last_upd_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + fac_id: Mapped[str | None] = mapped_column(primary_key=True) + fac_frcst_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + creator_id: Mapped[str | None] + created_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + + +class TfundactcatForecastHist(base.Base): + __tablename__ = "tfundactcat_forecast_hist" + + revision_number: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + opportunity_id: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + last_upd_id: Mapped[str | None] + last_upd_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + fac_id: Mapped[str | None] = mapped_column(primary_key=True) + fac_frcst_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + creator_id: Mapped[str | None] + created_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + + +class TfundinstrForecast(base.Base): + __tablename__ = "tfundinstr_forecast" + + opportunity_id: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + last_upd_id: Mapped[str | None] + last_upd_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + fi_id: Mapped[str | None] = mapped_column(primary_key=True) + fi_frcst_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + creator_id: Mapped[str | None] + created_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + + +class TfundinstrForecastHist(base.Base): + __tablename__ = "tfundinstr_forecast_hist" + + revision_number: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + opportunity_id: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + last_upd_id: Mapped[str | None] + last_upd_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + fi_id: Mapped[str | None] = mapped_column(primary_key=True) + fi_frcst_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + creator_id: Mapped[str | None] + created_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) diff --git a/api/src/db/foreign/opportunity.py b/api/src/db/foreign/opportunity.py index e43700b82..c10510ff4 100644 --- a/api/src/db/foreign/opportunity.py +++ b/api/src/db/foreign/opportunity.py @@ -31,3 +31,23 @@ class Topportunity(base.Base): category_explanation: Mapped[str | None] publisher_profile_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger) is_draft: Mapped[str | None] + + +class TopportunityCfda(base.Base): + __tablename__ = "topportunity_cfda" + + programtitle: Mapped[str | None] + origtoppid: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger) + origoppnum: Mapped[str | None] + opp_cfda_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + opportunity_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger) + oppidcfdanum: Mapped[str | None] + last_upd_id: Mapped[str | None] + last_upd_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + creator_id: Mapped[str | None] + created_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + cfdanumber: Mapped[str | None] diff --git a/api/src/db/foreign/synopsis.py b/api/src/db/foreign/synopsis.py new file mode 100644 index 000000000..4d2de6c9a --- /dev/null +++ b/api/src/db/foreign/synopsis.py @@ -0,0 +1,218 @@ +# +# SQLAlchemy models for foreign tables. +# + +import datetime + +import sqlalchemy +from sqlalchemy.orm import Mapped, mapped_column + +from . import base + + +class Tsynopsis(base.Base): + __tablename__ = "tsynopsis" + + version_nbr: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger) + unarchive_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + syn_desc: Mapped[str | None] + sendmail: Mapped[str | None] + response_date_desc: Mapped[str | None] + response_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + publisher_profile_id: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger) + publisheruid: Mapped[str | None] + posting_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + oth_cat_fa_desc: Mapped[str | None] + opportunity_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + number_of_awards: Mapped[str | None] + modification_comments: Mapped[str | None] + last_upd_id: Mapped[str | None] + last_upd_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + fd_link_url: Mapped[str | None] + fd_link_desc: Mapped[str | None] + est_funding: Mapped[str | None] + creator_id: Mapped[str | None] + create_ts: Mapped[datetime.datetime] = mapped_column(sqlalchemy.TIMESTAMP(timezone=True)) + created_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + cost_sharing: Mapped[str | None] + a_sa_code: Mapped[str | None] + award_floor: Mapped[str | None] + award_ceiling: Mapped[str | None] + archive_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + applicant_elig_desc: Mapped[str | None] + agency_phone: Mapped[str | None] + agency_name: Mapped[str | None] + agency_contact_desc: Mapped[str | None] + agency_addr_desc: Mapped[str | None] + ac_phone_number: Mapped[str | None] + ac_name: Mapped[str | None] + ac_email_desc: Mapped[str | None] + ac_email_addr: Mapped[str | None] + + +class TsynopsisHist(base.Base): + __tablename__ = "tsynopsis_hist" + + version_nbr: Mapped[int] = mapped_column(sqlalchemy.BigInteger) + unarchive_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + syn_desc: Mapped[str | None] + sendmail: Mapped[str | None] + revision_number: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + response_date_desc: Mapped[str | None] + response_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + publisher_profile_id: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger) + publisheruid: Mapped[str | None] + posting_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + oth_cat_fa_desc: Mapped[str | None] + opportunity_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + number_of_awards: Mapped[str | None] + modification_comments: Mapped[str | None] + last_upd_id: Mapped[str | None] + last_upd_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + fd_link_url: Mapped[str | None] + fd_link_desc: Mapped[str | None] + est_funding: Mapped[str | None] + creator_id: Mapped[str | None] + create_ts: Mapped[datetime.datetime] = mapped_column(sqlalchemy.TIMESTAMP(timezone=True)) + created_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + cost_sharing: Mapped[str | None] + a_sa_code: Mapped[str | None] + award_floor: Mapped[str | None] + award_ceiling: Mapped[str | None] + archive_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + applicant_elig_desc: Mapped[str | None] + agency_phone: Mapped[str | None] + agency_name: Mapped[str | None] + agency_contact_desc: Mapped[str | None] + agency_addr_desc: Mapped[str | None] + ac_phone_number: Mapped[str | None] + ac_name: Mapped[str | None] + ac_email_desc: Mapped[str | None] + ac_email_addr: Mapped[str | None] + action_type: Mapped[str | None] + action_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + + +class TapplicanttypesSynopsis(base.Base): + __tablename__ = "tapplicanttypes_synopsis" + + opportunity_id: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + last_upd_id: Mapped[str | None] + last_upd_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + creator_id: Mapped[str | None] + created_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + at_syn_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + at_id: Mapped[str | None] = mapped_column(primary_key=True) + + +class TapplicanttypesSynopsisHist(base.Base): + __tablename__ = "tapplicanttypes_synopsis_hist" + + revision_number: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + opportunity_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + last_upd_id: Mapped[str | None] + last_upd_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + creator_id: Mapped[str | None] + created_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + at_syn_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + at_id: Mapped[str] = mapped_column(primary_key=True) + + +class TfundactcatSynopsis(base.Base): + __tablename__ = "tfundactcat_synopsis" + + opportunity_id: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + last_upd_id: Mapped[str | None] + last_upd_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + fac_syn_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + fac_id: Mapped[str | None] = mapped_column(primary_key=True) + creator_id: Mapped[str | None] + created_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + + +class TfundactcatSynopsisHist(base.Base): + __tablename__ = "tfundactcat_synopsis_hist" + + revision_number: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + opportunity_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + last_upd_id: Mapped[str | None] + last_upd_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + fac_syn_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + fac_id: Mapped[str] = mapped_column(primary_key=True) + creator_id: Mapped[str | None] + created_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + + +class TfundinstrSynopsis(base.Base): + __tablename__ = "tfundinstr_synopsis" + + opportunity_id: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + last_upd_id: Mapped[str | None] + last_upd_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + fi_syn_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + fi_id: Mapped[str | None] = mapped_column(primary_key=True) + creator_id: Mapped[str | None] + created_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + + +class TfundinstrSynopsisHist(base.Base): + __tablename__ = "tfundinstr_synopsis_hist" + + revision_number: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + opportunity_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + last_upd_id: Mapped[str | None] + last_upd_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) + fi_syn_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + fi_id: Mapped[str] = mapped_column(primary_key=True) + creator_id: Mapped[str | None] + created_date: Mapped[datetime.datetime | None] = mapped_column( + sqlalchemy.TIMESTAMP(timezone=True) + ) From e19404443ac5142623026405e6725bb1471ba816 Mon Sep 17 00:00:00 2001 From: James Bursa Date: Thu, 18 Apr 2024 14:44:25 -0400 Subject: [PATCH 05/15] Simplify table definitions --- api/src/db/foreign/base.py | 7 + api/src/db/foreign/forecast.py | 168 ++++++------------ api/src/db/foreign/opportunity.py | 27 ++- api/src/db/foreign/synopsis.py | 149 +++++----------- .../test_setup_foreign_tables.py | 74 ++++---- 5 files changed, 159 insertions(+), 266 deletions(-) diff --git a/api/src/db/foreign/base.py b/api/src/db/foreign/base.py index 24f870f15..a4bc838a0 100644 --- a/api/src/db/foreign/base.py +++ b/api/src/db/foreign/base.py @@ -7,6 +7,7 @@ # - Migrations are not used to manage creation and changes as the tables are actually defined in a different system. # +import datetime from typing import Any, Iterable import sqlalchemy @@ -21,6 +22,12 @@ class Base(sqlalchemy.orm.DeclarativeBase): __table_args__ = {"schema": Schemas.FOREIGN} + type_annotation_map = { + int: sqlalchemy.BigInteger, + str: sqlalchemy.Text, + datetime.datetime: sqlalchemy.TIMESTAMP(timezone=True), + } + def _dict(self) -> dict: return {c.key: getattr(self, c.key) for c in sqlalchemy.inspect(self).mapper.column_attrs} diff --git a/api/src/db/foreign/forecast.py b/api/src/db/foreign/forecast.py index 26afcfc4e..d1370aca3 100644 --- a/api/src/db/foreign/forecast.py +++ b/api/src/db/foreign/forecast.py @@ -13,50 +13,34 @@ class Tforecast(base.Base): __tablename__ = "tforecast" - version_nbr: Mapped[int] = mapped_column(sqlalchemy.BigInteger) + version_nbr: Mapped[int] sendmail: Mapped[str | None] - publisher_profile_id: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger) + publisher_profile_id: Mapped[int | None] publisheruid: Mapped[str | None] - posting_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + posting_date: Mapped[datetime.datetime | None] oth_cat_fa_desc: Mapped[str | None] - opportunity_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + opportunity_id: Mapped[int] = mapped_column(primary_key=True) number_of_awards: Mapped[str | None] modification_comments: Mapped[str | None] last_upd_id: Mapped[str | None] - last_upd_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + last_upd_date: Mapped[datetime.datetime | None] forecast_desc: Mapped[str | None] - fiscal_year: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger) + fiscal_year: Mapped[int | None] fd_link_url: Mapped[str | None] fd_link_desc: Mapped[str | None] - est_synopsis_posting_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) - est_project_start_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + est_synopsis_posting_date: Mapped[datetime.datetime | None] + est_project_start_date: Mapped[datetime.datetime | None] est_funding: Mapped[str | None] - est_award_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + est_award_date: Mapped[datetime.datetime | None] est_appl_response_date_desc: Mapped[str | None] - est_appl_response_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + est_appl_response_date: Mapped[datetime.datetime | None] creator_id: Mapped[str | None] - create_ts: Mapped[datetime.datetime] = mapped_column(sqlalchemy.TIMESTAMP(timezone=True)) - created_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + create_ts: Mapped[datetime.datetime] + created_date: Mapped[datetime.datetime | None] cost_sharing: Mapped[str | None] award_floor: Mapped[str | None] award_ceiling: Mapped[str | None] - archive_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + archive_date: Mapped[datetime.datetime | None] applicant_elig_desc: Mapped[str | None] agency_code: Mapped[str | None] ac_phone: Mapped[str | None] @@ -68,15 +52,11 @@ class Tforecast(base.Base): class TforecastHist(base.Base): __tablename__ = "tforecast_hist" - created_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + created_date: Mapped[datetime.datetime | None] cost_sharing: Mapped[str | None] award_floor: Mapped[str | None] award_ceiling: Mapped[str | None] - archive_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + archive_date: Mapped[datetime.datetime | None] applicant_elig_desc: Mapped[str | None] agency_code: Mapped[str | None] ac_phone: Mapped[str | None] @@ -84,141 +64,103 @@ class TforecastHist(base.Base): ac_email_desc: Mapped[str | None] ac_email_addr: Mapped[str | None] action_type: Mapped[str | None] - action_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) - version_nbr: Mapped[int] = mapped_column(sqlalchemy.BigInteger) + action_date: Mapped[datetime.datetime | None] + version_nbr: Mapped[int] sendmail: Mapped[str | None] - revision_number: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) - publisher_profile_id: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger) + revision_number: Mapped[int] = mapped_column(primary_key=True) + publisher_profile_id: Mapped[int | None] publisheruid: Mapped[str | None] - posting_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + posting_date: Mapped[datetime.datetime | None] oth_cat_fa_desc: Mapped[str | None] - opportunity_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + opportunity_id: Mapped[int] = mapped_column(primary_key=True) number_of_awards: Mapped[str | None] modification_comments: Mapped[str | None] last_upd_id: Mapped[str | None] - last_upd_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + last_upd_date: Mapped[datetime.datetime | None] forecast_desc: Mapped[str | None] - fiscal_year: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger) + fiscal_year: Mapped[int | None] fd_link_url: Mapped[str | None] fd_link_desc: Mapped[str | None] - est_synopsis_posting_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) - est_project_start_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + est_synopsis_posting_date: Mapped[datetime.datetime | None] + est_project_start_date: Mapped[datetime.datetime | None] est_funding: Mapped[str | None] - est_award_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + est_award_date: Mapped[datetime.datetime | None] est_appl_response_date_desc: Mapped[str | None] - est_appl_response_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + est_appl_response_date: Mapped[datetime.datetime | None] creator_id: Mapped[str | None] - create_ts: Mapped[datetime.datetime] = mapped_column(sqlalchemy.TIMESTAMP(timezone=True)) + create_ts: Mapped[datetime.datetime] class TapplicanttypesForecast(base.Base): __tablename__ = "tapplicanttypes_forecast" - opportunity_id: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + opportunity_id: Mapped[int | None] = mapped_column(primary_key=True) last_upd_id: Mapped[str | None] - last_upd_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + last_upd_date: Mapped[datetime.datetime | None] creator_id: Mapped[str | None] - created_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + created_date: Mapped[datetime.datetime | None] at_id: Mapped[str | None] = mapped_column(primary_key=True) - at_frcst_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + at_frcst_id: Mapped[int] = mapped_column(primary_key=True) class TapplicanttypesForecastHist(base.Base): __tablename__ = "tapplicanttypes_forecast_hist" - revision_number: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) - opportunity_id: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + revision_number: Mapped[int] = mapped_column(primary_key=True) + opportunity_id: Mapped[int | None] = mapped_column(primary_key=True) last_upd_id: Mapped[str | None] - last_upd_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + last_upd_date: Mapped[datetime.datetime | None] creator_id: Mapped[str | None] - created_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + created_date: Mapped[datetime.datetime | None] at_id: Mapped[str | None] = mapped_column(primary_key=True) - at_frcst_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + at_frcst_id: Mapped[int] = mapped_column(primary_key=True) class TfundactcatForecast(base.Base): __tablename__ = "tfundactcat_forecast" - opportunity_id: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + opportunity_id: Mapped[int | None] = mapped_column(primary_key=True) last_upd_id: Mapped[str | None] - last_upd_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + last_upd_date: Mapped[datetime.datetime | None] fac_id: Mapped[str | None] = mapped_column(primary_key=True) - fac_frcst_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + fac_frcst_id: Mapped[int] = mapped_column(primary_key=True) creator_id: Mapped[str | None] - created_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + created_date: Mapped[datetime.datetime | None] class TfundactcatForecastHist(base.Base): __tablename__ = "tfundactcat_forecast_hist" - revision_number: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) - opportunity_id: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + revision_number: Mapped[int] = mapped_column(primary_key=True) + opportunity_id: Mapped[int | None] = mapped_column(primary_key=True) last_upd_id: Mapped[str | None] - last_upd_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + last_upd_date: Mapped[datetime.datetime | None] fac_id: Mapped[str | None] = mapped_column(primary_key=True) - fac_frcst_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + fac_frcst_id: Mapped[int] = mapped_column(primary_key=True) creator_id: Mapped[str | None] - created_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + created_date: Mapped[datetime.datetime | None] class TfundinstrForecast(base.Base): __tablename__ = "tfundinstr_forecast" - opportunity_id: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + opportunity_id: Mapped[int | None] = mapped_column(primary_key=True) last_upd_id: Mapped[str | None] - last_upd_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + last_upd_date: Mapped[datetime.datetime | None] fi_id: Mapped[str | None] = mapped_column(primary_key=True) - fi_frcst_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + fi_frcst_id: Mapped[int] = mapped_column(primary_key=True) creator_id: Mapped[str | None] - created_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + created_date: Mapped[datetime.datetime | None] class TfundinstrForecastHist(base.Base): __tablename__ = "tfundinstr_forecast_hist" - revision_number: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) - opportunity_id: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + revision_number: Mapped[int] = mapped_column(primary_key=True) + opportunity_id: Mapped[int | None] = mapped_column(primary_key=True) last_upd_id: Mapped[str | None] - last_upd_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + last_upd_date: Mapped[datetime.datetime | None] fi_id: Mapped[str | None] = mapped_column(primary_key=True) - fi_frcst_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + fi_frcst_id: Mapped[int] = mapped_column(primary_key=True) creator_id: Mapped[str | None] - created_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + created_date: Mapped[datetime.datetime | None] diff --git a/api/src/db/foreign/opportunity.py b/api/src/db/foreign/opportunity.py index c10510ff4..8d8380da5 100644 --- a/api/src/db/foreign/opportunity.py +++ b/api/src/db/foreign/opportunity.py @@ -4,7 +4,6 @@ import datetime -import sqlalchemy from sqlalchemy.orm import Mapped, mapped_column from . import base @@ -13,23 +12,23 @@ class Topportunity(base.Base): __tablename__ = "topportunity" - opportunity_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + opportunity_id: Mapped[int] = mapped_column(primary_key=True) oppnumber: Mapped[str | None] - revision_number: Mapped[int] = mapped_column(sqlalchemy.BigInteger) + revision_number: Mapped[int | None] opptitle: Mapped[str | None] owningagency: Mapped[str | None] publisheruid: Mapped[str | None] listed: Mapped[str | None] oppcategory: Mapped[str | None] - initial_opportunity_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger) + initial_opportunity_id: Mapped[int | None] modified_comments: Mapped[str | None] - created_date: Mapped[datetime.datetime] = mapped_column(sqlalchemy.TIMESTAMP(timezone=True)) - last_upd_date: Mapped[datetime.datetime] = mapped_column(sqlalchemy.TIMESTAMP(timezone=True)) + created_date: Mapped[datetime.datetime] + last_upd_date: Mapped[datetime.datetime] creator_id: Mapped[str | None] last_upd_id: Mapped[str | None] flag_2006: Mapped[str | None] category_explanation: Mapped[str | None] - publisher_profile_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger) + publisher_profile_id: Mapped[int | None] is_draft: Mapped[str | None] @@ -37,17 +36,13 @@ class TopportunityCfda(base.Base): __tablename__ = "topportunity_cfda" programtitle: Mapped[str | None] - origtoppid: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger) + origtoppid: Mapped[int | None] origoppnum: Mapped[str | None] - opp_cfda_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) - opportunity_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger) + opp_cfda_id: Mapped[int] = mapped_column(primary_key=True) + opportunity_id: Mapped[int] oppidcfdanum: Mapped[str | None] last_upd_id: Mapped[str | None] - last_upd_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + last_upd_date: Mapped[datetime.datetime | None] creator_id: Mapped[str | None] - created_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + created_date: Mapped[datetime.datetime | None] cfdanumber: Mapped[str | None] diff --git a/api/src/db/foreign/synopsis.py b/api/src/db/foreign/synopsis.py index 4d2de6c9a..5bc2bfe96 100644 --- a/api/src/db/foreign/synopsis.py +++ b/api/src/db/foreign/synopsis.py @@ -4,7 +4,6 @@ import datetime -import sqlalchemy from sqlalchemy.orm import Mapped, mapped_column from . import base @@ -13,44 +12,32 @@ class Tsynopsis(base.Base): __tablename__ = "tsynopsis" - version_nbr: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger) - unarchive_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + version_nbr: Mapped[int | None] + unarchive_date: Mapped[datetime.datetime | None] syn_desc: Mapped[str | None] sendmail: Mapped[str | None] response_date_desc: Mapped[str | None] - response_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) - publisher_profile_id: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger) + response_date: Mapped[datetime.datetime | None] + publisher_profile_id: Mapped[int | None] publisheruid: Mapped[str | None] - posting_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + posting_date: Mapped[datetime.datetime | None] oth_cat_fa_desc: Mapped[str | None] - opportunity_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + opportunity_id: Mapped[int] = mapped_column(primary_key=True) number_of_awards: Mapped[str | None] modification_comments: Mapped[str | None] last_upd_id: Mapped[str | None] - last_upd_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + last_upd_date: Mapped[datetime.datetime | None] fd_link_url: Mapped[str | None] fd_link_desc: Mapped[str | None] est_funding: Mapped[str | None] creator_id: Mapped[str | None] - create_ts: Mapped[datetime.datetime] = mapped_column(sqlalchemy.TIMESTAMP(timezone=True)) - created_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + create_ts: Mapped[datetime.datetime] + created_date: Mapped[datetime.datetime | None] cost_sharing: Mapped[str | None] a_sa_code: Mapped[str | None] award_floor: Mapped[str | None] award_ceiling: Mapped[str | None] - archive_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + archive_date: Mapped[datetime.datetime | None] applicant_elig_desc: Mapped[str | None] agency_phone: Mapped[str | None] agency_name: Mapped[str | None] @@ -65,45 +52,33 @@ class Tsynopsis(base.Base): class TsynopsisHist(base.Base): __tablename__ = "tsynopsis_hist" - version_nbr: Mapped[int] = mapped_column(sqlalchemy.BigInteger) - unarchive_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + version_nbr: Mapped[int] + unarchive_date: Mapped[datetime.datetime | None] syn_desc: Mapped[str | None] sendmail: Mapped[str | None] - revision_number: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + revision_number: Mapped[int] = mapped_column(primary_key=True) response_date_desc: Mapped[str | None] - response_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) - publisher_profile_id: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger) + response_date: Mapped[datetime.datetime | None] + publisher_profile_id: Mapped[int | None] publisheruid: Mapped[str | None] - posting_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + posting_date: Mapped[datetime.datetime | None] oth_cat_fa_desc: Mapped[str | None] - opportunity_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + opportunity_id: Mapped[int] = mapped_column(primary_key=True) number_of_awards: Mapped[str | None] modification_comments: Mapped[str | None] last_upd_id: Mapped[str | None] - last_upd_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + last_upd_date: Mapped[datetime.datetime | None] fd_link_url: Mapped[str | None] fd_link_desc: Mapped[str | None] est_funding: Mapped[str | None] creator_id: Mapped[str | None] - create_ts: Mapped[datetime.datetime] = mapped_column(sqlalchemy.TIMESTAMP(timezone=True)) - created_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + create_ts: Mapped[datetime.datetime] + created_date: Mapped[datetime.datetime | None] cost_sharing: Mapped[str | None] a_sa_code: Mapped[str | None] award_floor: Mapped[str | None] award_ceiling: Mapped[str | None] - archive_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + archive_date: Mapped[datetime.datetime | None] applicant_elig_desc: Mapped[str | None] agency_phone: Mapped[str | None] agency_name: Mapped[str | None] @@ -114,105 +89,79 @@ class TsynopsisHist(base.Base): ac_email_desc: Mapped[str | None] ac_email_addr: Mapped[str | None] action_type: Mapped[str | None] - action_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + action_date: Mapped[datetime.datetime | None] class TapplicanttypesSynopsis(base.Base): __tablename__ = "tapplicanttypes_synopsis" - opportunity_id: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + opportunity_id: Mapped[int | None] = mapped_column(primary_key=True) last_upd_id: Mapped[str | None] - last_upd_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + last_upd_date: Mapped[datetime.datetime | None] creator_id: Mapped[str | None] - created_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) - at_syn_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + created_date: Mapped[datetime.datetime | None] + at_syn_id: Mapped[int] = mapped_column(primary_key=True) at_id: Mapped[str | None] = mapped_column(primary_key=True) class TapplicanttypesSynopsisHist(base.Base): __tablename__ = "tapplicanttypes_synopsis_hist" - revision_number: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) - opportunity_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + revision_number: Mapped[int] = mapped_column(primary_key=True) + opportunity_id: Mapped[int] = mapped_column(primary_key=True) last_upd_id: Mapped[str | None] - last_upd_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + last_upd_date: Mapped[datetime.datetime | None] creator_id: Mapped[str | None] - created_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) - at_syn_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + created_date: Mapped[datetime.datetime | None] + at_syn_id: Mapped[int] = mapped_column(primary_key=True) at_id: Mapped[str] = mapped_column(primary_key=True) class TfundactcatSynopsis(base.Base): __tablename__ = "tfundactcat_synopsis" - opportunity_id: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + opportunity_id: Mapped[int | None] = mapped_column(primary_key=True) last_upd_id: Mapped[str | None] - last_upd_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) - fac_syn_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + last_upd_date: Mapped[datetime.datetime | None] + fac_syn_id: Mapped[int] = mapped_column(primary_key=True) fac_id: Mapped[str | None] = mapped_column(primary_key=True) creator_id: Mapped[str | None] - created_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + created_date: Mapped[datetime.datetime | None] class TfundactcatSynopsisHist(base.Base): __tablename__ = "tfundactcat_synopsis_hist" - revision_number: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) - opportunity_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + revision_number: Mapped[int] = mapped_column(primary_key=True) + opportunity_id: Mapped[int] = mapped_column(primary_key=True) last_upd_id: Mapped[str | None] - last_upd_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) - fac_syn_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + last_upd_date: Mapped[datetime.datetime | None] + fac_syn_id: Mapped[int] = mapped_column(primary_key=True) fac_id: Mapped[str] = mapped_column(primary_key=True) creator_id: Mapped[str | None] - created_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + created_date: Mapped[datetime.datetime | None] class TfundinstrSynopsis(base.Base): __tablename__ = "tfundinstr_synopsis" - opportunity_id: Mapped[int | None] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + opportunity_id: Mapped[int | None] = mapped_column(primary_key=True) last_upd_id: Mapped[str | None] - last_upd_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) - fi_syn_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + last_upd_date: Mapped[datetime.datetime | None] + fi_syn_id: Mapped[int] = mapped_column(primary_key=True) fi_id: Mapped[str | None] = mapped_column(primary_key=True) creator_id: Mapped[str | None] - created_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + created_date: Mapped[datetime.datetime | None] class TfundinstrSynopsisHist(base.Base): __tablename__ = "tfundinstr_synopsis_hist" - revision_number: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) - opportunity_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + revision_number: Mapped[int] = mapped_column(primary_key=True) + opportunity_id: Mapped[int] = mapped_column(primary_key=True) last_upd_id: Mapped[str | None] - last_upd_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) - fi_syn_id: Mapped[int] = mapped_column(sqlalchemy.BigInteger, primary_key=True) + last_upd_date: Mapped[datetime.datetime | None] + fi_syn_id: Mapped[int] = mapped_column(primary_key=True) fi_id: Mapped[str] = mapped_column(primary_key=True) creator_id: Mapped[str | None] - created_date: Mapped[datetime.datetime | None] = mapped_column( - sqlalchemy.TIMESTAMP(timezone=True) - ) + created_date: Mapped[datetime.datetime | None] diff --git a/api/tests/src/data_migration/test_setup_foreign_tables.py b/api/tests/src/data_migration/test_setup_foreign_tables.py index 7a5753283..2bd64ff19 100644 --- a/api/tests/src/data_migration/test_setup_foreign_tables.py +++ b/api/tests/src/data_migration/test_setup_foreign_tables.py @@ -8,48 +8,48 @@ EXPECTED_LOCAL_OPPORTUNITY_SQL = [ "CREATE TABLE IF NOT EXISTS __[SCHEMA_grants].topportunity (", - "opportunity_id NUMERIC(20) NOT NULL,", - "oppnumber VARCHAR(40),", - "revision_number NUMERIC(20),", - "opptitle VARCHAR(255),", - "owningagency VARCHAR(255),", - "publisheruid VARCHAR(255),", - "listed CHAR(1),", - "oppcategory CHAR(1),", - "initial_opportunity_id NUMERIC(20),", - "modified_comments VARCHAR(2000),", - "created_date DATE,", - "last_upd_date DATE,", - "creator_id VARCHAR(50),", - "last_upd_id VARCHAR(50),", - "flag_2006 CHAR(1),", - "category_explanation VARCHAR(255),", - "publisher_profile_id NUMERIC(20),", - "is_draft VARCHAR(1),", + "opportunity_id BIGSERIAL NOT NULL,", + "oppnumber TEXT,", + "revision_number BIGINT,", + "opptitle TEXT,", + "owningagency TEXT,", + "publisheruid TEXT,", + "listed TEXT,", + "oppcategory TEXT,", + "initial_opportunity_id BIGINT,", + "modified_comments TEXT,", + "created_date TIMESTAMP WITH TIME ZONE NOT NULL,", + "last_upd_date TIMESTAMP WITH TIME ZONE NOT NULL,", + "creator_id TEXT,", + "last_upd_id TEXT,", + "flag_2006 TEXT,", + "category_explanation TEXT,", + "publisher_profile_id BIGINT,", + "is_draft TEXT,", "PRIMARY KEY (opportunity_id)", ")", ] EXPECTED_NONLOCAL_OPPORTUNITY_SQL = [ "CREATE FOREIGN TABLE IF NOT EXISTS __[SCHEMA_grants].topportunity (", - "opportunity_id NUMERIC(20) OPTIONS (key 'true') NOT NULL,", - "oppnumber VARCHAR(40),", - "revision_number NUMERIC(20),", - "opptitle VARCHAR(255),", - "owningagency VARCHAR(255),", - "publisheruid VARCHAR(255),", - "listed CHAR(1),", - "oppcategory CHAR(1),", - "initial_opportunity_id NUMERIC(20),", - "modified_comments VARCHAR(2000),", - "created_date DATE,", - "last_upd_date DATE,", - "creator_id VARCHAR(50),", - "last_upd_id VARCHAR(50),", - "flag_2006 CHAR(1),", - "category_explanation VARCHAR(255),", - "publisher_profile_id NUMERIC(20),", - "is_draft VARCHAR(1)", + "opportunity_id BIGINT OPTIONS (key 'true') NOT NULL,", + "oppnumber TEXT,", + "revision_number BIGINT,", + "opptitle TEXT,", + "owningagency TEXT,", + "publisheruid TEXT,", + "listed TEXT,", + "oppcategory TEXT,", + "initial_opportunity_id BIGINT,", + "modified_comments TEXT,", + "created_date TIMESTAMP WITH TIME ZONE NOT NULL,", + "last_upd_date TIMESTAMP WITH TIME ZONE NOT NULL,", + "creator_id TEXT,", + "last_upd_id TEXT,", + "flag_2006 TEXT,", + "category_explanation TEXT,", + "publisher_profile_id BIGINT,", + "is_draft TEXT", ") SERVER grants OPTIONS (schema 'EGRANTSADMIN', table 'TOPPORTUNITY')", ] @@ -64,7 +64,7 @@ ) EXPECTED_LOCAL_TEST_SQL = [ "CREATE TABLE IF NOT EXISTS __[SCHEMA_schema1].test_table (", - "id INTEGER NOT NULL,", + "id SERIAL NOT NULL,", "description TEXT,", "PRIMARY KEY (id)", ")", From 9944d7d53f12c346316ac23cde689f5411be5d56 Mon Sep 17 00:00:00 2001 From: James Bursa Date: Fri, 19 Apr 2024 15:04:48 -0400 Subject: [PATCH 06/15] Fix column orders --- .../data_migration/setup_foreign_tables.py | 5 +- api/src/db/foreign/__init__.py | 2 + api/src/db/foreign/forecast.py | 162 ++++++++--------- api/src/db/foreign/opportunity.py | 15 +- api/src/db/foreign/synopsis.py | 163 +++++++++--------- 5 files changed, 180 insertions(+), 167 deletions(-) diff --git a/api/src/data_migration/setup_foreign_tables.py b/api/src/data_migration/setup_foreign_tables.py index 3cef180a3..b985b964d 100644 --- a/api/src/data_migration/setup_foreign_tables.py +++ b/api/src/data_migration/setup_foreign_tables.py @@ -53,7 +53,10 @@ def build_sql(table: sqlalchemy.schema.Table, is_local: bool, schema_name: str) create_table = sqlalchemy.schema.CreateTable(table, if_not_exists=True) if is_local: - compiler = create_table.compile(schema_translate_map={"grants": schema_name}) + compiler = create_table.compile( + dialect=sqlalchemy.dialects.postgresql.dialect(), + schema_translate_map={"grants": schema_name}, + ) else: compiler = create_table.compile( dialect=src.db.foreign.dialect.ForeignTableDialect(), diff --git a/api/src/db/foreign/__init__.py b/api/src/db/foreign/__init__.py index d1f28b3fd..e4ddc1985 100644 --- a/api/src/db/foreign/__init__.py +++ b/api/src/db/foreign/__init__.py @@ -5,3 +5,5 @@ from . import base, forecast, opportunity, synopsis metadata = base.metadata + +__all__ = ["metadata", "forecast", "opportunity", "synopsis"] diff --git a/api/src/db/foreign/forecast.py b/api/src/db/foreign/forecast.py index d1370aca3..07eab0d55 100644 --- a/api/src/db/foreign/forecast.py +++ b/api/src/db/foreign/forecast.py @@ -1,10 +1,12 @@ # # SQLAlchemy models for foreign tables. # +# The order of the columns must match the remote Oracle database. The names are not required to +# match by oracle_fdw, but we are matching them for maintainability. +# import datetime -import sqlalchemy from sqlalchemy.orm import Mapped, mapped_column from . import base @@ -13,154 +15,154 @@ class Tforecast(base.Base): __tablename__ = "tforecast" + opportunity_id: Mapped[int] = mapped_column(primary_key=True) version_nbr: Mapped[int] - sendmail: Mapped[str | None] - publisher_profile_id: Mapped[int | None] - publisheruid: Mapped[str | None] posting_date: Mapped[datetime.datetime | None] + archive_date: Mapped[datetime.datetime | None] + forecast_desc: Mapped[str | None] oth_cat_fa_desc: Mapped[str | None] - opportunity_id: Mapped[int] = mapped_column(primary_key=True) + cost_sharing: Mapped[str | None] number_of_awards: Mapped[str | None] - modification_comments: Mapped[str | None] - last_upd_id: Mapped[str | None] - last_upd_date: Mapped[datetime.datetime | None] - forecast_desc: Mapped[str | None] - fiscal_year: Mapped[int | None] + est_funding: Mapped[str | None] + award_ceiling: Mapped[str | None] + award_floor: Mapped[str | None] fd_link_url: Mapped[str | None] fd_link_desc: Mapped[str | None] + ac_name: Mapped[str | None] + ac_phone: Mapped[str | None] + ac_email_addr: Mapped[str | None] + ac_email_desc: Mapped[str | None] + agency_code: Mapped[str | None] + sendmail: Mapped[str | None] + applicant_elig_desc: Mapped[str | None] est_synopsis_posting_date: Mapped[datetime.datetime | None] - est_project_start_date: Mapped[datetime.datetime | None] - est_funding: Mapped[str | None] - est_award_date: Mapped[datetime.datetime | None] - est_appl_response_date_desc: Mapped[str | None] est_appl_response_date: Mapped[datetime.datetime | None] - creator_id: Mapped[str | None] + est_appl_response_date_desc: Mapped[str | None] + est_award_date: Mapped[datetime.datetime | None] + est_project_start_date: Mapped[datetime.datetime | None] + fiscal_year: Mapped[int | None] + modification_comments: Mapped[str | None] create_ts: Mapped[datetime.datetime] created_date: Mapped[datetime.datetime | None] - cost_sharing: Mapped[str | None] - award_floor: Mapped[str | None] - award_ceiling: Mapped[str | None] - archive_date: Mapped[datetime.datetime | None] - applicant_elig_desc: Mapped[str | None] - agency_code: Mapped[str | None] - ac_phone: Mapped[str | None] - ac_name: Mapped[str | None] - ac_email_desc: Mapped[str | None] - ac_email_addr: Mapped[str | None] + last_upd_date: Mapped[datetime.datetime | None] + creator_id: Mapped[str | None] + last_upd_id: Mapped[str | None] + publisheruid: Mapped[str | None] + publisher_profile_id: Mapped[int | None] class TforecastHist(base.Base): __tablename__ = "tforecast_hist" - created_date: Mapped[datetime.datetime | None] - cost_sharing: Mapped[str | None] - award_floor: Mapped[str | None] - award_ceiling: Mapped[str | None] - archive_date: Mapped[datetime.datetime | None] - applicant_elig_desc: Mapped[str | None] - agency_code: Mapped[str | None] - ac_phone: Mapped[str | None] - ac_name: Mapped[str | None] - ac_email_desc: Mapped[str | None] - ac_email_addr: Mapped[str | None] - action_type: Mapped[str | None] - action_date: Mapped[datetime.datetime | None] - version_nbr: Mapped[int] - sendmail: Mapped[str | None] + opportunity_id: Mapped[int] = mapped_column(primary_key=True) revision_number: Mapped[int] = mapped_column(primary_key=True) - publisher_profile_id: Mapped[int | None] - publisheruid: Mapped[str | None] + version_nbr: Mapped[int] posting_date: Mapped[datetime.datetime | None] + archive_date: Mapped[datetime.datetime | None] + forecast_desc: Mapped[str | None] oth_cat_fa_desc: Mapped[str | None] - opportunity_id: Mapped[int] = mapped_column(primary_key=True) + cost_sharing: Mapped[str | None] number_of_awards: Mapped[str | None] - modification_comments: Mapped[str | None] - last_upd_id: Mapped[str | None] - last_upd_date: Mapped[datetime.datetime | None] - forecast_desc: Mapped[str | None] - fiscal_year: Mapped[int | None] + est_funding: Mapped[str | None] + award_ceiling: Mapped[str | None] + award_floor: Mapped[str | None] fd_link_url: Mapped[str | None] fd_link_desc: Mapped[str | None] + ac_name: Mapped[str | None] + ac_phone: Mapped[str | None] + ac_email_addr: Mapped[str | None] + ac_email_desc: Mapped[str | None] + agency_code: Mapped[str | None] + sendmail: Mapped[str | None] + applicant_elig_desc: Mapped[str | None] est_synopsis_posting_date: Mapped[datetime.datetime | None] - est_project_start_date: Mapped[datetime.datetime | None] - est_funding: Mapped[str | None] - est_award_date: Mapped[datetime.datetime | None] - est_appl_response_date_desc: Mapped[str | None] est_appl_response_date: Mapped[datetime.datetime | None] - creator_id: Mapped[str | None] + est_appl_response_date_desc: Mapped[str | None] + est_award_date: Mapped[datetime.datetime | None] + est_project_start_date: Mapped[datetime.datetime | None] + fiscal_year: Mapped[int | None] + modification_comments: Mapped[str | None] + action_type: Mapped[str | None] + action_date: Mapped[datetime.datetime | None] create_ts: Mapped[datetime.datetime] + created_date: Mapped[datetime.datetime | None] + last_upd_date: Mapped[datetime.datetime | None] + creator_id: Mapped[str | None] + last_upd_id: Mapped[str | None] + publisheruid: Mapped[str | None] + publisher_profile_id: Mapped[int | None] class TapplicanttypesForecast(base.Base): __tablename__ = "tapplicanttypes_forecast" + at_frcst_id: Mapped[int] = mapped_column(primary_key=True) + at_id: Mapped[str | None] = mapped_column(primary_key=True) opportunity_id: Mapped[int | None] = mapped_column(primary_key=True) - last_upd_id: Mapped[str | None] + created_date: Mapped[datetime.datetime | None] last_upd_date: Mapped[datetime.datetime | None] creator_id: Mapped[str | None] - created_date: Mapped[datetime.datetime | None] - at_id: Mapped[str | None] = mapped_column(primary_key=True) - at_frcst_id: Mapped[int] = mapped_column(primary_key=True) + last_upd_id: Mapped[str | None] class TapplicanttypesForecastHist(base.Base): __tablename__ = "tapplicanttypes_forecast_hist" - revision_number: Mapped[int] = mapped_column(primary_key=True) + at_frcst_id: Mapped[int] = mapped_column(primary_key=True) + at_id: Mapped[str | None] = mapped_column(primary_key=True) opportunity_id: Mapped[int | None] = mapped_column(primary_key=True) - last_upd_id: Mapped[str | None] + revision_number: Mapped[int] = mapped_column(primary_key=True) + created_date: Mapped[datetime.datetime | None] last_upd_date: Mapped[datetime.datetime | None] creator_id: Mapped[str | None] - created_date: Mapped[datetime.datetime | None] - at_id: Mapped[str | None] = mapped_column(primary_key=True) - at_frcst_id: Mapped[int] = mapped_column(primary_key=True) + last_upd_id: Mapped[str | None] class TfundactcatForecast(base.Base): __tablename__ = "tfundactcat_forecast" + fac_frcst_id: Mapped[int] = mapped_column(primary_key=True) + fac_id: Mapped[str | None] = mapped_column(primary_key=True) opportunity_id: Mapped[int | None] = mapped_column(primary_key=True) - last_upd_id: Mapped[str | None] + created_date: Mapped[datetime.datetime | None] last_upd_date: Mapped[datetime.datetime | None] - fac_id: Mapped[str | None] = mapped_column(primary_key=True) - fac_frcst_id: Mapped[int] = mapped_column(primary_key=True) creator_id: Mapped[str | None] - created_date: Mapped[datetime.datetime | None] + last_upd_id: Mapped[str | None] class TfundactcatForecastHist(base.Base): __tablename__ = "tfundactcat_forecast_hist" - revision_number: Mapped[int] = mapped_column(primary_key=True) + fac_frcst_id: Mapped[int] = mapped_column(primary_key=True) + fac_id: Mapped[str | None] = mapped_column(primary_key=True) opportunity_id: Mapped[int | None] = mapped_column(primary_key=True) - last_upd_id: Mapped[str | None] + revision_number: Mapped[int] = mapped_column(primary_key=True) + created_date: Mapped[datetime.datetime | None] last_upd_date: Mapped[datetime.datetime | None] - fac_id: Mapped[str | None] = mapped_column(primary_key=True) - fac_frcst_id: Mapped[int] = mapped_column(primary_key=True) creator_id: Mapped[str | None] - created_date: Mapped[datetime.datetime | None] + last_upd_id: Mapped[str | None] class TfundinstrForecast(base.Base): __tablename__ = "tfundinstr_forecast" + fi_frcst_id: Mapped[int] = mapped_column(primary_key=True) + fi_id: Mapped[str | None] = mapped_column(primary_key=True) opportunity_id: Mapped[int | None] = mapped_column(primary_key=True) - last_upd_id: Mapped[str | None] + created_date: Mapped[datetime.datetime | None] last_upd_date: Mapped[datetime.datetime | None] - fi_id: Mapped[str | None] = mapped_column(primary_key=True) - fi_frcst_id: Mapped[int] = mapped_column(primary_key=True) creator_id: Mapped[str | None] - created_date: Mapped[datetime.datetime | None] + last_upd_id: Mapped[str | None] class TfundinstrForecastHist(base.Base): __tablename__ = "tfundinstr_forecast_hist" - revision_number: Mapped[int] = mapped_column(primary_key=True) + fi_frcst_id: Mapped[int] = mapped_column(primary_key=True) + fi_id: Mapped[str | None] = mapped_column(primary_key=True) opportunity_id: Mapped[int | None] = mapped_column(primary_key=True) - last_upd_id: Mapped[str | None] + revision_number: Mapped[int] = mapped_column(primary_key=True) + created_date: Mapped[datetime.datetime | None] last_upd_date: Mapped[datetime.datetime | None] - fi_id: Mapped[str | None] = mapped_column(primary_key=True) - fi_frcst_id: Mapped[int] = mapped_column(primary_key=True) creator_id: Mapped[str | None] - created_date: Mapped[datetime.datetime | None] + last_upd_id: Mapped[str | None] diff --git a/api/src/db/foreign/opportunity.py b/api/src/db/foreign/opportunity.py index 8d8380da5..f191c8c5b 100644 --- a/api/src/db/foreign/opportunity.py +++ b/api/src/db/foreign/opportunity.py @@ -1,6 +1,9 @@ # # SQLAlchemy models for foreign tables. # +# The order of the columns must match the remote Oracle database. The names are not required to +# match by oracle_fdw, but we are matching them for maintainability. +# import datetime @@ -35,14 +38,14 @@ class Topportunity(base.Base): class TopportunityCfda(base.Base): __tablename__ = "topportunity_cfda" - programtitle: Mapped[str | None] - origtoppid: Mapped[int | None] - origoppnum: Mapped[str | None] opp_cfda_id: Mapped[int] = mapped_column(primary_key=True) opportunity_id: Mapped[int] + cfdanumber: Mapped[str | None] + programtitle: Mapped[str | None] + origtoppid: Mapped[int | None] oppidcfdanum: Mapped[str | None] - last_upd_id: Mapped[str | None] + origoppnum: Mapped[str | None] + created_date: Mapped[datetime.datetime | None] last_upd_date: Mapped[datetime.datetime | None] creator_id: Mapped[str | None] - created_date: Mapped[datetime.datetime | None] - cfdanumber: Mapped[str | None] + last_upd_id: Mapped[str | None] diff --git a/api/src/db/foreign/synopsis.py b/api/src/db/foreign/synopsis.py index 5bc2bfe96..fae6946ec 100644 --- a/api/src/db/foreign/synopsis.py +++ b/api/src/db/foreign/synopsis.py @@ -1,6 +1,9 @@ # # SQLAlchemy models for foreign tables. # +# The order of the columns must match the remote Oracle database. The names are not required to +# match by oracle_fdw, but we are matching them for maintainability. +# import datetime @@ -12,156 +15,156 @@ class Tsynopsis(base.Base): __tablename__ = "tsynopsis" - version_nbr: Mapped[int | None] + opportunity_id: Mapped[int] = mapped_column(primary_key=True) + posting_date: Mapped[datetime.datetime | None] + response_date: Mapped[datetime.datetime | None] + archive_date: Mapped[datetime.datetime | None] unarchive_date: Mapped[datetime.datetime | None] syn_desc: Mapped[str | None] - sendmail: Mapped[str | None] - response_date_desc: Mapped[str | None] - response_date: Mapped[datetime.datetime | None] - publisher_profile_id: Mapped[int | None] - publisheruid: Mapped[str | None] - posting_date: Mapped[datetime.datetime | None] oth_cat_fa_desc: Mapped[str | None] - opportunity_id: Mapped[int] = mapped_column(primary_key=True) + agency_addr_desc: Mapped[str | None] + cost_sharing: Mapped[str | None] number_of_awards: Mapped[str | None] - modification_comments: Mapped[str | None] - last_upd_id: Mapped[str | None] - last_upd_date: Mapped[datetime.datetime | None] - fd_link_url: Mapped[str | None] - fd_link_desc: Mapped[str | None] est_funding: Mapped[str | None] - creator_id: Mapped[str | None] - create_ts: Mapped[datetime.datetime] - created_date: Mapped[datetime.datetime | None] - cost_sharing: Mapped[str | None] - a_sa_code: Mapped[str | None] - award_floor: Mapped[str | None] award_ceiling: Mapped[str | None] - archive_date: Mapped[datetime.datetime | None] - applicant_elig_desc: Mapped[str | None] - agency_phone: Mapped[str | None] - agency_name: Mapped[str | None] + award_floor: Mapped[str | None] + fd_link_url: Mapped[str | None] + fd_link_desc: Mapped[str | None] agency_contact_desc: Mapped[str | None] - agency_addr_desc: Mapped[str | None] + ac_email_addr: Mapped[str | None] + ac_email_desc: Mapped[str | None] + agency_name: Mapped[str | None] + agency_phone: Mapped[str | None] + a_sa_code: Mapped[str | None] ac_phone_number: Mapped[str | None] ac_name: Mapped[str | None] - ac_email_desc: Mapped[str | None] - ac_email_addr: Mapped[str | None] + create_ts: Mapped[datetime.datetime] + created_date: Mapped[datetime.datetime | None] + last_upd_date: Mapped[datetime.datetime | None] + creator_id: Mapped[str | None] + last_upd_id: Mapped[str | None] + sendmail: Mapped[str | None] + response_date_desc: Mapped[str | None] + applicant_elig_desc: Mapped[str | None] + version_nbr: Mapped[int | None] + modification_comments: Mapped[str | None] + publisheruid: Mapped[str | None] + publisher_profile_id: Mapped[int | None] class TsynopsisHist(base.Base): __tablename__ = "tsynopsis_hist" - version_nbr: Mapped[int] - unarchive_date: Mapped[datetime.datetime | None] - syn_desc: Mapped[str | None] - sendmail: Mapped[str | None] + opportunity_id: Mapped[int] = mapped_column(primary_key=True) revision_number: Mapped[int] = mapped_column(primary_key=True) - response_date_desc: Mapped[str | None] - response_date: Mapped[datetime.datetime | None] - publisher_profile_id: Mapped[int | None] - publisheruid: Mapped[str | None] posting_date: Mapped[datetime.datetime | None] + response_date: Mapped[datetime.datetime | None] + archive_date: Mapped[datetime.datetime | None] + unarchive_date: Mapped[datetime.datetime | None] + syn_desc: Mapped[str | None] oth_cat_fa_desc: Mapped[str | None] - opportunity_id: Mapped[int] = mapped_column(primary_key=True) + agency_addr_desc: Mapped[str | None] + cost_sharing: Mapped[str | None] number_of_awards: Mapped[str | None] - modification_comments: Mapped[str | None] - last_upd_id: Mapped[str | None] - last_upd_date: Mapped[datetime.datetime | None] - fd_link_url: Mapped[str | None] - fd_link_desc: Mapped[str | None] est_funding: Mapped[str | None] - creator_id: Mapped[str | None] - create_ts: Mapped[datetime.datetime] - created_date: Mapped[datetime.datetime | None] - cost_sharing: Mapped[str | None] - a_sa_code: Mapped[str | None] - award_floor: Mapped[str | None] award_ceiling: Mapped[str | None] - archive_date: Mapped[datetime.datetime | None] - applicant_elig_desc: Mapped[str | None] - agency_phone: Mapped[str | None] - agency_name: Mapped[str | None] + award_floor: Mapped[str | None] + fd_link_url: Mapped[str | None] + fd_link_desc: Mapped[str | None] agency_contact_desc: Mapped[str | None] - agency_addr_desc: Mapped[str | None] + ac_email_addr: Mapped[str | None] + ac_email_desc: Mapped[str | None] + agency_name: Mapped[str | None] + agency_phone: Mapped[str | None] + a_sa_code: Mapped[str | None] ac_phone_number: Mapped[str | None] ac_name: Mapped[str | None] - ac_email_desc: Mapped[str | None] - ac_email_addr: Mapped[str | None] + create_ts: Mapped[datetime.datetime] + created_date: Mapped[datetime.datetime | None] + last_upd_date: Mapped[datetime.datetime | None] + creator_id: Mapped[str | None] + last_upd_id: Mapped[str | None] + sendmail: Mapped[str | None] + response_date_desc: Mapped[str | None] + applicant_elig_desc: Mapped[str | None] action_type: Mapped[str | None] action_date: Mapped[datetime.datetime | None] + version_nbr: Mapped[int] + modification_comments: Mapped[str | None] + publisheruid: Mapped[str | None] + publisher_profile_id: Mapped[int | None] class TapplicanttypesSynopsis(base.Base): __tablename__ = "tapplicanttypes_synopsis" + at_syn_id: Mapped[int] = mapped_column(primary_key=True) + at_id: Mapped[str | None] = mapped_column(primary_key=True) opportunity_id: Mapped[int | None] = mapped_column(primary_key=True) - last_upd_id: Mapped[str | None] + created_date: Mapped[datetime.datetime | None] last_upd_date: Mapped[datetime.datetime | None] creator_id: Mapped[str | None] - created_date: Mapped[datetime.datetime | None] - at_syn_id: Mapped[int] = mapped_column(primary_key=True) - at_id: Mapped[str | None] = mapped_column(primary_key=True) + last_upd_id: Mapped[str | None] class TapplicanttypesSynopsisHist(base.Base): __tablename__ = "tapplicanttypes_synopsis_hist" - revision_number: Mapped[int] = mapped_column(primary_key=True) + at_syn_id: Mapped[int] = mapped_column(primary_key=True) + at_id: Mapped[str] = mapped_column(primary_key=True) opportunity_id: Mapped[int] = mapped_column(primary_key=True) - last_upd_id: Mapped[str | None] + revision_number: Mapped[int] = mapped_column(primary_key=True) + created_date: Mapped[datetime.datetime | None] last_upd_date: Mapped[datetime.datetime | None] creator_id: Mapped[str | None] - created_date: Mapped[datetime.datetime | None] - at_syn_id: Mapped[int] = mapped_column(primary_key=True) - at_id: Mapped[str] = mapped_column(primary_key=True) + last_upd_id: Mapped[str | None] class TfundactcatSynopsis(base.Base): __tablename__ = "tfundactcat_synopsis" - opportunity_id: Mapped[int | None] = mapped_column(primary_key=True) - last_upd_id: Mapped[str | None] - last_upd_date: Mapped[datetime.datetime | None] fac_syn_id: Mapped[int] = mapped_column(primary_key=True) fac_id: Mapped[str | None] = mapped_column(primary_key=True) - creator_id: Mapped[str | None] + opportunity_id: Mapped[int | None] = mapped_column(primary_key=True) created_date: Mapped[datetime.datetime | None] + last_upd_date: Mapped[datetime.datetime | None] + creator_id: Mapped[str | None] + last_upd_id: Mapped[str | None] class TfundactcatSynopsisHist(base.Base): __tablename__ = "tfundactcat_synopsis_hist" - revision_number: Mapped[int] = mapped_column(primary_key=True) - opportunity_id: Mapped[int] = mapped_column(primary_key=True) - last_upd_id: Mapped[str | None] - last_upd_date: Mapped[datetime.datetime | None] fac_syn_id: Mapped[int] = mapped_column(primary_key=True) fac_id: Mapped[str] = mapped_column(primary_key=True) - creator_id: Mapped[str | None] + opportunity_id: Mapped[int] = mapped_column(primary_key=True) + revision_number: Mapped[int] = mapped_column(primary_key=True) created_date: Mapped[datetime.datetime | None] + last_upd_date: Mapped[datetime.datetime | None] + creator_id: Mapped[str | None] + last_upd_id: Mapped[str | None] class TfundinstrSynopsis(base.Base): __tablename__ = "tfundinstr_synopsis" - opportunity_id: Mapped[int | None] = mapped_column(primary_key=True) - last_upd_id: Mapped[str | None] - last_upd_date: Mapped[datetime.datetime | None] fi_syn_id: Mapped[int] = mapped_column(primary_key=True) fi_id: Mapped[str | None] = mapped_column(primary_key=True) - creator_id: Mapped[str | None] + opportunity_id: Mapped[int | None] = mapped_column(primary_key=True) created_date: Mapped[datetime.datetime | None] + last_upd_date: Mapped[datetime.datetime | None] + creator_id: Mapped[str | None] + last_upd_id: Mapped[str | None] class TfundinstrSynopsisHist(base.Base): __tablename__ = "tfundinstr_synopsis_hist" - revision_number: Mapped[int] = mapped_column(primary_key=True) - opportunity_id: Mapped[int] = mapped_column(primary_key=True) - last_upd_id: Mapped[str | None] - last_upd_date: Mapped[datetime.datetime | None] fi_syn_id: Mapped[int] = mapped_column(primary_key=True) + opportunity_id: Mapped[int] = mapped_column(primary_key=True) fi_id: Mapped[str] = mapped_column(primary_key=True) - creator_id: Mapped[str | None] + revision_number: Mapped[int] = mapped_column(primary_key=True) created_date: Mapped[datetime.datetime | None] + last_upd_date: Mapped[datetime.datetime | None] + creator_id: Mapped[str | None] + last_upd_id: Mapped[str | None] From 5b28b7fbef88f8ffec6d6229107f7a3a1b07a395 Mon Sep 17 00:00:00 2001 From: James Bursa Date: Wed, 24 Apr 2024 10:57:02 -0400 Subject: [PATCH 07/15] Update schema names --- api/src/constants/schema.py | 3 ++- api/src/data_migration/copy_oracle_data.py | 2 +- api/src/data_migration/setup_foreign_tables.py | 2 +- api/src/db/foreign/base.py | 2 +- api/src/db/migrations/setup_local_postgres_db.py | 1 - api/tests/conftest.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/src/constants/schema.py b/api/src/constants/schema.py index f233b9674..01d0ce2e4 100644 --- a/api/src/constants/schema.py +++ b/api/src/constants/schema.py @@ -3,4 +3,5 @@ class Schemas(StrEnum): API = "api" - FOREIGN = "grants" + LEGACY = "legacy" + STAGING = "staging" diff --git a/api/src/data_migration/copy_oracle_data.py b/api/src/data_migration/copy_oracle_data.py index 65a5d07d6..0674cfe53 100644 --- a/api/src/data_migration/copy_oracle_data.py +++ b/api/src/data_migration/copy_oracle_data.py @@ -55,7 +55,7 @@ def copy_oracle_data(db_session: db.Session) -> None: try: with db_session.begin(): - _run_copy_commands(db_session, Schemas.API, Schemas.FOREIGN) + _run_copy_commands(db_session, Schemas.API, Schemas.LEGACY) except Exception: logger.exception("Failed to run copy-oracle-data command") raise diff --git a/api/src/data_migration/setup_foreign_tables.py b/api/src/data_migration/setup_foreign_tables.py index b985b964d..bf8e60a8f 100644 --- a/api/src/data_migration/setup_foreign_tables.py +++ b/api/src/data_migration/setup_foreign_tables.py @@ -16,7 +16,7 @@ class ForeignTableConfig(PydanticBaseEnvConfig): is_local_foreign_table: bool = Field(False) - schema_name: str = Field(Schemas.FOREIGN) + schema_name: str = Field(Schemas.LEGACY) @data_migration_blueprint.cli.command( diff --git a/api/src/db/foreign/base.py b/api/src/db/foreign/base.py index a4bc838a0..d8ed3d1e0 100644 --- a/api/src/db/foreign/base.py +++ b/api/src/db/foreign/base.py @@ -20,7 +20,7 @@ class Base(sqlalchemy.orm.DeclarativeBase): metadata = metadata - __table_args__ = {"schema": Schemas.FOREIGN} + __table_args__ = {"schema": Schemas.LEGACY} type_annotation_map = { int: sqlalchemy.BigInteger, diff --git a/api/src/db/migrations/setup_local_postgres_db.py b/api/src/db/migrations/setup_local_postgres_db.py index 27aa16a9c..a34fc2727 100644 --- a/api/src/db/migrations/setup_local_postgres_db.py +++ b/api/src/db/migrations/setup_local_postgres_db.py @@ -1,7 +1,6 @@ import logging import sqlalchemy -from sqlalchemy import text import src.adapters.db as db import src.logging diff --git a/api/tests/conftest.py b/api/tests/conftest.py index 856023dbc..e87f736bf 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -138,7 +138,7 @@ def test_api_schema(db_schema_prefix): @pytest.fixture def test_foreign_schema(db_schema_prefix): - return f"{db_schema_prefix}{Schemas.FOREIGN}" + return f"{db_schema_prefix}{Schemas.LEGACY}" #################### From bfd7d7fb7af0fca6098a10d33545b1c21087cf0a Mon Sep 17 00:00:00 2001 From: James Bursa Date: Wed, 24 Apr 2024 12:47:46 -0400 Subject: [PATCH 08/15] Rename Base to ForeignBase --- api/src/data_migration/setup_foreign_tables.py | 10 ++++++++++ api/src/db/foreign/__init__.py | 4 ++-- api/src/db/foreign/forecast.py | 18 +++++++++--------- api/src/db/foreign/{base.py => foreignbase.py} | 5 ++++- api/src/db/foreign/opportunity.py | 6 +++--- api/src/db/foreign/synopsis.py | 18 +++++++++--------- .../test_setup_foreign_tables.py | 8 ++++---- 7 files changed, 41 insertions(+), 28 deletions(-) rename api/src/db/foreign/{base.py => foreignbase.py} (80%) diff --git a/api/src/data_migration/setup_foreign_tables.py b/api/src/data_migration/setup_foreign_tables.py index bf8e60a8f..16ac74919 100644 --- a/api/src/data_migration/setup_foreign_tables.py +++ b/api/src/data_migration/setup_foreign_tables.py @@ -66,7 +66,17 @@ def build_sql(table: sqlalchemy.schema.Table, is_local: bool, schema_name: str) def _run_create_table_commands(db_session: db.Session, config: ForeignTableConfig) -> None: + sql_file = open("create_foreign_table.sql", "w") + sql_file.write("CREATE SCHEMA IF NOT EXISTS legacy\n") for table in src.db.foreign.metadata.tables.values(): sql = build_sql(table, config.is_local_foreign_table, config.schema_name) logger.info("create table", extra={"table": table.name, "sql": sql}) db_session.execute(sqlalchemy.text(sql)) + sql_file.write(sql.replace("\n", " ").replace("__[SCHEMA_legacy]", "legacy")) + sql_file.write("\n") + + sql_file = open("select_from_foreign_table.sql", "w") + for table in src.db.foreign.metadata.tables.values(): + sql_file.write( + "SELECT * FROM legacy.%s ORDER BY created_date DESC LIMIT 8\n" % (table.name) + ) diff --git a/api/src/db/foreign/__init__.py b/api/src/db/foreign/__init__.py index e4ddc1985..9d55582c3 100644 --- a/api/src/db/foreign/__init__.py +++ b/api/src/db/foreign/__init__.py @@ -2,8 +2,8 @@ # SQLAlchemy models for foreign tables. # -from . import base, forecast, opportunity, synopsis +from . import forecast, foreignbase, opportunity, synopsis -metadata = base.metadata +metadata = foreignbase.metadata __all__ = ["metadata", "forecast", "opportunity", "synopsis"] diff --git a/api/src/db/foreign/forecast.py b/api/src/db/foreign/forecast.py index 07eab0d55..46ade0129 100644 --- a/api/src/db/foreign/forecast.py +++ b/api/src/db/foreign/forecast.py @@ -9,10 +9,10 @@ from sqlalchemy.orm import Mapped, mapped_column -from . import base +from . import foreignbase -class Tforecast(base.Base): +class Tforecast(foreignbase.ForeignBase): __tablename__ = "tforecast" opportunity_id: Mapped[int] = mapped_column(primary_key=True) @@ -51,7 +51,7 @@ class Tforecast(base.Base): publisher_profile_id: Mapped[int | None] -class TforecastHist(base.Base): +class TforecastHist(foreignbase.ForeignBase): __tablename__ = "tforecast_hist" opportunity_id: Mapped[int] = mapped_column(primary_key=True) @@ -93,7 +93,7 @@ class TforecastHist(base.Base): publisher_profile_id: Mapped[int | None] -class TapplicanttypesForecast(base.Base): +class TapplicanttypesForecast(foreignbase.ForeignBase): __tablename__ = "tapplicanttypes_forecast" at_frcst_id: Mapped[int] = mapped_column(primary_key=True) @@ -105,7 +105,7 @@ class TapplicanttypesForecast(base.Base): last_upd_id: Mapped[str | None] -class TapplicanttypesForecastHist(base.Base): +class TapplicanttypesForecastHist(foreignbase.ForeignBase): __tablename__ = "tapplicanttypes_forecast_hist" at_frcst_id: Mapped[int] = mapped_column(primary_key=True) @@ -118,7 +118,7 @@ class TapplicanttypesForecastHist(base.Base): last_upd_id: Mapped[str | None] -class TfundactcatForecast(base.Base): +class TfundactcatForecast(foreignbase.ForeignBase): __tablename__ = "tfundactcat_forecast" fac_frcst_id: Mapped[int] = mapped_column(primary_key=True) @@ -130,7 +130,7 @@ class TfundactcatForecast(base.Base): last_upd_id: Mapped[str | None] -class TfundactcatForecastHist(base.Base): +class TfundactcatForecastHist(foreignbase.ForeignBase): __tablename__ = "tfundactcat_forecast_hist" fac_frcst_id: Mapped[int] = mapped_column(primary_key=True) @@ -143,7 +143,7 @@ class TfundactcatForecastHist(base.Base): last_upd_id: Mapped[str | None] -class TfundinstrForecast(base.Base): +class TfundinstrForecast(foreignbase.ForeignBase): __tablename__ = "tfundinstr_forecast" fi_frcst_id: Mapped[int] = mapped_column(primary_key=True) @@ -155,7 +155,7 @@ class TfundinstrForecast(base.Base): last_upd_id: Mapped[str | None] -class TfundinstrForecastHist(base.Base): +class TfundinstrForecastHist(foreignbase.ForeignBase): __tablename__ = "tfundinstr_forecast_hist" fi_frcst_id: Mapped[int] = mapped_column(primary_key=True) diff --git a/api/src/db/foreign/base.py b/api/src/db/foreign/foreignbase.py similarity index 80% rename from api/src/db/foreign/base.py rename to api/src/db/foreign/foreignbase.py index d8ed3d1e0..5f1b5c7b0 100644 --- a/api/src/db/foreign/base.py +++ b/api/src/db/foreign/foreignbase.py @@ -17,11 +17,14 @@ metadata = sqlalchemy.MetaData() -class Base(sqlalchemy.orm.DeclarativeBase): +class ForeignBase(sqlalchemy.orm.DeclarativeBase): metadata = metadata __table_args__ = {"schema": Schemas.LEGACY} + # These types are selected so that the underlying Oracle types are mapped to a more general + # type. For example all CHAR and VARCHAR types can be mapped to TEXT for simplicity. See + # https://github.com/laurenz/oracle_fdw?tab=readme-ov-file#data-types type_annotation_map = { int: sqlalchemy.BigInteger, str: sqlalchemy.Text, diff --git a/api/src/db/foreign/opportunity.py b/api/src/db/foreign/opportunity.py index f191c8c5b..37a7fa593 100644 --- a/api/src/db/foreign/opportunity.py +++ b/api/src/db/foreign/opportunity.py @@ -9,10 +9,10 @@ from sqlalchemy.orm import Mapped, mapped_column -from . import base +from . import foreignbase -class Topportunity(base.Base): +class Topportunity(foreignbase.ForeignBase): __tablename__ = "topportunity" opportunity_id: Mapped[int] = mapped_column(primary_key=True) @@ -35,7 +35,7 @@ class Topportunity(base.Base): is_draft: Mapped[str | None] -class TopportunityCfda(base.Base): +class TopportunityCfda(foreignbase.ForeignBase): __tablename__ = "topportunity_cfda" opp_cfda_id: Mapped[int] = mapped_column(primary_key=True) diff --git a/api/src/db/foreign/synopsis.py b/api/src/db/foreign/synopsis.py index fae6946ec..7c39b3a72 100644 --- a/api/src/db/foreign/synopsis.py +++ b/api/src/db/foreign/synopsis.py @@ -9,10 +9,10 @@ from sqlalchemy.orm import Mapped, mapped_column -from . import base +from . import foreignbase -class Tsynopsis(base.Base): +class Tsynopsis(foreignbase.ForeignBase): __tablename__ = "tsynopsis" opportunity_id: Mapped[int] = mapped_column(primary_key=True) @@ -52,7 +52,7 @@ class Tsynopsis(base.Base): publisher_profile_id: Mapped[int | None] -class TsynopsisHist(base.Base): +class TsynopsisHist(foreignbase.ForeignBase): __tablename__ = "tsynopsis_hist" opportunity_id: Mapped[int] = mapped_column(primary_key=True) @@ -95,7 +95,7 @@ class TsynopsisHist(base.Base): publisher_profile_id: Mapped[int | None] -class TapplicanttypesSynopsis(base.Base): +class TapplicanttypesSynopsis(foreignbase.ForeignBase): __tablename__ = "tapplicanttypes_synopsis" at_syn_id: Mapped[int] = mapped_column(primary_key=True) @@ -107,7 +107,7 @@ class TapplicanttypesSynopsis(base.Base): last_upd_id: Mapped[str | None] -class TapplicanttypesSynopsisHist(base.Base): +class TapplicanttypesSynopsisHist(foreignbase.ForeignBase): __tablename__ = "tapplicanttypes_synopsis_hist" at_syn_id: Mapped[int] = mapped_column(primary_key=True) @@ -120,7 +120,7 @@ class TapplicanttypesSynopsisHist(base.Base): last_upd_id: Mapped[str | None] -class TfundactcatSynopsis(base.Base): +class TfundactcatSynopsis(foreignbase.ForeignBase): __tablename__ = "tfundactcat_synopsis" fac_syn_id: Mapped[int] = mapped_column(primary_key=True) @@ -132,7 +132,7 @@ class TfundactcatSynopsis(base.Base): last_upd_id: Mapped[str | None] -class TfundactcatSynopsisHist(base.Base): +class TfundactcatSynopsisHist(foreignbase.ForeignBase): __tablename__ = "tfundactcat_synopsis_hist" fac_syn_id: Mapped[int] = mapped_column(primary_key=True) @@ -145,7 +145,7 @@ class TfundactcatSynopsisHist(base.Base): last_upd_id: Mapped[str | None] -class TfundinstrSynopsis(base.Base): +class TfundinstrSynopsis(foreignbase.ForeignBase): __tablename__ = "tfundinstr_synopsis" fi_syn_id: Mapped[int] = mapped_column(primary_key=True) @@ -157,7 +157,7 @@ class TfundinstrSynopsis(base.Base): last_upd_id: Mapped[str | None] -class TfundinstrSynopsisHist(base.Base): +class TfundinstrSynopsisHist(foreignbase.ForeignBase): __tablename__ = "tfundinstr_synopsis_hist" fi_syn_id: Mapped[int] = mapped_column(primary_key=True) diff --git a/api/tests/src/data_migration/test_setup_foreign_tables.py b/api/tests/src/data_migration/test_setup_foreign_tables.py index 2bd64ff19..400415f9b 100644 --- a/api/tests/src/data_migration/test_setup_foreign_tables.py +++ b/api/tests/src/data_migration/test_setup_foreign_tables.py @@ -7,7 +7,7 @@ from src.data_migration.setup_foreign_tables import build_sql EXPECTED_LOCAL_OPPORTUNITY_SQL = [ - "CREATE TABLE IF NOT EXISTS __[SCHEMA_grants].topportunity (", + "CREATE TABLE IF NOT EXISTS __[SCHEMA_legacy].topportunity (", "opportunity_id BIGSERIAL NOT NULL,", "oppnumber TEXT,", "revision_number BIGINT,", @@ -31,7 +31,7 @@ ] EXPECTED_NONLOCAL_OPPORTUNITY_SQL = [ - "CREATE FOREIGN TABLE IF NOT EXISTS __[SCHEMA_grants].topportunity (", + "CREATE FOREIGN TABLE IF NOT EXISTS __[SCHEMA_legacy].topportunity (", "opportunity_id BIGINT OPTIONS (key 'true') NOT NULL,", "oppnumber TEXT,", "revision_number BIGINT,", @@ -83,12 +83,12 @@ (TEST_TABLE, True, EXPECTED_LOCAL_TEST_SQL), (TEST_TABLE, False, EXPECTED_NONLOCAL_TEST_SQL), ( - src.db.foreign.metadata.tables["grants.topportunity"], + src.db.foreign.metadata.tables["legacy.topportunity"], True, EXPECTED_LOCAL_OPPORTUNITY_SQL, ), ( - src.db.foreign.metadata.tables["grants.topportunity"], + src.db.foreign.metadata.tables["legacy.topportunity"], False, EXPECTED_NONLOCAL_OPPORTUNITY_SQL, ), From 0465bdcb3b5c727a1d8839a8534b24b700ecccc7 Mon Sep 17 00:00:00 2001 From: James Bursa Date: Wed, 24 Apr 2024 12:49:50 -0400 Subject: [PATCH 09/15] Use constant instead of hard-code schema name --- api/src/data_migration/setup_foreign_tables.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/data_migration/setup_foreign_tables.py b/api/src/data_migration/setup_foreign_tables.py index 16ac74919..f9c722539 100644 --- a/api/src/data_migration/setup_foreign_tables.py +++ b/api/src/data_migration/setup_foreign_tables.py @@ -55,12 +55,12 @@ def build_sql(table: sqlalchemy.schema.Table, is_local: bool, schema_name: str) if is_local: compiler = create_table.compile( dialect=sqlalchemy.dialects.postgresql.dialect(), - schema_translate_map={"grants": schema_name}, + schema_translate_map={Schemas.LEGACY.name: schema_name}, ) else: compiler = create_table.compile( dialect=src.db.foreign.dialect.ForeignTableDialect(), - schema_translate_map={"grants": schema_name}, + schema_translate_map={Schemas.LEGACY.name: schema_name}, ) return str(compiler).strip() From 8d4a24c0677e5bc04cf9a34f0b66c0f42dc5855a Mon Sep 17 00:00:00 2001 From: James Bursa Date: Wed, 24 Apr 2024 13:10:21 -0400 Subject: [PATCH 10/15] Remove unwanted code from setup_foreign_tables.py --- api/src/data_migration/setup_foreign_tables.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/api/src/data_migration/setup_foreign_tables.py b/api/src/data_migration/setup_foreign_tables.py index f9c722539..e88cbd12c 100644 --- a/api/src/data_migration/setup_foreign_tables.py +++ b/api/src/data_migration/setup_foreign_tables.py @@ -66,17 +66,7 @@ def build_sql(table: sqlalchemy.schema.Table, is_local: bool, schema_name: str) def _run_create_table_commands(db_session: db.Session, config: ForeignTableConfig) -> None: - sql_file = open("create_foreign_table.sql", "w") - sql_file.write("CREATE SCHEMA IF NOT EXISTS legacy\n") for table in src.db.foreign.metadata.tables.values(): sql = build_sql(table, config.is_local_foreign_table, config.schema_name) logger.info("create table", extra={"table": table.name, "sql": sql}) db_session.execute(sqlalchemy.text(sql)) - sql_file.write(sql.replace("\n", " ").replace("__[SCHEMA_legacy]", "legacy")) - sql_file.write("\n") - - sql_file = open("select_from_foreign_table.sql", "w") - for table in src.db.foreign.metadata.tables.values(): - sql_file.write( - "SELECT * FROM legacy.%s ORDER BY created_date DESC LIMIT 8\n" % (table.name) - ) From f377748df8aba2d1a7792902c16ae755b2132f9c Mon Sep 17 00:00:00 2001 From: James Bursa Date: Wed, 24 Apr 2024 15:58:35 -0400 Subject: [PATCH 11/15] Make primary keys non-null --- api/src/db/foreign/forecast.py | 24 ++++++++++++------------ api/src/db/foreign/synopsis.py | 12 ++++++------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/api/src/db/foreign/forecast.py b/api/src/db/foreign/forecast.py index 46ade0129..44c42d2bd 100644 --- a/api/src/db/foreign/forecast.py +++ b/api/src/db/foreign/forecast.py @@ -97,8 +97,8 @@ class TapplicanttypesForecast(foreignbase.ForeignBase): __tablename__ = "tapplicanttypes_forecast" at_frcst_id: Mapped[int] = mapped_column(primary_key=True) - at_id: Mapped[str | None] = mapped_column(primary_key=True) - opportunity_id: Mapped[int | None] = mapped_column(primary_key=True) + at_id: Mapped[str] = mapped_column(primary_key=True) + opportunity_id: Mapped[int] = mapped_column(primary_key=True) created_date: Mapped[datetime.datetime | None] last_upd_date: Mapped[datetime.datetime | None] creator_id: Mapped[str | None] @@ -109,8 +109,8 @@ class TapplicanttypesForecastHist(foreignbase.ForeignBase): __tablename__ = "tapplicanttypes_forecast_hist" at_frcst_id: Mapped[int] = mapped_column(primary_key=True) - at_id: Mapped[str | None] = mapped_column(primary_key=True) - opportunity_id: Mapped[int | None] = mapped_column(primary_key=True) + at_id: Mapped[str] = mapped_column(primary_key=True) + opportunity_id: Mapped[int] = mapped_column(primary_key=True) revision_number: Mapped[int] = mapped_column(primary_key=True) created_date: Mapped[datetime.datetime | None] last_upd_date: Mapped[datetime.datetime | None] @@ -122,8 +122,8 @@ class TfundactcatForecast(foreignbase.ForeignBase): __tablename__ = "tfundactcat_forecast" fac_frcst_id: Mapped[int] = mapped_column(primary_key=True) - fac_id: Mapped[str | None] = mapped_column(primary_key=True) - opportunity_id: Mapped[int | None] = mapped_column(primary_key=True) + fac_id: Mapped[str] = mapped_column(primary_key=True) + opportunity_id: Mapped[int] = mapped_column(primary_key=True) created_date: Mapped[datetime.datetime | None] last_upd_date: Mapped[datetime.datetime | None] creator_id: Mapped[str | None] @@ -134,8 +134,8 @@ class TfundactcatForecastHist(foreignbase.ForeignBase): __tablename__ = "tfundactcat_forecast_hist" fac_frcst_id: Mapped[int] = mapped_column(primary_key=True) - fac_id: Mapped[str | None] = mapped_column(primary_key=True) - opportunity_id: Mapped[int | None] = mapped_column(primary_key=True) + fac_id: Mapped[str] = mapped_column(primary_key=True) + opportunity_id: Mapped[int] = mapped_column(primary_key=True) revision_number: Mapped[int] = mapped_column(primary_key=True) created_date: Mapped[datetime.datetime | None] last_upd_date: Mapped[datetime.datetime | None] @@ -147,8 +147,8 @@ class TfundinstrForecast(foreignbase.ForeignBase): __tablename__ = "tfundinstr_forecast" fi_frcst_id: Mapped[int] = mapped_column(primary_key=True) - fi_id: Mapped[str | None] = mapped_column(primary_key=True) - opportunity_id: Mapped[int | None] = mapped_column(primary_key=True) + fi_id: Mapped[str] = mapped_column(primary_key=True) + opportunity_id: Mapped[int] = mapped_column(primary_key=True) created_date: Mapped[datetime.datetime | None] last_upd_date: Mapped[datetime.datetime | None] creator_id: Mapped[str | None] @@ -159,8 +159,8 @@ class TfundinstrForecastHist(foreignbase.ForeignBase): __tablename__ = "tfundinstr_forecast_hist" fi_frcst_id: Mapped[int] = mapped_column(primary_key=True) - fi_id: Mapped[str | None] = mapped_column(primary_key=True) - opportunity_id: Mapped[int | None] = mapped_column(primary_key=True) + fi_id: Mapped[str] = mapped_column(primary_key=True) + opportunity_id: Mapped[int] = mapped_column(primary_key=True) revision_number: Mapped[int] = mapped_column(primary_key=True) created_date: Mapped[datetime.datetime | None] last_upd_date: Mapped[datetime.datetime | None] diff --git a/api/src/db/foreign/synopsis.py b/api/src/db/foreign/synopsis.py index 7c39b3a72..35230b1a9 100644 --- a/api/src/db/foreign/synopsis.py +++ b/api/src/db/foreign/synopsis.py @@ -99,8 +99,8 @@ class TapplicanttypesSynopsis(foreignbase.ForeignBase): __tablename__ = "tapplicanttypes_synopsis" at_syn_id: Mapped[int] = mapped_column(primary_key=True) - at_id: Mapped[str | None] = mapped_column(primary_key=True) - opportunity_id: Mapped[int | None] = mapped_column(primary_key=True) + at_id: Mapped[str] = mapped_column(primary_key=True) + opportunity_id: Mapped[int] = mapped_column(primary_key=True) created_date: Mapped[datetime.datetime | None] last_upd_date: Mapped[datetime.datetime | None] creator_id: Mapped[str | None] @@ -124,8 +124,8 @@ class TfundactcatSynopsis(foreignbase.ForeignBase): __tablename__ = "tfundactcat_synopsis" fac_syn_id: Mapped[int] = mapped_column(primary_key=True) - fac_id: Mapped[str | None] = mapped_column(primary_key=True) - opportunity_id: Mapped[int | None] = mapped_column(primary_key=True) + fac_id: Mapped[str] = mapped_column(primary_key=True) + opportunity_id: Mapped[int] = mapped_column(primary_key=True) created_date: Mapped[datetime.datetime | None] last_upd_date: Mapped[datetime.datetime | None] creator_id: Mapped[str | None] @@ -149,8 +149,8 @@ class TfundinstrSynopsis(foreignbase.ForeignBase): __tablename__ = "tfundinstr_synopsis" fi_syn_id: Mapped[int] = mapped_column(primary_key=True) - fi_id: Mapped[str | None] = mapped_column(primary_key=True) - opportunity_id: Mapped[int | None] = mapped_column(primary_key=True) + fi_id: Mapped[str] = mapped_column(primary_key=True) + opportunity_id: Mapped[int] = mapped_column(primary_key=True) created_date: Mapped[datetime.datetime | None] last_upd_date: Mapped[datetime.datetime | None] creator_id: Mapped[str | None] From 15e92eca332e1f212a5b3272fcdc4c8d9ccda760 Mon Sep 17 00:00:00 2001 From: James Bursa Date: Wed, 24 Apr 2024 17:09:34 -0400 Subject: [PATCH 12/15] Add type annotations --- api/src/db/foreign/dialect.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/api/src/db/foreign/dialect.py b/api/src/db/foreign/dialect.py index ea388e480..073d893cf 100644 --- a/api/src/db/foreign/dialect.py +++ b/api/src/db/foreign/dialect.py @@ -3,28 +3,32 @@ # import re +from typing import Any import sqlalchemy +import sqlalchemy.dialects.postgresql class ForeignTableDDLCompiler(sqlalchemy.sql.compiler.DDLCompiler): """SQLAlchemy compiler for creating foreign tables.""" - def create_table_constraints(self, _table, _include_foreign_key_constraints=None, **kw): + def create_table_constraints( + self, _table: Any, _include_foreign_key_constraints: Any = None, **kw: Any + ) -> str: return "" # Don't generate any constraints. - def visit_create_table(self, create, **kw): + def visit_create_table(self, create: Any, **kw: Any) -> str: table = create.element table._prefixes = ("FOREIGN",) # Add "FOREIGN" before "TABLE". sql = super().visit_create_table(create, **kw) table._prefixes = () return sql - def post_create_table(self, table): + def post_create_table(self, table: Any) -> str: # Add foreign options at the end. return f" SERVER grants OPTIONS (schema 'EGRANTSADMIN', table '{table.name.upper()}')" - def visit_create_column(self, create, first_pk=False, **kw): + def visit_create_column(self, create: Any, first_pk: bool = False, **kw: Any) -> str: column = create.element sql = super().visit_create_column(create, first_pk, **kw) if sql and column.primary_key: From 8027c224f6a56e91ecb706cd0271ec139b41db46 Mon Sep 17 00:00:00 2001 From: James Bursa Date: Thu, 25 Apr 2024 12:27:36 -0400 Subject: [PATCH 13/15] Exclude dialect.py from type annotations --- api/src/db/foreign/dialect.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/api/src/db/foreign/dialect.py b/api/src/db/foreign/dialect.py index 073d893cf..8603a04c3 100644 --- a/api/src/db/foreign/dialect.py +++ b/api/src/db/foreign/dialect.py @@ -1,34 +1,31 @@ # # Support for generating SQL for "CREATE FOREIGN TABLE". # +# mypy: ignore-errors import re -from typing import Any import sqlalchemy -import sqlalchemy.dialects.postgresql class ForeignTableDDLCompiler(sqlalchemy.sql.compiler.DDLCompiler): """SQLAlchemy compiler for creating foreign tables.""" - def create_table_constraints( - self, _table: Any, _include_foreign_key_constraints: Any = None, **kw: Any - ) -> str: + def create_table_constraints(self, _table, _include_foreign_key_constraints=None, **kw): return "" # Don't generate any constraints. - def visit_create_table(self, create: Any, **kw: Any) -> str: + def visit_create_table(self, create, **kw): table = create.element table._prefixes = ("FOREIGN",) # Add "FOREIGN" before "TABLE". sql = super().visit_create_table(create, **kw) table._prefixes = () return sql - def post_create_table(self, table: Any) -> str: + def post_create_table(self, table): # Add foreign options at the end. return f" SERVER grants OPTIONS (schema 'EGRANTSADMIN', table '{table.name.upper()}')" - def visit_create_column(self, create: Any, first_pk: bool = False, **kw: Any) -> str: + def visit_create_column(self, create, first_pk=False, **kw): column = create.element sql = super().visit_create_column(create, first_pk, **kw) if sql and column.primary_key: From 3cba153757fd813c2818898d8cbb436c77d16199 Mon Sep 17 00:00:00 2001 From: James Bursa Date: Thu, 25 Apr 2024 12:28:08 -0400 Subject: [PATCH 14/15] Update search path for `make login-db` --- api/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/Makefile b/api/Makefile index 1858dc561..79a913a04 100644 --- a/api/Makefile +++ b/api/Makefile @@ -256,7 +256,7 @@ setup-foreign-tables: login: start ## Start shell in running container docker exec -it $(APP_NAME) bash -DB_URI := postgresql://$(DB_USER)@$(DB_HOST):$(DB_PORT)/$(DB_NAME)?options=-csearch_path%3dapi +DB_URI := postgresql://$(DB_USER)@$(DB_HOST):$(DB_PORT)/$(DB_NAME)?options=-csearch_path%3dapi,legacy,staging login-db: ## Start psql with project environment variables PGPASSWORD=$$DB_PASSWORD psql $(DB_URI) From 7e2efad053b5324c40f06bc7241723d533488fba Mon Sep 17 00:00:00 2001 From: James Bursa Date: Thu, 25 Apr 2024 13:07:31 -0400 Subject: [PATCH 15/15] Fix test failure in test_via_cli --- .../test_set_current_opportunities_task.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/api/tests/src/task/opportunities/test_set_current_opportunities_task.py b/api/tests/src/task/opportunities/test_set_current_opportunities_task.py index 33968de1e..38160fa68 100644 --- a/api/tests/src/task/opportunities/test_set_current_opportunities_task.py +++ b/api/tests/src/task/opportunities/test_set_current_opportunities_task.py @@ -6,6 +6,7 @@ from src.constants.lookup_constants import OpportunityStatus from src.db.models.opportunity_models import CurrentOpportunitySummary, OpportunitySummary from src.task.opportunities.set_current_opportunities_task import SetCurrentOpportunitiesTask +from src.util.datetime_util import get_now_us_eastern_date from tests.conftest import BaseTestClass from tests.src.db.models.factories import ( CurrentOpportunitySummaryFactory, @@ -525,12 +526,14 @@ def test_via_cli(cli_runner, db_session, enable_factory_create): # note that the script will always use todays date as the current date, so we # need to generate the scenario from that instead + today = get_now_us_eastern_date() + # A basic posted scenario container1 = OpportunityContainer().with_summary( is_forecast=False, - post_date=CURRENT_DATE - timedelta(days=10), - close_date=CURRENT_DATE + timedelta(days=30), - archive_date=CURRENT_DATE + timedelta(days=60), + post_date=today - timedelta(days=10), + close_date=today + timedelta(days=30), + archive_date=today + timedelta(days=60), is_expected_current=True, ) @@ -539,21 +542,21 @@ def test_via_cli(cli_runner, db_session, enable_factory_create): OpportunityContainer() .with_summary( is_forecast=True, - post_date=CURRENT_DATE - timedelta(days=5), - archive_date=CURRENT_DATE + timedelta(days=60), + post_date=today - timedelta(days=5), + archive_date=today + timedelta(days=60), is_already_current=True, revision_number=2, ) .with_summary( is_forecast=True, - post_date=CURRENT_DATE - timedelta(days=5), - archive_date=CURRENT_DATE + timedelta(days=60), + post_date=today - timedelta(days=5), + archive_date=today + timedelta(days=60), revision_number=1, ) .with_summary( is_forecast=True, - post_date=CURRENT_DATE - timedelta(days=5), - archive_date=CURRENT_DATE + timedelta(days=120), + post_date=today - timedelta(days=5), + archive_date=today + timedelta(days=120), is_expected_current=True, revision_number=3, )