diff --git a/api/bin/create_erds.py b/api/bin/create_erds.py index 951b5a676..11e12b1b7 100755 --- a/api/bin/create_erds.py +++ b/api/bin/create_erds.py @@ -8,6 +8,9 @@ import pydot import sadisplay +import src.db.models.staging.forecast as staging_forecast_models +import src.db.models.staging.opportunity as staging_opportunity_models +import src.db.models.staging.synopsis as staging_synopsis_models import src.logging from src.db.models import opportunity_models from src.db.models.transfer import topportunity_models @@ -20,10 +23,17 @@ ERD_FOLDER = pathlib.Path(__file__).parent.resolve() # If we want to generate separate files for more specific groups, we can set that up here -ALL_MODULES = (opportunity_models, topportunity_models) - +API_MODULES = (opportunity_models,) +STAGING_TABLE_MODULES = ( + staging_opportunity_models, + staging_forecast_models, + staging_synopsis_models, +) TRANSFER_TABLE_MODULES = (topportunity_models,) +# Any modules you add above, merge together for the full schema to be generated +ALL_MODULES = API_MODULES + STAGING_TABLE_MODULES + TRANSFER_TABLE_MODULES + def create_erds(modules: Any, file_name: str) -> None: logger.info("Generating ERD diagrams for %s", file_name) @@ -60,5 +70,6 @@ def main() -> None: logger.info("Generating ERD diagrams") create_erds(ALL_MODULES, "full-schema") - + create_erds(API_MODULES, "api-schema") + create_erds(STAGING_TABLE_MODULES, "staging-schema") create_erds(TRANSFER_TABLE_MODULES, "transfer-schema") diff --git a/api/src/db/foreign/forecast.py b/api/src/db/foreign/forecast.py index 44c42d2bd..aaff5bc4f 100644 --- a/api/src/db/foreign/forecast.py +++ b/api/src/db/foreign/forecast.py @@ -5,164 +5,40 @@ # match by oracle_fdw, but we are matching them for maintainability. # -import datetime - -from sqlalchemy.orm import Mapped, mapped_column +from src.db.legacy_mixin import forecast_mixin from . import foreignbase -class Tforecast(foreignbase.ForeignBase): +class Tforecast(foreignbase.ForeignBase, forecast_mixin.TforecastMixin): __tablename__ = "tforecast" - opportunity_id: Mapped[int] = mapped_column(primary_key=True) - version_nbr: Mapped[int] - posting_date: Mapped[datetime.datetime | None] - archive_date: Mapped[datetime.datetime | None] - forecast_desc: Mapped[str | None] - oth_cat_fa_desc: Mapped[str | None] - cost_sharing: Mapped[str | None] - number_of_awards: Mapped[str | None] - est_funding: Mapped[str | None] - award_ceiling: Mapped[str | None] - award_floor: Mapped[str | None] - fd_link_url: Mapped[str | None] - fd_link_desc: Mapped[str | None] - ac_name: Mapped[str | None] - ac_phone: Mapped[str | None] - ac_email_addr: Mapped[str | None] - ac_email_desc: Mapped[str | None] - agency_code: Mapped[str | None] - sendmail: Mapped[str | None] - applicant_elig_desc: Mapped[str | None] - est_synopsis_posting_date: Mapped[datetime.datetime | None] - est_appl_response_date: Mapped[datetime.datetime | None] - est_appl_response_date_desc: Mapped[str | None] - est_award_date: Mapped[datetime.datetime | None] - est_project_start_date: Mapped[datetime.datetime | None] - fiscal_year: Mapped[int | None] - modification_comments: Mapped[str | None] - create_ts: Mapped[datetime.datetime] - created_date: Mapped[datetime.datetime | None] - last_upd_date: Mapped[datetime.datetime | None] - creator_id: Mapped[str | None] - last_upd_id: Mapped[str | None] - publisheruid: Mapped[str | None] - publisher_profile_id: Mapped[int | None] - - -class TforecastHist(foreignbase.ForeignBase): + +class TforecastHist(foreignbase.ForeignBase, forecast_mixin.TforecastHistMixin): __tablename__ = "tforecast_hist" - opportunity_id: Mapped[int] = mapped_column(primary_key=True) - revision_number: Mapped[int] = mapped_column(primary_key=True) - version_nbr: Mapped[int] - posting_date: Mapped[datetime.datetime | None] - archive_date: Mapped[datetime.datetime | None] - forecast_desc: Mapped[str | None] - oth_cat_fa_desc: Mapped[str | None] - cost_sharing: Mapped[str | None] - number_of_awards: Mapped[str | None] - est_funding: Mapped[str | None] - award_ceiling: Mapped[str | None] - award_floor: Mapped[str | None] - fd_link_url: Mapped[str | None] - fd_link_desc: Mapped[str | None] - ac_name: Mapped[str | None] - ac_phone: Mapped[str | None] - ac_email_addr: Mapped[str | None] - ac_email_desc: Mapped[str | None] - agency_code: Mapped[str | None] - sendmail: Mapped[str | None] - applicant_elig_desc: Mapped[str | None] - est_synopsis_posting_date: Mapped[datetime.datetime | None] - est_appl_response_date: Mapped[datetime.datetime | None] - est_appl_response_date_desc: Mapped[str | None] - est_award_date: Mapped[datetime.datetime | None] - est_project_start_date: Mapped[datetime.datetime | None] - fiscal_year: Mapped[int | None] - modification_comments: Mapped[str | None] - action_type: Mapped[str | None] - action_date: Mapped[datetime.datetime | None] - create_ts: Mapped[datetime.datetime] - created_date: Mapped[datetime.datetime | None] - last_upd_date: Mapped[datetime.datetime | None] - creator_id: Mapped[str | None] - last_upd_id: Mapped[str | None] - publisheruid: Mapped[str | None] - publisher_profile_id: Mapped[int | None] - - -class TapplicanttypesForecast(foreignbase.ForeignBase): - __tablename__ = "tapplicanttypes_forecast" - at_frcst_id: Mapped[int] = mapped_column(primary_key=True) - at_id: Mapped[str] = mapped_column(primary_key=True) - opportunity_id: Mapped[int] = mapped_column(primary_key=True) - created_date: Mapped[datetime.datetime | None] - last_upd_date: Mapped[datetime.datetime | None] - creator_id: Mapped[str | None] - last_upd_id: Mapped[str | None] +class TapplicanttypesForecast(foreignbase.ForeignBase, forecast_mixin.TapplicanttypesForecastMixin): + __tablename__ = "tapplicanttypes_forecast" -class TapplicanttypesForecastHist(foreignbase.ForeignBase): +class TapplicanttypesForecastHist( + foreignbase.ForeignBase, forecast_mixin.TapplicanttypesForecastHistMixin +): __tablename__ = "tapplicanttypes_forecast_hist" - at_frcst_id: Mapped[int] = mapped_column(primary_key=True) - at_id: Mapped[str] = mapped_column(primary_key=True) - opportunity_id: Mapped[int] = mapped_column(primary_key=True) - revision_number: Mapped[int] = mapped_column(primary_key=True) - created_date: Mapped[datetime.datetime | None] - last_upd_date: Mapped[datetime.datetime | None] - creator_id: Mapped[str | None] - last_upd_id: Mapped[str | None] - -class TfundactcatForecast(foreignbase.ForeignBase): +class TfundactcatForecast(foreignbase.ForeignBase, forecast_mixin.TfundactcatForecastMixin): __tablename__ = "tfundactcat_forecast" - fac_frcst_id: Mapped[int] = mapped_column(primary_key=True) - fac_id: Mapped[str] = mapped_column(primary_key=True) - opportunity_id: Mapped[int] = mapped_column(primary_key=True) - created_date: Mapped[datetime.datetime | None] - last_upd_date: Mapped[datetime.datetime | None] - creator_id: Mapped[str | None] - last_upd_id: Mapped[str | None] - -class TfundactcatForecastHist(foreignbase.ForeignBase): +class TfundactcatForecastHist(foreignbase.ForeignBase, forecast_mixin.TfundactcatForecastHistMixin): __tablename__ = "tfundactcat_forecast_hist" - fac_frcst_id: Mapped[int] = mapped_column(primary_key=True) - fac_id: Mapped[str] = mapped_column(primary_key=True) - opportunity_id: Mapped[int] = mapped_column(primary_key=True) - revision_number: Mapped[int] = mapped_column(primary_key=True) - created_date: Mapped[datetime.datetime | None] - last_upd_date: Mapped[datetime.datetime | None] - creator_id: Mapped[str | None] - last_upd_id: Mapped[str | None] - -class TfundinstrForecast(foreignbase.ForeignBase): +class TfundinstrForecast(foreignbase.ForeignBase, forecast_mixin.TfundinstrForecastMixin): __tablename__ = "tfundinstr_forecast" - fi_frcst_id: Mapped[int] = mapped_column(primary_key=True) - fi_id: Mapped[str] = mapped_column(primary_key=True) - opportunity_id: Mapped[int] = mapped_column(primary_key=True) - created_date: Mapped[datetime.datetime | None] - last_upd_date: Mapped[datetime.datetime | None] - creator_id: Mapped[str | None] - last_upd_id: Mapped[str | None] - -class TfundinstrForecastHist(foreignbase.ForeignBase): +class TfundinstrForecastHist(foreignbase.ForeignBase, forecast_mixin.TfundinstrForecastHistMixin): __tablename__ = "tfundinstr_forecast_hist" - - fi_frcst_id: Mapped[int] = mapped_column(primary_key=True) - fi_id: Mapped[str] = mapped_column(primary_key=True) - opportunity_id: Mapped[int] = mapped_column(primary_key=True) - revision_number: Mapped[int] = mapped_column(primary_key=True) - created_date: Mapped[datetime.datetime | None] - last_upd_date: Mapped[datetime.datetime | None] - creator_id: Mapped[str | None] - last_upd_id: Mapped[str | None] diff --git a/api/src/db/foreign/opportunity.py b/api/src/db/foreign/opportunity.py index 37a7fa593..70992e674 100644 --- a/api/src/db/foreign/opportunity.py +++ b/api/src/db/foreign/opportunity.py @@ -5,47 +5,14 @@ # match by oracle_fdw, but we are matching them for maintainability. # -import datetime - -from sqlalchemy.orm import Mapped, mapped_column +from src.db.legacy_mixin import opportunity_mixin from . import foreignbase -class Topportunity(foreignbase.ForeignBase): +class Topportunity(foreignbase.ForeignBase, opportunity_mixin.TopportunityMixin): __tablename__ = "topportunity" - opportunity_id: Mapped[int] = mapped_column(primary_key=True) - oppnumber: Mapped[str | None] - revision_number: Mapped[int | None] - opptitle: Mapped[str | None] - owningagency: Mapped[str | None] - publisheruid: Mapped[str | None] - listed: Mapped[str | None] - oppcategory: Mapped[str | None] - initial_opportunity_id: Mapped[int | None] - modified_comments: Mapped[str | None] - created_date: Mapped[datetime.datetime] - last_upd_date: Mapped[datetime.datetime] - creator_id: Mapped[str | None] - last_upd_id: Mapped[str | None] - flag_2006: Mapped[str | None] - category_explanation: Mapped[str | None] - publisher_profile_id: Mapped[int | None] - is_draft: Mapped[str | None] - -class TopportunityCfda(foreignbase.ForeignBase): +class TopportunityCfda(foreignbase.ForeignBase, opportunity_mixin.TopportunityCfdaMixin): __tablename__ = "topportunity_cfda" - - opp_cfda_id: Mapped[int] = mapped_column(primary_key=True) - opportunity_id: Mapped[int] - cfdanumber: Mapped[str | None] - programtitle: Mapped[str | None] - origtoppid: Mapped[int | None] - oppidcfdanum: Mapped[str | None] - origoppnum: Mapped[str | None] - created_date: Mapped[datetime.datetime | None] - last_upd_date: Mapped[datetime.datetime | None] - creator_id: Mapped[str | None] - last_upd_id: Mapped[str | None] diff --git a/api/src/db/foreign/synopsis.py b/api/src/db/foreign/synopsis.py index 35230b1a9..b016f568d 100644 --- a/api/src/db/foreign/synopsis.py +++ b/api/src/db/foreign/synopsis.py @@ -5,166 +5,40 @@ # match by oracle_fdw, but we are matching them for maintainability. # -import datetime - -from sqlalchemy.orm import Mapped, mapped_column +from src.db.legacy_mixin import synopsis_mixin from . import foreignbase -class Tsynopsis(foreignbase.ForeignBase): +class Tsynopsis(foreignbase.ForeignBase, synopsis_mixin.TsynopsisMixin): __tablename__ = "tsynopsis" - opportunity_id: Mapped[int] = mapped_column(primary_key=True) - posting_date: Mapped[datetime.datetime | None] - response_date: Mapped[datetime.datetime | None] - archive_date: Mapped[datetime.datetime | None] - unarchive_date: Mapped[datetime.datetime | None] - syn_desc: Mapped[str | None] - oth_cat_fa_desc: Mapped[str | None] - agency_addr_desc: Mapped[str | None] - cost_sharing: Mapped[str | None] - number_of_awards: Mapped[str | None] - est_funding: Mapped[str | None] - award_ceiling: Mapped[str | None] - award_floor: Mapped[str | None] - fd_link_url: Mapped[str | None] - fd_link_desc: Mapped[str | None] - agency_contact_desc: Mapped[str | None] - ac_email_addr: Mapped[str | None] - ac_email_desc: Mapped[str | None] - agency_name: Mapped[str | None] - agency_phone: Mapped[str | None] - a_sa_code: Mapped[str | None] - ac_phone_number: Mapped[str | None] - ac_name: Mapped[str | None] - create_ts: Mapped[datetime.datetime] - created_date: Mapped[datetime.datetime | None] - last_upd_date: Mapped[datetime.datetime | None] - creator_id: Mapped[str | None] - last_upd_id: Mapped[str | None] - sendmail: Mapped[str | None] - response_date_desc: Mapped[str | None] - applicant_elig_desc: Mapped[str | None] - version_nbr: Mapped[int | None] - modification_comments: Mapped[str | None] - publisheruid: Mapped[str | None] - publisher_profile_id: Mapped[int | None] - - -class TsynopsisHist(foreignbase.ForeignBase): + +class TsynopsisHist(foreignbase.ForeignBase, synopsis_mixin.TsynopsisHistMixin): __tablename__ = "tsynopsis_hist" - opportunity_id: Mapped[int] = mapped_column(primary_key=True) - revision_number: Mapped[int] = mapped_column(primary_key=True) - posting_date: Mapped[datetime.datetime | None] - response_date: Mapped[datetime.datetime | None] - archive_date: Mapped[datetime.datetime | None] - unarchive_date: Mapped[datetime.datetime | None] - syn_desc: Mapped[str | None] - oth_cat_fa_desc: Mapped[str | None] - agency_addr_desc: Mapped[str | None] - cost_sharing: Mapped[str | None] - number_of_awards: Mapped[str | None] - est_funding: Mapped[str | None] - award_ceiling: Mapped[str | None] - award_floor: Mapped[str | None] - fd_link_url: Mapped[str | None] - fd_link_desc: Mapped[str | None] - agency_contact_desc: Mapped[str | None] - ac_email_addr: Mapped[str | None] - ac_email_desc: Mapped[str | None] - agency_name: Mapped[str | None] - agency_phone: Mapped[str | None] - a_sa_code: Mapped[str | None] - ac_phone_number: Mapped[str | None] - ac_name: Mapped[str | None] - create_ts: Mapped[datetime.datetime] - created_date: Mapped[datetime.datetime | None] - last_upd_date: Mapped[datetime.datetime | None] - creator_id: Mapped[str | None] - last_upd_id: Mapped[str | None] - sendmail: Mapped[str | None] - response_date_desc: Mapped[str | None] - applicant_elig_desc: Mapped[str | None] - action_type: Mapped[str | None] - action_date: Mapped[datetime.datetime | None] - version_nbr: Mapped[int] - modification_comments: Mapped[str | None] - publisheruid: Mapped[str | None] - publisher_profile_id: Mapped[int | None] - - -class TapplicanttypesSynopsis(foreignbase.ForeignBase): - __tablename__ = "tapplicanttypes_synopsis" - at_syn_id: Mapped[int] = mapped_column(primary_key=True) - at_id: Mapped[str] = mapped_column(primary_key=True) - opportunity_id: Mapped[int] = mapped_column(primary_key=True) - created_date: Mapped[datetime.datetime | None] - last_upd_date: Mapped[datetime.datetime | None] - creator_id: Mapped[str | None] - last_upd_id: Mapped[str | None] +class TapplicanttypesSynopsis(foreignbase.ForeignBase, synopsis_mixin.TapplicanttypesSynopsisMixin): + __tablename__ = "tapplicanttypes_synopsis" -class TapplicanttypesSynopsisHist(foreignbase.ForeignBase): +class TapplicanttypesSynopsisHist( + foreignbase.ForeignBase, synopsis_mixin.TapplicanttypesSynopsisHistMixin +): __tablename__ = "tapplicanttypes_synopsis_hist" - at_syn_id: Mapped[int] = mapped_column(primary_key=True) - at_id: Mapped[str] = mapped_column(primary_key=True) - opportunity_id: Mapped[int] = mapped_column(primary_key=True) - revision_number: Mapped[int] = mapped_column(primary_key=True) - created_date: Mapped[datetime.datetime | None] - last_upd_date: Mapped[datetime.datetime | None] - creator_id: Mapped[str | None] - last_upd_id: Mapped[str | None] - -class TfundactcatSynopsis(foreignbase.ForeignBase): +class TfundactcatSynopsis(foreignbase.ForeignBase, synopsis_mixin.TfundactcatSynopsisMixin): __tablename__ = "tfundactcat_synopsis" - fac_syn_id: Mapped[int] = mapped_column(primary_key=True) - fac_id: Mapped[str] = mapped_column(primary_key=True) - opportunity_id: Mapped[int] = mapped_column(primary_key=True) - created_date: Mapped[datetime.datetime | None] - last_upd_date: Mapped[datetime.datetime | None] - creator_id: Mapped[str | None] - last_upd_id: Mapped[str | None] - -class TfundactcatSynopsisHist(foreignbase.ForeignBase): +class TfundactcatSynopsisHist(foreignbase.ForeignBase, synopsis_mixin.TfundactcatSynopsisHistMixin): __tablename__ = "tfundactcat_synopsis_hist" - fac_syn_id: Mapped[int] = mapped_column(primary_key=True) - fac_id: Mapped[str] = mapped_column(primary_key=True) - opportunity_id: Mapped[int] = mapped_column(primary_key=True) - revision_number: Mapped[int] = mapped_column(primary_key=True) - created_date: Mapped[datetime.datetime | None] - last_upd_date: Mapped[datetime.datetime | None] - creator_id: Mapped[str | None] - last_upd_id: Mapped[str | None] - -class TfundinstrSynopsis(foreignbase.ForeignBase): +class TfundinstrSynopsis(foreignbase.ForeignBase, synopsis_mixin.TfundinstrSynopsisMixin): __tablename__ = "tfundinstr_synopsis" - fi_syn_id: Mapped[int] = mapped_column(primary_key=True) - fi_id: Mapped[str] = mapped_column(primary_key=True) - opportunity_id: Mapped[int] = mapped_column(primary_key=True) - created_date: Mapped[datetime.datetime | None] - last_upd_date: Mapped[datetime.datetime | None] - creator_id: Mapped[str | None] - last_upd_id: Mapped[str | None] - -class TfundinstrSynopsisHist(foreignbase.ForeignBase): +class TfundinstrSynopsisHist(foreignbase.ForeignBase, synopsis_mixin.TfundinstrSynopsisHistMixin): __tablename__ = "tfundinstr_synopsis_hist" - - fi_syn_id: Mapped[int] = mapped_column(primary_key=True) - opportunity_id: Mapped[int] = mapped_column(primary_key=True) - fi_id: Mapped[str] = mapped_column(primary_key=True) - revision_number: Mapped[int] = mapped_column(primary_key=True) - created_date: Mapped[datetime.datetime | None] - last_upd_date: Mapped[datetime.datetime | None] - creator_id: Mapped[str | None] - last_upd_id: Mapped[str | None] diff --git a/api/src/db/legacy_mixin/__init__.py b/api/src/db/legacy_mixin/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/api/src/db/legacy_mixin/forecast_mixin.py b/api/src/db/legacy_mixin/forecast_mixin.py new file mode 100644 index 000000000..728fba6e6 --- /dev/null +++ b/api/src/db/legacy_mixin/forecast_mixin.py @@ -0,0 +1,158 @@ +# +# SQLAlchemy models for foreign tables. +# +# The order of the columns must match the remote Oracle database. The names are not required to +# match by oracle_fdw, but we are matching them for maintainability. +# + +import datetime + +from sqlalchemy.orm import Mapped, declarative_mixin, mapped_column + + +@declarative_mixin +class TforecastMixin: + opportunity_id: Mapped[int] = mapped_column(primary_key=True) + version_nbr: Mapped[int] + posting_date: Mapped[datetime.datetime | None] + archive_date: Mapped[datetime.datetime | None] + forecast_desc: Mapped[str | None] + oth_cat_fa_desc: Mapped[str | None] + cost_sharing: Mapped[str | None] + number_of_awards: Mapped[str | None] + est_funding: Mapped[str | None] + award_ceiling: Mapped[str | None] + award_floor: Mapped[str | None] + fd_link_url: Mapped[str | None] + fd_link_desc: Mapped[str | None] + ac_name: Mapped[str | None] + ac_phone: Mapped[str | None] + ac_email_addr: Mapped[str | None] + ac_email_desc: Mapped[str | None] + agency_code: Mapped[str | None] + sendmail: Mapped[str | None] + applicant_elig_desc: Mapped[str | None] + est_synopsis_posting_date: Mapped[datetime.datetime | None] + est_appl_response_date: Mapped[datetime.datetime | None] + est_appl_response_date_desc: Mapped[str | None] + est_award_date: Mapped[datetime.datetime | None] + est_project_start_date: Mapped[datetime.datetime | None] + fiscal_year: Mapped[int | None] + modification_comments: Mapped[str | None] + create_ts: Mapped[datetime.datetime] + created_date: Mapped[datetime.datetime | None] + last_upd_date: Mapped[datetime.datetime | None] + creator_id: Mapped[str | None] + last_upd_id: Mapped[str | None] + publisheruid: Mapped[str | None] + publisher_profile_id: Mapped[int | None] + + +@declarative_mixin +class TforecastHistMixin: + opportunity_id: Mapped[int] = mapped_column(primary_key=True) + revision_number: Mapped[int] = mapped_column(primary_key=True) + version_nbr: Mapped[int] + posting_date: Mapped[datetime.datetime | None] + archive_date: Mapped[datetime.datetime | None] + forecast_desc: Mapped[str | None] + oth_cat_fa_desc: Mapped[str | None] + cost_sharing: Mapped[str | None] + number_of_awards: Mapped[str | None] + est_funding: Mapped[str | None] + award_ceiling: Mapped[str | None] + award_floor: Mapped[str | None] + fd_link_url: Mapped[str | None] + fd_link_desc: Mapped[str | None] + ac_name: Mapped[str | None] + ac_phone: Mapped[str | None] + ac_email_addr: Mapped[str | None] + ac_email_desc: Mapped[str | None] + agency_code: Mapped[str | None] + sendmail: Mapped[str | None] + applicant_elig_desc: Mapped[str | None] + est_synopsis_posting_date: Mapped[datetime.datetime | None] + est_appl_response_date: Mapped[datetime.datetime | None] + est_appl_response_date_desc: Mapped[str | None] + est_award_date: Mapped[datetime.datetime | None] + est_project_start_date: Mapped[datetime.datetime | None] + fiscal_year: Mapped[int | None] + modification_comments: Mapped[str | None] + action_type: Mapped[str | None] + action_date: Mapped[datetime.datetime | None] + create_ts: Mapped[datetime.datetime] + created_date: Mapped[datetime.datetime | None] + last_upd_date: Mapped[datetime.datetime | None] + creator_id: Mapped[str | None] + last_upd_id: Mapped[str | None] + publisheruid: Mapped[str | None] + publisher_profile_id: Mapped[int | None] + + +@declarative_mixin +class TapplicanttypesForecastMixin: + at_frcst_id: Mapped[int] = mapped_column(primary_key=True) + at_id: Mapped[str] = mapped_column(primary_key=True) + opportunity_id: Mapped[int] = mapped_column(primary_key=True) + created_date: Mapped[datetime.datetime | None] + last_upd_date: Mapped[datetime.datetime | None] + creator_id: Mapped[str | None] + last_upd_id: Mapped[str | None] + + +@declarative_mixin +class TapplicanttypesForecastHistMixin: + at_frcst_id: Mapped[int] = mapped_column(primary_key=True) + at_id: Mapped[str] = mapped_column(primary_key=True) + opportunity_id: Mapped[int] = mapped_column(primary_key=True) + revision_number: Mapped[int] = mapped_column(primary_key=True) + created_date: Mapped[datetime.datetime | None] + last_upd_date: Mapped[datetime.datetime | None] + creator_id: Mapped[str | None] + last_upd_id: Mapped[str | None] + + +@declarative_mixin +class TfundactcatForecastMixin: + fac_frcst_id: Mapped[int] = mapped_column(primary_key=True) + fac_id: Mapped[str] = mapped_column(primary_key=True) + opportunity_id: Mapped[int] = mapped_column(primary_key=True) + created_date: Mapped[datetime.datetime | None] + last_upd_date: Mapped[datetime.datetime | None] + creator_id: Mapped[str | None] + last_upd_id: Mapped[str | None] + + +@declarative_mixin +class TfundactcatForecastHistMixin: + fac_frcst_id: Mapped[int] = mapped_column(primary_key=True) + fac_id: Mapped[str] = mapped_column(primary_key=True) + opportunity_id: Mapped[int] = mapped_column(primary_key=True) + revision_number: Mapped[int] = mapped_column(primary_key=True) + created_date: Mapped[datetime.datetime | None] + last_upd_date: Mapped[datetime.datetime | None] + creator_id: Mapped[str | None] + last_upd_id: Mapped[str | None] + + +@declarative_mixin +class TfundinstrForecastMixin: + fi_frcst_id: Mapped[int] = mapped_column(primary_key=True) + fi_id: Mapped[str] = mapped_column(primary_key=True) + opportunity_id: Mapped[int] = mapped_column(primary_key=True) + created_date: Mapped[datetime.datetime | None] + last_upd_date: Mapped[datetime.datetime | None] + creator_id: Mapped[str | None] + last_upd_id: Mapped[str | None] + + +@declarative_mixin +class TfundinstrForecastHistMixin: + fi_frcst_id: Mapped[int] = mapped_column(primary_key=True) + fi_id: Mapped[str] = mapped_column(primary_key=True) + opportunity_id: Mapped[int] = mapped_column(primary_key=True) + revision_number: Mapped[int] = mapped_column(primary_key=True) + created_date: Mapped[datetime.datetime | None] + last_upd_date: Mapped[datetime.datetime | None] + creator_id: Mapped[str | None] + last_upd_id: Mapped[str | None] diff --git a/api/src/db/legacy_mixin/opportunity_mixin.py b/api/src/db/legacy_mixin/opportunity_mixin.py new file mode 100644 index 000000000..3d55f4ece --- /dev/null +++ b/api/src/db/legacy_mixin/opportunity_mixin.py @@ -0,0 +1,47 @@ +# +# SQLAlchemy models for foreign tables. +# +# The order of the columns must match the remote Oracle database. The names are not required to +# match by oracle_fdw, but we are matching them for maintainability. +# + +import datetime + +from sqlalchemy.orm import Mapped, declarative_mixin, mapped_column + + +@declarative_mixin +class TopportunityMixin: + opportunity_id: Mapped[int] = mapped_column(primary_key=True) + oppnumber: Mapped[str | None] + revision_number: Mapped[int | None] + opptitle: Mapped[str | None] + owningagency: Mapped[str | None] + publisheruid: Mapped[str | None] + listed: Mapped[str | None] + oppcategory: Mapped[str | None] + initial_opportunity_id: Mapped[int | None] + modified_comments: Mapped[str | None] + created_date: Mapped[datetime.datetime | None] + last_upd_date: Mapped[datetime.datetime | None] + creator_id: Mapped[str | None] + last_upd_id: Mapped[str | None] + flag_2006: Mapped[str | None] + category_explanation: Mapped[str | None] + publisher_profile_id: Mapped[int | None] + is_draft: Mapped[str | None] + + +@declarative_mixin +class TopportunityCfdaMixin: + opp_cfda_id: Mapped[int] = mapped_column(primary_key=True) + opportunity_id: Mapped[int] + cfdanumber: Mapped[str | None] + programtitle: Mapped[str | None] + origtoppid: Mapped[int | None] + oppidcfdanum: Mapped[str | None] + origoppnum: Mapped[str | None] + created_date: Mapped[datetime.datetime | None] + last_upd_date: Mapped[datetime.datetime | None] + creator_id: Mapped[str | None] + last_upd_id: Mapped[str | None] diff --git a/api/src/db/legacy_mixin/synopsis_mixin.py b/api/src/db/legacy_mixin/synopsis_mixin.py new file mode 100644 index 000000000..e478bb56c --- /dev/null +++ b/api/src/db/legacy_mixin/synopsis_mixin.py @@ -0,0 +1,160 @@ +# +# SQLAlchemy models for foreign tables. +# +# The order of the columns must match the remote Oracle database. The names are not required to +# match by oracle_fdw, but we are matching them for maintainability. +# + +import datetime + +from sqlalchemy.orm import Mapped, declarative_mixin, mapped_column + + +@declarative_mixin +class TsynopsisMixin: + opportunity_id: Mapped[int] = mapped_column(primary_key=True) + posting_date: Mapped[datetime.datetime | None] + response_date: Mapped[datetime.datetime | None] + archive_date: Mapped[datetime.datetime | None] + unarchive_date: Mapped[datetime.datetime | None] + syn_desc: Mapped[str | None] + oth_cat_fa_desc: Mapped[str | None] + agency_addr_desc: Mapped[str | None] + cost_sharing: Mapped[str | None] + number_of_awards: Mapped[str | None] + est_funding: Mapped[str | None] + award_ceiling: Mapped[str | None] + award_floor: Mapped[str | None] + fd_link_url: Mapped[str | None] + fd_link_desc: Mapped[str | None] + agency_contact_desc: Mapped[str | None] + ac_email_addr: Mapped[str | None] + ac_email_desc: Mapped[str | None] + agency_name: Mapped[str | None] + agency_phone: Mapped[str | None] + a_sa_code: Mapped[str | None] + ac_phone_number: Mapped[str | None] + ac_name: Mapped[str | None] + create_ts: Mapped[datetime.datetime] + created_date: Mapped[datetime.datetime | None] + last_upd_date: Mapped[datetime.datetime | None] + creator_id: Mapped[str | None] + last_upd_id: Mapped[str | None] + sendmail: Mapped[str | None] + response_date_desc: Mapped[str | None] + applicant_elig_desc: Mapped[str | None] + version_nbr: Mapped[int | None] + modification_comments: Mapped[str | None] + publisheruid: Mapped[str | None] + publisher_profile_id: Mapped[int | None] + + +@declarative_mixin +class TsynopsisHistMixin: + opportunity_id: Mapped[int] = mapped_column(primary_key=True) + revision_number: Mapped[int] = mapped_column(primary_key=True) + posting_date: Mapped[datetime.datetime | None] + response_date: Mapped[datetime.datetime | None] + archive_date: Mapped[datetime.datetime | None] + unarchive_date: Mapped[datetime.datetime | None] + syn_desc: Mapped[str | None] + oth_cat_fa_desc: Mapped[str | None] + agency_addr_desc: Mapped[str | None] + cost_sharing: Mapped[str | None] + number_of_awards: Mapped[str | None] + est_funding: Mapped[str | None] + award_ceiling: Mapped[str | None] + award_floor: Mapped[str | None] + fd_link_url: Mapped[str | None] + fd_link_desc: Mapped[str | None] + agency_contact_desc: Mapped[str | None] + ac_email_addr: Mapped[str | None] + ac_email_desc: Mapped[str | None] + agency_name: Mapped[str | None] + agency_phone: Mapped[str | None] + a_sa_code: Mapped[str | None] + ac_phone_number: Mapped[str | None] + ac_name: Mapped[str | None] + create_ts: Mapped[datetime.datetime] + created_date: Mapped[datetime.datetime | None] + last_upd_date: Mapped[datetime.datetime | None] + creator_id: Mapped[str | None] + last_upd_id: Mapped[str | None] + sendmail: Mapped[str | None] + response_date_desc: Mapped[str | None] + applicant_elig_desc: Mapped[str | None] + action_type: Mapped[str | None] + action_date: Mapped[datetime.datetime | None] + version_nbr: Mapped[int] + modification_comments: Mapped[str | None] + publisheruid: Mapped[str | None] + publisher_profile_id: Mapped[int | None] + + +@declarative_mixin +class TapplicanttypesSynopsisMixin: + at_syn_id: Mapped[int] = mapped_column(primary_key=True) + at_id: Mapped[str] = mapped_column(primary_key=True) + opportunity_id: Mapped[int] = mapped_column(primary_key=True) + created_date: Mapped[datetime.datetime | None] + last_upd_date: Mapped[datetime.datetime | None] + creator_id: Mapped[str | None] + last_upd_id: Mapped[str | None] + + +@declarative_mixin +class TapplicanttypesSynopsisHistMixin: + at_syn_id: Mapped[int] = mapped_column(primary_key=True) + at_id: Mapped[str] = mapped_column(primary_key=True) + opportunity_id: Mapped[int] = mapped_column(primary_key=True) + revision_number: Mapped[int] = mapped_column(primary_key=True) + created_date: Mapped[datetime.datetime | None] + last_upd_date: Mapped[datetime.datetime | None] + creator_id: Mapped[str | None] + last_upd_id: Mapped[str | None] + + +@declarative_mixin +class TfundactcatSynopsisMixin: + fac_syn_id: Mapped[int] = mapped_column(primary_key=True) + fac_id: Mapped[str] = mapped_column(primary_key=True) + opportunity_id: Mapped[int] = mapped_column(primary_key=True) + created_date: Mapped[datetime.datetime | None] + last_upd_date: Mapped[datetime.datetime | None] + creator_id: Mapped[str | None] + last_upd_id: Mapped[str | None] + + +@declarative_mixin +class TfundactcatSynopsisHistMixin: + fac_syn_id: Mapped[int] = mapped_column(primary_key=True) + fac_id: Mapped[str] = mapped_column(primary_key=True) + opportunity_id: Mapped[int] = mapped_column(primary_key=True) + revision_number: Mapped[int] = mapped_column(primary_key=True) + created_date: Mapped[datetime.datetime | None] + last_upd_date: Mapped[datetime.datetime | None] + creator_id: Mapped[str | None] + last_upd_id: Mapped[str | None] + + +@declarative_mixin +class TfundinstrSynopsisMixin: + fi_syn_id: Mapped[int] = mapped_column(primary_key=True) + fi_id: Mapped[str] = mapped_column(primary_key=True) + opportunity_id: Mapped[int] = mapped_column(primary_key=True) + created_date: Mapped[datetime.datetime | None] + last_upd_date: Mapped[datetime.datetime | None] + creator_id: Mapped[str | None] + last_upd_id: Mapped[str | None] + + +@declarative_mixin +class TfundinstrSynopsisHistMixin: + fi_syn_id: Mapped[int] = mapped_column(primary_key=True) + opportunity_id: Mapped[int] = mapped_column(primary_key=True) + fi_id: Mapped[str] = mapped_column(primary_key=True) + revision_number: Mapped[int] = mapped_column(primary_key=True) + created_date: Mapped[datetime.datetime | None] + last_upd_date: Mapped[datetime.datetime | None] + creator_id: Mapped[str | None] + last_upd_id: Mapped[str | None] diff --git a/api/src/db/migrations/env.py b/api/src/db/migrations/env.py index 3100d77b6..4abeee6ca 100644 --- a/api/src/db/migrations/env.py +++ b/api/src/db/migrations/env.py @@ -8,6 +8,7 @@ import src.logging from src.constants.schema import Schemas from src.db.models import metadata +from src.db.models.staging import metadata as staging_metadata from src.adapters.db.type_decorators.postgres_type_decorators import LookupColumn # isort:skip @@ -23,7 +24,7 @@ # for 'autogenerate' support # from myapp import mymodel # target_metadata = mymodel.Base.metadata - target_metadata = metadata + target_metadata = [metadata, staging_metadata] # other values from the config, defined by the needs of env.py, # can be acquired: diff --git a/api/src/db/migrations/versions/2024_04_24_add_staging_tables.py b/api/src/db/migrations/versions/2024_04_24_add_staging_tables.py new file mode 100644 index 000000000..30534bb76 --- /dev/null +++ b/api/src/db/migrations/versions/2024_04_24_add_staging_tables.py @@ -0,0 +1,686 @@ +"""Add staging tables + +Revision ID: e3a1be603d26 +Revises: c6878bae0c60 +Create Date: 2024-04-24 15:27:13.602523 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "e3a1be603d26" +down_revision = "c6878bae0c60" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "tapplicanttypes_forecast", + sa.Column("at_frcst_id", sa.BigInteger(), nullable=False), + sa.Column("at_id", sa.Text(), nullable=False), + sa.Column("opportunity_id", sa.BigInteger(), nullable=False), + sa.Column("created_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("last_upd_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("creator_id", sa.Text(), nullable=True), + sa.Column("last_upd_id", sa.Text(), nullable=True), + sa.Column("is_deleted", sa.Boolean(), nullable=False), + sa.Column("transformed_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.PrimaryKeyConstraint( + "at_frcst_id", "at_id", "opportunity_id", name=op.f("tapplicanttypes_forecast_pkey") + ), + schema="staging", + ) + op.create_index( + op.f("tapplicanttypes_forecast_transformed_at_idx"), + "tapplicanttypes_forecast", + ["transformed_at"], + unique=False, + schema="staging", + ) + op.create_table( + "tapplicanttypes_forecast_hist", + sa.Column("at_frcst_id", sa.BigInteger(), nullable=False), + sa.Column("at_id", sa.Text(), nullable=False), + sa.Column("opportunity_id", sa.BigInteger(), nullable=False), + sa.Column("revision_number", sa.BigInteger(), nullable=False), + sa.Column("created_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("last_upd_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("creator_id", sa.Text(), nullable=True), + sa.Column("last_upd_id", sa.Text(), nullable=True), + sa.Column("is_deleted", sa.Boolean(), nullable=False), + sa.Column("transformed_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.PrimaryKeyConstraint( + "at_frcst_id", + "at_id", + "opportunity_id", + "revision_number", + name=op.f("tapplicanttypes_forecast_hist_pkey"), + ), + schema="staging", + ) + op.create_index( + op.f("tapplicanttypes_forecast_hist_transformed_at_idx"), + "tapplicanttypes_forecast_hist", + ["transformed_at"], + unique=False, + schema="staging", + ) + op.create_table( + "tapplicanttypes_synopsis", + sa.Column("at_syn_id", sa.BigInteger(), nullable=False), + sa.Column("at_id", sa.Text(), nullable=False), + sa.Column("opportunity_id", sa.BigInteger(), nullable=False), + sa.Column("created_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("last_upd_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("creator_id", sa.Text(), nullable=True), + sa.Column("last_upd_id", sa.Text(), nullable=True), + sa.Column("is_deleted", sa.Boolean(), nullable=False), + sa.Column("transformed_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.PrimaryKeyConstraint( + "at_syn_id", "at_id", "opportunity_id", name=op.f("tapplicanttypes_synopsis_pkey") + ), + schema="staging", + ) + op.create_index( + op.f("tapplicanttypes_synopsis_transformed_at_idx"), + "tapplicanttypes_synopsis", + ["transformed_at"], + unique=False, + schema="staging", + ) + op.create_table( + "tapplicanttypes_synopsis_hist", + sa.Column("at_syn_id", sa.BigInteger(), nullable=False), + sa.Column("at_id", sa.Text(), nullable=False), + sa.Column("opportunity_id", sa.BigInteger(), nullable=False), + sa.Column("revision_number", sa.BigInteger(), nullable=False), + sa.Column("created_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("last_upd_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("creator_id", sa.Text(), nullable=True), + sa.Column("last_upd_id", sa.Text(), nullable=True), + sa.Column("is_deleted", sa.Boolean(), nullable=False), + sa.Column("transformed_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.PrimaryKeyConstraint( + "at_syn_id", + "at_id", + "opportunity_id", + "revision_number", + name=op.f("tapplicanttypes_synopsis_hist_pkey"), + ), + schema="staging", + ) + op.create_index( + op.f("tapplicanttypes_synopsis_hist_transformed_at_idx"), + "tapplicanttypes_synopsis_hist", + ["transformed_at"], + unique=False, + schema="staging", + ) + op.create_table( + "tforecast", + sa.Column("opportunity_id", sa.BigInteger(), nullable=False), + sa.Column("version_nbr", sa.BigInteger(), nullable=False), + sa.Column("posting_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("archive_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("forecast_desc", sa.Text(), nullable=True), + sa.Column("oth_cat_fa_desc", sa.Text(), nullable=True), + sa.Column("cost_sharing", sa.Text(), nullable=True), + sa.Column("number_of_awards", sa.Text(), nullable=True), + sa.Column("est_funding", sa.Text(), nullable=True), + sa.Column("award_ceiling", sa.Text(), nullable=True), + sa.Column("award_floor", sa.Text(), nullable=True), + sa.Column("fd_link_url", sa.Text(), nullable=True), + sa.Column("fd_link_desc", sa.Text(), nullable=True), + sa.Column("ac_name", sa.Text(), nullable=True), + sa.Column("ac_phone", sa.Text(), nullable=True), + sa.Column("ac_email_addr", sa.Text(), nullable=True), + sa.Column("ac_email_desc", sa.Text(), nullable=True), + sa.Column("agency_code", sa.Text(), nullable=True), + sa.Column("sendmail", sa.Text(), nullable=True), + sa.Column("applicant_elig_desc", sa.Text(), nullable=True), + sa.Column("est_synopsis_posting_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("est_appl_response_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("est_appl_response_date_desc", sa.Text(), nullable=True), + sa.Column("est_award_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("est_project_start_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("fiscal_year", sa.BigInteger(), nullable=True), + sa.Column("modification_comments", sa.Text(), nullable=True), + sa.Column("create_ts", sa.TIMESTAMP(timezone=True), nullable=False), + sa.Column("created_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("last_upd_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("creator_id", sa.Text(), nullable=True), + sa.Column("last_upd_id", sa.Text(), nullable=True), + sa.Column("publisheruid", sa.Text(), nullable=True), + sa.Column("publisher_profile_id", sa.BigInteger(), nullable=True), + sa.Column("is_deleted", sa.Boolean(), nullable=False), + sa.Column("transformed_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.PrimaryKeyConstraint("opportunity_id", name=op.f("tforecast_pkey")), + schema="staging", + ) + op.create_index( + op.f("tforecast_transformed_at_idx"), + "tforecast", + ["transformed_at"], + unique=False, + schema="staging", + ) + op.create_table( + "tforecast_hist", + sa.Column("opportunity_id", sa.BigInteger(), nullable=False), + sa.Column("revision_number", sa.BigInteger(), nullable=False), + sa.Column("version_nbr", sa.BigInteger(), nullable=False), + sa.Column("posting_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("archive_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("forecast_desc", sa.Text(), nullable=True), + sa.Column("oth_cat_fa_desc", sa.Text(), nullable=True), + sa.Column("cost_sharing", sa.Text(), nullable=True), + sa.Column("number_of_awards", sa.Text(), nullable=True), + sa.Column("est_funding", sa.Text(), nullable=True), + sa.Column("award_ceiling", sa.Text(), nullable=True), + sa.Column("award_floor", sa.Text(), nullable=True), + sa.Column("fd_link_url", sa.Text(), nullable=True), + sa.Column("fd_link_desc", sa.Text(), nullable=True), + sa.Column("ac_name", sa.Text(), nullable=True), + sa.Column("ac_phone", sa.Text(), nullable=True), + sa.Column("ac_email_addr", sa.Text(), nullable=True), + sa.Column("ac_email_desc", sa.Text(), nullable=True), + sa.Column("agency_code", sa.Text(), nullable=True), + sa.Column("sendmail", sa.Text(), nullable=True), + sa.Column("applicant_elig_desc", sa.Text(), nullable=True), + sa.Column("est_synopsis_posting_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("est_appl_response_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("est_appl_response_date_desc", sa.Text(), nullable=True), + sa.Column("est_award_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("est_project_start_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("fiscal_year", sa.BigInteger(), nullable=True), + sa.Column("modification_comments", sa.Text(), nullable=True), + sa.Column("action_type", sa.Text(), nullable=True), + sa.Column("action_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("create_ts", sa.TIMESTAMP(timezone=True), nullable=False), + sa.Column("created_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("last_upd_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("creator_id", sa.Text(), nullable=True), + sa.Column("last_upd_id", sa.Text(), nullable=True), + sa.Column("publisheruid", sa.Text(), nullable=True), + sa.Column("publisher_profile_id", sa.BigInteger(), nullable=True), + sa.Column("is_deleted", sa.Boolean(), nullable=False), + sa.Column("transformed_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.PrimaryKeyConstraint( + "opportunity_id", "revision_number", name=op.f("tforecast_hist_pkey") + ), + schema="staging", + ) + op.create_index( + op.f("tforecast_hist_transformed_at_idx"), + "tforecast_hist", + ["transformed_at"], + unique=False, + schema="staging", + ) + op.create_table( + "tfundactcat_forecast", + sa.Column("fac_frcst_id", sa.BigInteger(), nullable=False), + sa.Column("fac_id", sa.Text(), nullable=False), + sa.Column("opportunity_id", sa.BigInteger(), nullable=False), + sa.Column("created_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("last_upd_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("creator_id", sa.Text(), nullable=True), + sa.Column("last_upd_id", sa.Text(), nullable=True), + sa.Column("is_deleted", sa.Boolean(), nullable=False), + sa.Column("transformed_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.PrimaryKeyConstraint( + "fac_frcst_id", "fac_id", "opportunity_id", name=op.f("tfundactcat_forecast_pkey") + ), + schema="staging", + ) + op.create_index( + op.f("tfundactcat_forecast_transformed_at_idx"), + "tfundactcat_forecast", + ["transformed_at"], + unique=False, + schema="staging", + ) + op.create_table( + "tfundactcat_forecast_hist", + sa.Column("fac_frcst_id", sa.BigInteger(), nullable=False), + sa.Column("fac_id", sa.Text(), nullable=False), + sa.Column("opportunity_id", sa.BigInteger(), nullable=False), + sa.Column("revision_number", sa.BigInteger(), nullable=False), + sa.Column("created_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("last_upd_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("creator_id", sa.Text(), nullable=True), + sa.Column("last_upd_id", sa.Text(), nullable=True), + sa.Column("is_deleted", sa.Boolean(), nullable=False), + sa.Column("transformed_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.PrimaryKeyConstraint( + "fac_frcst_id", + "fac_id", + "opportunity_id", + "revision_number", + name=op.f("tfundactcat_forecast_hist_pkey"), + ), + schema="staging", + ) + op.create_index( + op.f("tfundactcat_forecast_hist_transformed_at_idx"), + "tfundactcat_forecast_hist", + ["transformed_at"], + unique=False, + schema="staging", + ) + op.create_table( + "tfundactcat_synopsis", + sa.Column("fac_syn_id", sa.BigInteger(), nullable=False), + sa.Column("fac_id", sa.Text(), nullable=False), + sa.Column("opportunity_id", sa.BigInteger(), nullable=False), + sa.Column("created_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("last_upd_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("creator_id", sa.Text(), nullable=True), + sa.Column("last_upd_id", sa.Text(), nullable=True), + sa.Column("is_deleted", sa.Boolean(), nullable=False), + sa.Column("transformed_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.PrimaryKeyConstraint( + "fac_syn_id", "fac_id", "opportunity_id", name=op.f("tfundactcat_synopsis_pkey") + ), + schema="staging", + ) + op.create_index( + op.f("tfundactcat_synopsis_transformed_at_idx"), + "tfundactcat_synopsis", + ["transformed_at"], + unique=False, + schema="staging", + ) + op.create_table( + "tfundactcat_synopsis_hist", + sa.Column("fac_syn_id", sa.BigInteger(), nullable=False), + sa.Column("fac_id", sa.Text(), nullable=False), + sa.Column("opportunity_id", sa.BigInteger(), nullable=False), + sa.Column("revision_number", sa.BigInteger(), nullable=False), + sa.Column("created_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("last_upd_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("creator_id", sa.Text(), nullable=True), + sa.Column("last_upd_id", sa.Text(), nullable=True), + sa.Column("is_deleted", sa.Boolean(), nullable=False), + sa.Column("transformed_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.PrimaryKeyConstraint( + "fac_syn_id", + "fac_id", + "opportunity_id", + "revision_number", + name=op.f("tfundactcat_synopsis_hist_pkey"), + ), + schema="staging", + ) + op.create_index( + op.f("tfundactcat_synopsis_hist_transformed_at_idx"), + "tfundactcat_synopsis_hist", + ["transformed_at"], + unique=False, + schema="staging", + ) + op.create_table( + "tfundinstr_forecast", + sa.Column("fi_frcst_id", sa.BigInteger(), nullable=False), + sa.Column("fi_id", sa.Text(), nullable=False), + sa.Column("opportunity_id", sa.BigInteger(), nullable=False), + sa.Column("created_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("last_upd_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("creator_id", sa.Text(), nullable=True), + sa.Column("last_upd_id", sa.Text(), nullable=True), + sa.Column("is_deleted", sa.Boolean(), nullable=False), + sa.Column("transformed_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.PrimaryKeyConstraint( + "fi_frcst_id", "fi_id", "opportunity_id", name=op.f("tfundinstr_forecast_pkey") + ), + schema="staging", + ) + op.create_index( + op.f("tfundinstr_forecast_transformed_at_idx"), + "tfundinstr_forecast", + ["transformed_at"], + unique=False, + schema="staging", + ) + op.create_table( + "tfundinstr_forecast_hist", + sa.Column("fi_frcst_id", sa.BigInteger(), nullable=False), + sa.Column("fi_id", sa.Text(), nullable=False), + sa.Column("opportunity_id", sa.BigInteger(), nullable=False), + sa.Column("revision_number", sa.BigInteger(), nullable=False), + sa.Column("created_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("last_upd_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("creator_id", sa.Text(), nullable=True), + sa.Column("last_upd_id", sa.Text(), nullable=True), + sa.Column("is_deleted", sa.Boolean(), nullable=False), + sa.Column("transformed_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.PrimaryKeyConstraint( + "fi_frcst_id", + "fi_id", + "opportunity_id", + "revision_number", + name=op.f("tfundinstr_forecast_hist_pkey"), + ), + schema="staging", + ) + op.create_index( + op.f("tfundinstr_forecast_hist_transformed_at_idx"), + "tfundinstr_forecast_hist", + ["transformed_at"], + unique=False, + schema="staging", + ) + op.create_table( + "tfundinstr_synopsis", + sa.Column("fi_syn_id", sa.BigInteger(), nullable=False), + sa.Column("fi_id", sa.Text(), nullable=False), + sa.Column("opportunity_id", sa.BigInteger(), nullable=False), + sa.Column("created_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("last_upd_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("creator_id", sa.Text(), nullable=True), + sa.Column("last_upd_id", sa.Text(), nullable=True), + sa.Column("is_deleted", sa.Boolean(), nullable=False), + sa.Column("transformed_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.PrimaryKeyConstraint( + "fi_syn_id", "fi_id", "opportunity_id", name=op.f("tfundinstr_synopsis_pkey") + ), + schema="staging", + ) + op.create_index( + op.f("tfundinstr_synopsis_transformed_at_idx"), + "tfundinstr_synopsis", + ["transformed_at"], + unique=False, + schema="staging", + ) + op.create_table( + "tfundinstr_synopsis_hist", + sa.Column("fi_syn_id", sa.BigInteger(), nullable=False), + sa.Column("opportunity_id", sa.BigInteger(), nullable=False), + sa.Column("fi_id", sa.Text(), nullable=False), + sa.Column("revision_number", sa.BigInteger(), nullable=False), + sa.Column("created_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("last_upd_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("creator_id", sa.Text(), nullable=True), + sa.Column("last_upd_id", sa.Text(), nullable=True), + sa.Column("is_deleted", sa.Boolean(), nullable=False), + sa.Column("transformed_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.PrimaryKeyConstraint( + "fi_syn_id", + "opportunity_id", + "fi_id", + "revision_number", + name=op.f("tfundinstr_synopsis_hist_pkey"), + ), + schema="staging", + ) + op.create_index( + op.f("tfundinstr_synopsis_hist_transformed_at_idx"), + "tfundinstr_synopsis_hist", + ["transformed_at"], + unique=False, + schema="staging", + ) + op.create_table( + "topportunity", + sa.Column("opportunity_id", sa.BigInteger(), nullable=False), + sa.Column("oppnumber", sa.Text(), nullable=True), + sa.Column("revision_number", sa.BigInteger(), nullable=True), + sa.Column("opptitle", sa.Text(), nullable=True), + sa.Column("owningagency", sa.Text(), nullable=True), + sa.Column("publisheruid", sa.Text(), nullable=True), + sa.Column("listed", sa.Text(), nullable=True), + sa.Column("oppcategory", sa.Text(), nullable=True), + sa.Column("initial_opportunity_id", sa.BigInteger(), nullable=True), + sa.Column("modified_comments", sa.Text(), nullable=True), + sa.Column("created_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("last_upd_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("creator_id", sa.Text(), nullable=True), + sa.Column("last_upd_id", sa.Text(), nullable=True), + sa.Column("flag_2006", sa.Text(), nullable=True), + sa.Column("category_explanation", sa.Text(), nullable=True), + sa.Column("publisher_profile_id", sa.BigInteger(), nullable=True), + sa.Column("is_draft", sa.Text(), nullable=True), + sa.Column("is_deleted", sa.Boolean(), nullable=False), + sa.Column("transformed_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.PrimaryKeyConstraint("opportunity_id", name=op.f("topportunity_pkey")), + schema="staging", + ) + op.create_index( + op.f("topportunity_transformed_at_idx"), + "topportunity", + ["transformed_at"], + unique=False, + schema="staging", + ) + op.create_table( + "topportunity_cfda", + sa.Column("opp_cfda_id", sa.BigInteger(), nullable=False), + sa.Column("opportunity_id", sa.BigInteger(), nullable=False), + sa.Column("cfdanumber", sa.Text(), nullable=True), + sa.Column("programtitle", sa.Text(), nullable=True), + sa.Column("origtoppid", sa.BigInteger(), nullable=True), + sa.Column("oppidcfdanum", sa.Text(), nullable=True), + sa.Column("origoppnum", sa.Text(), nullable=True), + sa.Column("created_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("last_upd_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("creator_id", sa.Text(), nullable=True), + sa.Column("last_upd_id", sa.Text(), nullable=True), + sa.Column("is_deleted", sa.Boolean(), nullable=False), + sa.Column("transformed_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.PrimaryKeyConstraint("opp_cfda_id", name=op.f("topportunity_cfda_pkey")), + schema="staging", + ) + op.create_index( + op.f("topportunity_cfda_transformed_at_idx"), + "topportunity_cfda", + ["transformed_at"], + unique=False, + schema="staging", + ) + op.create_table( + "tsynopsis", + sa.Column("opportunity_id", sa.BigInteger(), nullable=False), + sa.Column("posting_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("response_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("archive_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("unarchive_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("syn_desc", sa.Text(), nullable=True), + sa.Column("oth_cat_fa_desc", sa.Text(), nullable=True), + sa.Column("agency_addr_desc", sa.Text(), nullable=True), + sa.Column("cost_sharing", sa.Text(), nullable=True), + sa.Column("number_of_awards", sa.Text(), nullable=True), + sa.Column("est_funding", sa.Text(), nullable=True), + sa.Column("award_ceiling", sa.Text(), nullable=True), + sa.Column("award_floor", sa.Text(), nullable=True), + sa.Column("fd_link_url", sa.Text(), nullable=True), + sa.Column("fd_link_desc", sa.Text(), nullable=True), + sa.Column("agency_contact_desc", sa.Text(), nullable=True), + sa.Column("ac_email_addr", sa.Text(), nullable=True), + sa.Column("ac_email_desc", sa.Text(), nullable=True), + sa.Column("agency_name", sa.Text(), nullable=True), + sa.Column("agency_phone", sa.Text(), nullable=True), + sa.Column("a_sa_code", sa.Text(), nullable=True), + sa.Column("ac_phone_number", sa.Text(), nullable=True), + sa.Column("ac_name", sa.Text(), nullable=True), + sa.Column("create_ts", sa.TIMESTAMP(timezone=True), nullable=False), + sa.Column("created_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("last_upd_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("creator_id", sa.Text(), nullable=True), + sa.Column("last_upd_id", sa.Text(), nullable=True), + sa.Column("sendmail", sa.Text(), nullable=True), + sa.Column("response_date_desc", sa.Text(), nullable=True), + sa.Column("applicant_elig_desc", sa.Text(), nullable=True), + sa.Column("version_nbr", sa.BigInteger(), nullable=True), + sa.Column("modification_comments", sa.Text(), nullable=True), + sa.Column("publisheruid", sa.Text(), nullable=True), + sa.Column("publisher_profile_id", sa.BigInteger(), nullable=True), + sa.Column("is_deleted", sa.Boolean(), nullable=False), + sa.Column("transformed_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.PrimaryKeyConstraint("opportunity_id", name=op.f("tsynopsis_pkey")), + schema="staging", + ) + op.create_index( + op.f("tsynopsis_transformed_at_idx"), + "tsynopsis", + ["transformed_at"], + unique=False, + schema="staging", + ) + op.create_table( + "tsynopsis_hist", + sa.Column("opportunity_id", sa.BigInteger(), nullable=False), + sa.Column("revision_number", sa.BigInteger(), nullable=False), + sa.Column("posting_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("response_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("archive_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("unarchive_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("syn_desc", sa.Text(), nullable=True), + sa.Column("oth_cat_fa_desc", sa.Text(), nullable=True), + sa.Column("agency_addr_desc", sa.Text(), nullable=True), + sa.Column("cost_sharing", sa.Text(), nullable=True), + sa.Column("number_of_awards", sa.Text(), nullable=True), + sa.Column("est_funding", sa.Text(), nullable=True), + sa.Column("award_ceiling", sa.Text(), nullable=True), + sa.Column("award_floor", sa.Text(), nullable=True), + sa.Column("fd_link_url", sa.Text(), nullable=True), + sa.Column("fd_link_desc", sa.Text(), nullable=True), + sa.Column("agency_contact_desc", sa.Text(), nullable=True), + sa.Column("ac_email_addr", sa.Text(), nullable=True), + sa.Column("ac_email_desc", sa.Text(), nullable=True), + sa.Column("agency_name", sa.Text(), nullable=True), + sa.Column("agency_phone", sa.Text(), nullable=True), + sa.Column("a_sa_code", sa.Text(), nullable=True), + sa.Column("ac_phone_number", sa.Text(), nullable=True), + sa.Column("ac_name", sa.Text(), nullable=True), + sa.Column("create_ts", sa.TIMESTAMP(timezone=True), nullable=False), + sa.Column("created_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("last_upd_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("creator_id", sa.Text(), nullable=True), + sa.Column("last_upd_id", sa.Text(), nullable=True), + sa.Column("sendmail", sa.Text(), nullable=True), + sa.Column("response_date_desc", sa.Text(), nullable=True), + sa.Column("applicant_elig_desc", sa.Text(), nullable=True), + sa.Column("action_type", sa.Text(), nullable=True), + sa.Column("action_date", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("version_nbr", sa.BigInteger(), nullable=False), + sa.Column("modification_comments", sa.Text(), nullable=True), + sa.Column("publisheruid", sa.Text(), nullable=True), + sa.Column("publisher_profile_id", sa.BigInteger(), nullable=True), + sa.Column("is_deleted", sa.Boolean(), nullable=False), + sa.Column("transformed_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.PrimaryKeyConstraint( + "opportunity_id", "revision_number", name=op.f("tsynopsis_hist_pkey") + ), + schema="staging", + ) + op.create_index( + op.f("tsynopsis_hist_transformed_at_idx"), + "tsynopsis_hist", + ["transformed_at"], + unique=False, + schema="staging", + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index( + op.f("tsynopsis_hist_transformed_at_idx"), table_name="tsynopsis_hist", schema="staging" + ) + op.drop_table("tsynopsis_hist", schema="staging") + op.drop_index(op.f("tsynopsis_transformed_at_idx"), table_name="tsynopsis", schema="staging") + op.drop_table("tsynopsis", schema="staging") + op.drop_index( + op.f("topportunity_cfda_transformed_at_idx"), + table_name="topportunity_cfda", + schema="staging", + ) + op.drop_table("topportunity_cfda", schema="staging") + op.drop_index( + op.f("topportunity_transformed_at_idx"), table_name="topportunity", schema="staging" + ) + op.drop_table("topportunity", schema="staging") + op.drop_index( + op.f("tfundinstr_synopsis_hist_transformed_at_idx"), + table_name="tfundinstr_synopsis_hist", + schema="staging", + ) + op.drop_table("tfundinstr_synopsis_hist", schema="staging") + op.drop_index( + op.f("tfundinstr_synopsis_transformed_at_idx"), + table_name="tfundinstr_synopsis", + schema="staging", + ) + op.drop_table("tfundinstr_synopsis", schema="staging") + op.drop_index( + op.f("tfundinstr_forecast_hist_transformed_at_idx"), + table_name="tfundinstr_forecast_hist", + schema="staging", + ) + op.drop_table("tfundinstr_forecast_hist", schema="staging") + op.drop_index( + op.f("tfundinstr_forecast_transformed_at_idx"), + table_name="tfundinstr_forecast", + schema="staging", + ) + op.drop_table("tfundinstr_forecast", schema="staging") + op.drop_index( + op.f("tfundactcat_synopsis_hist_transformed_at_idx"), + table_name="tfundactcat_synopsis_hist", + schema="staging", + ) + op.drop_table("tfundactcat_synopsis_hist", schema="staging") + op.drop_index( + op.f("tfundactcat_synopsis_transformed_at_idx"), + table_name="tfundactcat_synopsis", + schema="staging", + ) + op.drop_table("tfundactcat_synopsis", schema="staging") + op.drop_index( + op.f("tfundactcat_forecast_hist_transformed_at_idx"), + table_name="tfundactcat_forecast_hist", + schema="staging", + ) + op.drop_table("tfundactcat_forecast_hist", schema="staging") + op.drop_index( + op.f("tfundactcat_forecast_transformed_at_idx"), + table_name="tfundactcat_forecast", + schema="staging", + ) + op.drop_table("tfundactcat_forecast", schema="staging") + op.drop_index( + op.f("tforecast_hist_transformed_at_idx"), table_name="tforecast_hist", schema="staging" + ) + op.drop_table("tforecast_hist", schema="staging") + op.drop_index(op.f("tforecast_transformed_at_idx"), table_name="tforecast", schema="staging") + op.drop_table("tforecast", schema="staging") + op.drop_index( + op.f("tapplicanttypes_synopsis_hist_transformed_at_idx"), + table_name="tapplicanttypes_synopsis_hist", + schema="staging", + ) + op.drop_table("tapplicanttypes_synopsis_hist", schema="staging") + op.drop_index( + op.f("tapplicanttypes_synopsis_transformed_at_idx"), + table_name="tapplicanttypes_synopsis", + schema="staging", + ) + op.drop_table("tapplicanttypes_synopsis", schema="staging") + op.drop_index( + op.f("tapplicanttypes_forecast_hist_transformed_at_idx"), + table_name="tapplicanttypes_forecast_hist", + schema="staging", + ) + op.drop_table("tapplicanttypes_forecast_hist", schema="staging") + op.drop_index( + op.f("tapplicanttypes_forecast_transformed_at_idx"), + table_name="tapplicanttypes_forecast", + schema="staging", + ) + op.drop_table("tapplicanttypes_forecast", schema="staging") + # ### end Alembic commands ### diff --git a/api/src/db/models/staging/__init__.py b/api/src/db/models/staging/__init__.py new file mode 100644 index 000000000..d89da9dc9 --- /dev/null +++ b/api/src/db/models/staging/__init__.py @@ -0,0 +1,5 @@ +from . import forecast, opportunity, staging_base, synopsis + +metadata = staging_base.metadata + +__all__ = ["metadata", "opportunity", "forecast", "synopsis"] diff --git a/api/src/db/models/staging/forecast.py b/api/src/db/models/staging/forecast.py new file mode 100644 index 000000000..1fa99bdc4 --- /dev/null +++ b/api/src/db/models/staging/forecast.py @@ -0,0 +1,42 @@ +from src.db.legacy_mixin import forecast_mixin +from src.db.models.staging.staging_base import StagingBase, StagingParamMixin + + +class Tforecast(StagingBase, forecast_mixin.TforecastMixin, StagingParamMixin): + __tablename__ = "tforecast" + + +class TforecastHist(StagingBase, forecast_mixin.TforecastHistMixin, StagingParamMixin): + __tablename__ = "tforecast_hist" + + +class TapplicanttypesForecast( + StagingBase, forecast_mixin.TapplicanttypesForecastMixin, StagingParamMixin +): + __tablename__ = "tapplicanttypes_forecast" + + +class TapplicanttypesForecastHist( + StagingBase, forecast_mixin.TapplicanttypesForecastHistMixin, StagingParamMixin +): + __tablename__ = "tapplicanttypes_forecast_hist" + + +class TfundactcatForecast(StagingBase, forecast_mixin.TfundactcatForecastMixin, StagingParamMixin): + __tablename__ = "tfundactcat_forecast" + + +class TfundactcatForecastHist( + StagingBase, forecast_mixin.TfundactcatForecastHistMixin, StagingParamMixin +): + __tablename__ = "tfundactcat_forecast_hist" + + +class TfundinstrForecast(StagingBase, forecast_mixin.TfundinstrForecastMixin, StagingParamMixin): + __tablename__ = "tfundinstr_forecast" + + +class TfundinstrForecastHist( + StagingBase, forecast_mixin.TfundinstrForecastHistMixin, StagingParamMixin +): + __tablename__ = "tfundinstr_forecast_hist" diff --git a/api/src/db/models/staging/opportunity.py b/api/src/db/models/staging/opportunity.py new file mode 100644 index 000000000..5ec200645 --- /dev/null +++ b/api/src/db/models/staging/opportunity.py @@ -0,0 +1,10 @@ +from src.db.legacy_mixin import opportunity_mixin +from src.db.models.staging.staging_base import StagingBase, StagingParamMixin + + +class Topportunity(StagingBase, opportunity_mixin.TopportunityMixin, StagingParamMixin): + __tablename__ = "topportunity" + + +class TopportunityCfda(StagingBase, opportunity_mixin.TopportunityCfdaMixin, StagingParamMixin): + __tablename__ = "topportunity_cfda" diff --git a/api/src/db/models/staging/staging_base.py b/api/src/db/models/staging/staging_base.py new file mode 100644 index 000000000..a4b101f51 --- /dev/null +++ b/api/src/db/models/staging/staging_base.py @@ -0,0 +1,50 @@ +import datetime +from typing import Any, Iterable + +import sqlalchemy +from sqlalchemy.orm import Mapped, declarative_mixin, mapped_column + +from src.constants.schema import Schemas + +metadata = sqlalchemy.MetaData( + naming_convention={ + "ix": "%(table_name)s_%(column_0_name)s_idx", + "uq": "%(table_name)s_%(column_0_name)s_uniq", + "ck": "%(table_name)s_`%(constraint_name)s_check`", + "fk": "%(table_name)s_%(column_0_name)s_%(referred_table_name)s_fkey", + "pk": "%(table_name)s_pkey", + } +) + + +class StagingBase(sqlalchemy.orm.DeclarativeBase): + metadata = metadata + + __table_args__ = {"schema": Schemas.STAGING} + + # These types are selected so that the underlying Oracle types are mapped to a more general + # type. For example all CHAR and VARCHAR types can be mapped to TEXT for simplicity. See + # https://github.com/laurenz/oracle_fdw?tab=readme-ov-file#data-types + type_annotation_map = { + int: sqlalchemy.BigInteger, + str: sqlalchemy.Text, + datetime.datetime: sqlalchemy.TIMESTAMP(timezone=True), + } + + def _dict(self) -> dict: + return {c.key: getattr(self, c.key) for c in sqlalchemy.inspect(self).mapper.column_attrs} + + def __repr__(self) -> str: + return f"<{type(self).__name__}({self._dict()!r})" + + def __rich_repr__(self) -> Iterable[tuple[str, Any]]: + """Rich repr for interactive console. + See https://rich.readthedocs.io/en/latest/pretty.html#rich-repr-protocol + """ + return self._dict().items() + + +@declarative_mixin +class StagingParamMixin: + is_deleted: Mapped[bool] + transformed_at: Mapped[datetime.datetime | None] = mapped_column(index=True) diff --git a/api/src/db/models/staging/synopsis.py b/api/src/db/models/staging/synopsis.py new file mode 100644 index 000000000..574c1b1c1 --- /dev/null +++ b/api/src/db/models/staging/synopsis.py @@ -0,0 +1,42 @@ +from src.db.legacy_mixin import synopsis_mixin +from src.db.models.staging.staging_base import StagingBase, StagingParamMixin + + +class Tsynopsis(StagingBase, synopsis_mixin.TsynopsisMixin, StagingParamMixin): + __tablename__ = "tsynopsis" + + +class TsynopsisHist(StagingBase, synopsis_mixin.TsynopsisHistMixin, StagingParamMixin): + __tablename__ = "tsynopsis_hist" + + +class TapplicanttypesSynopsis( + StagingBase, synopsis_mixin.TapplicanttypesSynopsisMixin, StagingParamMixin +): + __tablename__ = "tapplicanttypes_synopsis" + + +class TapplicanttypesSynopsisHist( + StagingBase, synopsis_mixin.TapplicanttypesSynopsisHistMixin, StagingParamMixin +): + __tablename__ = "tapplicanttypes_synopsis_hist" + + +class TfundactcatSynopsis(StagingBase, synopsis_mixin.TfundactcatSynopsisMixin, StagingParamMixin): + __tablename__ = "tfundactcat_synopsis" + + +class TfundactcatSynopsisHist( + StagingBase, synopsis_mixin.TfundactcatSynopsisHistMixin, StagingParamMixin +): + __tablename__ = "tfundactcat_synopsis_hist" + + +class TfundinstrSynopsis(StagingBase, synopsis_mixin.TfundinstrSynopsisMixin, StagingParamMixin): + __tablename__ = "tfundinstr_synopsis" + + +class TfundinstrSynopsisHist( + StagingBase, synopsis_mixin.TfundinstrSynopsisHistMixin, StagingParamMixin +): + __tablename__ = "tfundinstr_synopsis_hist" diff --git a/api/tests/src/data_migration/test_setup_foreign_tables.py b/api/tests/src/data_migration/test_setup_foreign_tables.py index 400415f9b..1994f68e8 100644 --- a/api/tests/src/data_migration/test_setup_foreign_tables.py +++ b/api/tests/src/data_migration/test_setup_foreign_tables.py @@ -18,8 +18,8 @@ "oppcategory TEXT,", "initial_opportunity_id BIGINT,", "modified_comments TEXT,", - "created_date TIMESTAMP WITH TIME ZONE NOT NULL,", - "last_upd_date TIMESTAMP WITH TIME ZONE NOT NULL,", + "created_date TIMESTAMP WITH TIME ZONE,", + "last_upd_date TIMESTAMP WITH TIME ZONE,", "creator_id TEXT,", "last_upd_id TEXT,", "flag_2006 TEXT,", @@ -42,8 +42,8 @@ "oppcategory TEXT,", "initial_opportunity_id BIGINT,", "modified_comments TEXT,", - "created_date TIMESTAMP WITH TIME ZONE NOT NULL,", - "last_upd_date TIMESTAMP WITH TIME ZONE NOT NULL,", + "created_date TIMESTAMP WITH TIME ZONE,", + "last_upd_date TIMESTAMP WITH TIME ZONE,", "creator_id TEXT,", "last_upd_id TEXT,", "flag_2006 TEXT,", diff --git a/api/tests/src/db/models/factories.py b/api/tests/src/db/models/factories.py index 9553c228d..3b9658d6c 100644 --- a/api/tests/src/db/models/factories.py +++ b/api/tests/src/db/models/factories.py @@ -19,6 +19,7 @@ import src.adapters.db as db import src.db.models.opportunity_models as opportunity_models +import src.db.models.staging as staging import src.db.models.transfer.topportunity_models as transfer_topportunity_models import src.util.datetime_util as datetime_util from src.constants.lookup_constants import ( @@ -577,6 +578,47 @@ class Meta: applicant_type = factory.Iterator(ApplicantType) +#################################### +# Staging Table Factories +#################################### + + +class StagingTopportunityFactory(BaseFactory): + class Meta: + model = staging.opportunity.Topportunity + + opportunity_id = factory.Sequence(lambda n: n) + + oppnumber = factory.Faker("opportunity_number") + opptitle = factory.Faker("opportunity_title") + + owningagency = factory.Faker("agency") + + oppcategory = factory.fuzzy.FuzzyChoice(OpportunityCategoryLegacy) + # only set the category explanation if category is Other + category_explanation = factory.Maybe( + decider=factory.LazyAttribute(lambda o: o.oppcategory == OpportunityCategoryLegacy.OTHER), + yes_declaration=factory.Sequence(lambda n: f"Category as chosen by order #{n * n - 1}"), + no_declaration=None, + ) + + is_draft = "N" # Because we filter out drafts, just default these to False + + revision_number = 0 + + created_date = factory.Faker("date_between", start_date="-10y", end_date="-5y") + last_upd_date = factory.Faker("date_between", start_date="-5y", end_date="today") + + is_deleted = False + transformed_at = None + + class Params: + never_updated = factory.Trait(last_upd_date=None) + already_transformed = factory.Trait( + transformed_at=factory.Faker("date_time_between", start_date="-7d", end_date="-1d") + ) + + #################################### # Transfer Table Factories #################################### diff --git a/documentation/api/database/erds/README.md b/documentation/api/database/erds/README.md index 3e359baa8..9c5f07cfe 100644 --- a/documentation/api/database/erds/README.md +++ b/documentation/api/database/erds/README.md @@ -15,8 +15,14 @@ The diagrams generated are based on our SQLAlchemy models, and not the database # Files -## Full Schema -![Postgres ERD](full-schema.png) +## API Schema +![API Table ERD](api-schema.png) + +## Staging Schema +![Staging Table ERD](staging-schema.png) ## Transfer Table Schema -![Transfer Table ERD](transfer-schema.png) \ No newline at end of file +![Transfer Table ERD](transfer-schema.png) + +## Full Schema +![Postgres ERD](full-schema.png) \ No newline at end of file diff --git a/documentation/api/database/erds/api-schema.png b/documentation/api/database/erds/api-schema.png new file mode 100644 index 000000000..d35990fa4 Binary files /dev/null and b/documentation/api/database/erds/api-schema.png differ diff --git a/documentation/api/database/erds/full-schema.png b/documentation/api/database/erds/full-schema.png index 032c46be3..63cc23b7e 100644 Binary files a/documentation/api/database/erds/full-schema.png and b/documentation/api/database/erds/full-schema.png differ diff --git a/documentation/api/database/erds/staging-schema.png b/documentation/api/database/erds/staging-schema.png new file mode 100644 index 000000000..a795744bd Binary files /dev/null and b/documentation/api/database/erds/staging-schema.png differ