[Issue #1744] Setup staging tables for Oracle data load

api/bin/create_erds.py

@@ -8,6 +8,9 @@
 import pydot
 import sadisplay
 
+import src.db.models.staging.forecast as staging_forecast_models
+import src.db.models.staging.opportunity as staging_opportunity_models
+import src.db.models.staging.synopsis as staging_synopsis_models
 import src.logging
 from src.db.models import opportunity_models
 from src.db.models.transfer import topportunity_models
@@ -20,10 +23,17 @@
 ERD_FOLDER = pathlib.Path(__file__).parent.resolve()
 
 # If we want to generate separate files for more specific groups, we can set that up here
-ALL_MODULES = (opportunity_models, topportunity_models)
-
+API_MODULES = (opportunity_models,)
+STAGING_TABLE_MODULES = (
+    staging_opportunity_models,
+    staging_forecast_models,
+    staging_synopsis_models,
+)
 TRANSFER_TABLE_MODULES = (topportunity_models,)
 
+# Any modules you add above, merge together for the full schema to be generated
+ALL_MODULES = API_MODULES + STAGING_TABLE_MODULES + TRANSFER_TABLE_MODULES
+
 
 def create_erds(modules: Any, file_name: str) -> None:
     logger.info("Generating ERD diagrams for %s", file_name)
@@ -60,5 +70,6 @@ def main() -> None:
         logger.info("Generating ERD diagrams")
 
         create_erds(ALL_MODULES, "full-schema")
-
+        create_erds(API_MODULES, "api-schema")
+        create_erds(STAGING_TABLE_MODULES, "staging-schema")
         create_erds(TRANSFER_TABLE_MODULES, "transfer-schema")

api/src/db/foreign/forecast.py

@@ -5,164 +5,40 @@
 # match by oracle_fdw, but we are matching them for maintainability.
 #
 
-import datetime
-
-from sqlalchemy.orm import Mapped, mapped_column
+from src.db.legacy_mixin import forecast_mixin
 
 from . import foreignbase
 
 
-class Tforecast(foreignbase.ForeignBase):
+class Tforecast(foreignbase.ForeignBase, forecast_mixin.TforecastMixin):
     __tablename__ = "tforecast"
 
-    opportunity_id: Mapped[int] = mapped_column(primary_key=True)
-    version_nbr: Mapped[int]
-    posting_date: Mapped[datetime.datetime | None]
-    archive_date: Mapped[datetime.datetime | None]
-    forecast_desc: Mapped[str | None]
-    oth_cat_fa_desc: Mapped[str | None]
-    cost_sharing: Mapped[str | None]
-    number_of_awards: Mapped[str | None]
-    est_funding: Mapped[str | None]
-    award_ceiling: Mapped[str | None]
-    award_floor: Mapped[str | None]
-    fd_link_url: Mapped[str | None]
-    fd_link_desc: Mapped[str | None]
-    ac_name: Mapped[str | None]
-    ac_phone: Mapped[str | None]
-    ac_email_addr: Mapped[str | None]
-    ac_email_desc: Mapped[str | None]
-    agency_code: Mapped[str | None]
-    sendmail: Mapped[str | None]
-    applicant_elig_desc: Mapped[str | None]
-    est_synopsis_posting_date: Mapped[datetime.datetime | None]
-    est_appl_response_date: Mapped[datetime.datetime | None]
-    est_appl_response_date_desc: Mapped[str | None]
-    est_award_date: Mapped[datetime.datetime | None]
-    est_project_start_date: Mapped[datetime.datetime | None]
-    fiscal_year: Mapped[int | None]
-    modification_comments: Mapped[str | None]
-    create_ts: Mapped[datetime.datetime]
-    created_date: Mapped[datetime.datetime | None]
-    last_upd_date: Mapped[datetime.datetime | None]
-    creator_id: Mapped[str | None]
-    last_upd_id: Mapped[str | None]
-    publisheruid: Mapped[str | None]
-    publisher_profile_id: Mapped[int | None]
-
-
-class TforecastHist(foreignbase.ForeignBase):
+
+class TforecastHist(foreignbase.ForeignBase, forecast_mixin.TforecastHistMixin):
     __tablename__ = "tforecast_hist"
 
-    opportunity_id: Mapped[int] = mapped_column(primary_key=True)
-    revision_number: Mapped[int] = mapped_column(primary_key=True)
-    version_nbr: Mapped[int]
-    posting_date: Mapped[datetime.datetime | None]
-    archive_date: Mapped[datetime.datetime | None]
-    forecast_desc: Mapped[str | None]
-    oth_cat_fa_desc: Mapped[str | None]
-    cost_sharing: Mapped[str | None]
-    number_of_awards: Mapped[str | None]
-    est_funding: Mapped[str | None]
-    award_ceiling: Mapped[str | None]
-    award_floor: Mapped[str | None]
-    fd_link_url: Mapped[str | None]
-    fd_link_desc: Mapped[str | None]
-    ac_name: Mapped[str | None]
-    ac_phone: Mapped[str | None]
-    ac_email_addr: Mapped[str | None]
-    ac_email_desc: Mapped[str | None]
-    agency_code: Mapped[str | None]
-    sendmail: Mapped[str | None]
-    applicant_elig_desc: Mapped[str | None]
-    est_synopsis_posting_date: Mapped[datetime.datetime | None]
-    est_appl_response_date: Mapped[datetime.datetime | None]
-    est_appl_response_date_desc: Mapped[str | None]
-    est_award_date: Mapped[datetime.datetime | None]
-    est_project_start_date: Mapped[datetime.datetime | None]
-    fiscal_year: Mapped[int | None]
-    modification_comments: Mapped[str | None]
-    action_type: Mapped[str | None]
-    action_date: Mapped[datetime.datetime | None]
-    create_ts: Mapped[datetime.datetime]
-    created_date: Mapped[datetime.datetime | None]
-    last_upd_date: Mapped[datetime.datetime | None]
-    creator_id: Mapped[str | None]
-    last_upd_id: Mapped[str | None]
-    publisheruid: Mapped[str | None]
-    publisher_profile_id: Mapped[int | None]
-
-
-class TapplicanttypesForecast(foreignbase.ForeignBase):
-    __tablename__ = "tapplicanttypes_forecast"
 
-    at_frcst_id: Mapped[int] = mapped_column(primary_key=True)
-    at_id: Mapped[str] = mapped_column(primary_key=True)
-    opportunity_id: Mapped[int] = mapped_column(primary_key=True)
-    created_date: Mapped[datetime.datetime | None]
-    last_upd_date: Mapped[datetime.datetime | None]
-    creator_id: Mapped[str | None]
-    last_upd_id: Mapped[str | None]
+class TapplicanttypesForecast(foreignbase.ForeignBase, forecast_mixin.TapplicanttypesForecastMixin):
+    __tablename__ = "tapplicanttypes_forecast"
 
 
-class TapplicanttypesForecastHist(foreignbase.ForeignBase):
+class TapplicanttypesForecastHist(
+    foreignbase.ForeignBase, forecast_mixin.TapplicanttypesForecastHistMixin
+):
     __tablename__ = "tapplicanttypes_forecast_hist"
 
-    at_frcst_id: Mapped[int] = mapped_column(primary_key=True)
-    at_id: Mapped[str] = mapped_column(primary_key=True)
-    opportunity_id: Mapped[int] = mapped_column(primary_key=True)
-    revision_number: Mapped[int] = mapped_column(primary_key=True)
-    created_date: Mapped[datetime.datetime | None]
-    last_upd_date: Mapped[datetime.datetime | None]
-    creator_id: Mapped[str | None]
-    last_upd_id: Mapped[str | None]
-
 
-class TfundactcatForecast(foreignbase.ForeignBase):
+class TfundactcatForecast(foreignbase.ForeignBase, forecast_mixin.TfundactcatForecastMixin):
     __tablename__ = "tfundactcat_forecast"
 
-    fac_frcst_id: Mapped[int] = mapped_column(primary_key=True)
-    fac_id: Mapped[str] = mapped_column(primary_key=True)
-    opportunity_id: Mapped[int] = mapped_column(primary_key=True)
-    created_date: Mapped[datetime.datetime | None]
-    last_upd_date: Mapped[datetime.datetime | None]
-    creator_id: Mapped[str | None]
-    last_upd_id: Mapped[str | None]
 
-
-class TfundactcatForecastHist(foreignbase.ForeignBase):
+class TfundactcatForecastHist(foreignbase.ForeignBase, forecast_mixin.TfundactcatForecastHistMixin):
     __tablename__ = "tfundactcat_forecast_hist"
 
-    fac_frcst_id: Mapped[int] = mapped_column(primary_key=True)
-    fac_id: Mapped[str] = mapped_column(primary_key=True)
-    opportunity_id: Mapped[int] = mapped_column(primary_key=True)
-    revision_number: Mapped[int] = mapped_column(primary_key=True)
-    created_date: Mapped[datetime.datetime | None]
-    last_upd_date: Mapped[datetime.datetime | None]
-    creator_id: Mapped[str | None]
-    last_upd_id: Mapped[str | None]
-
 
-class TfundinstrForecast(foreignbase.ForeignBase):
+class TfundinstrForecast(foreignbase.ForeignBase, forecast_mixin.TfundinstrForecastMixin):
     __tablename__ = "tfundinstr_forecast"
 
-    fi_frcst_id: Mapped[int] = mapped_column(primary_key=True)
-    fi_id: Mapped[str] = mapped_column(primary_key=True)
-    opportunity_id: Mapped[int] = mapped_column(primary_key=True)
-    created_date: Mapped[datetime.datetime | None]
-    last_upd_date: Mapped[datetime.datetime | None]
-    creator_id: Mapped[str | None]
-    last_upd_id: Mapped[str | None]
 
-
-class TfundinstrForecastHist(foreignbase.ForeignBase):
+class TfundinstrForecastHist(foreignbase.ForeignBase, forecast_mixin.TfundinstrForecastHistMixin):
     __tablename__ = "tfundinstr_forecast_hist"
-
-    fi_frcst_id: Mapped[int] = mapped_column(primary_key=True)
-    fi_id: Mapped[str] = mapped_column(primary_key=True)
-    opportunity_id: Mapped[int] = mapped_column(primary_key=True)
-    revision_number: Mapped[int] = mapped_column(primary_key=True)
-    created_date: Mapped[datetime.datetime | None]
-    last_upd_date: Mapped[datetime.datetime | None]
-    creator_id: Mapped[str | None]
-    last_upd_id: Mapped[str | None]

api/src/db/foreign/opportunity.py

@@ -5,47 +5,14 @@
 # match by oracle_fdw, but we are matching them for maintainability.
 #
 
-import datetime
-
-from sqlalchemy.orm import Mapped, mapped_column
+from src.db.legacy_mixin import opportunity_mixin
 
 from . import foreignbase
 
 
-class Topportunity(foreignbase.ForeignBase):
+class Topportunity(foreignbase.ForeignBase, opportunity_mixin.TopportunityMixin):
     __tablename__ = "topportunity"
 
-    opportunity_id: Mapped[int] = mapped_column(primary_key=True)
-    oppnumber: Mapped[str | None]
-    revision_number: Mapped[int | None]
-    opptitle: Mapped[str | None]
-    owningagency: Mapped[str | None]
-    publisheruid: Mapped[str | None]
-    listed: Mapped[str | None]
-    oppcategory: Mapped[str | None]
-    initial_opportunity_id: Mapped[int | None]
-    modified_comments: Mapped[str | None]
-    created_date: Mapped[datetime.datetime]
-    last_upd_date: Mapped[datetime.datetime]
-    creator_id: Mapped[str | None]
-    last_upd_id: Mapped[str | None]
-    flag_2006: Mapped[str | None]
-    category_explanation: Mapped[str | None]
-    publisher_profile_id: Mapped[int | None]
-    is_draft: Mapped[str | None]
-
 
-class TopportunityCfda(foreignbase.ForeignBase):
+class TopportunityCfda(foreignbase.ForeignBase, opportunity_mixin.TopportunityCfdaMixin):
     __tablename__ = "topportunity_cfda"
-
-    opp_cfda_id: Mapped[int] = mapped_column(primary_key=True)
-    opportunity_id: Mapped[int]
-    cfdanumber: Mapped[str | None]
-    programtitle: Mapped[str | None]
-    origtoppid: Mapped[int | None]
-    oppidcfdanum: Mapped[str | None]
-    origoppnum: Mapped[str | None]
-    created_date: Mapped[datetime.datetime | None]
-    last_upd_date: Mapped[datetime.datetime | None]
-    creator_id: Mapped[str | None]
-    last_upd_id: Mapped[str | None]