diff --git a/bin/migrate-oats-data/common/__init__.py b/bin/migrate-oats-data/common/__init__.py index 5ecfec974a..12452975fe 100644 --- a/bin/migrate-oats-data/common/__init__.py +++ b/bin/migrate-oats-data/common/__init__.py @@ -5,4 +5,5 @@ from .oats_to_alcs_adjacent_land_use_type_enum import * from .json_encoder import * from .oats_to_alcs_soil_change_code_enum import * +from .date_time_helper import * from .oats_to_alcs_naru_code_enum import * diff --git a/bin/migrate-oats-data/common/date_time_helper.py b/bin/migrate-oats-data/common/date_time_helper.py new file mode 100644 index 0000000000..b7106444e3 --- /dev/null +++ b/bin/migrate-oats-data/common/date_time_helper.py @@ -0,0 +1,31 @@ +from datetime import datetime +import pytz + + +def convert_timezone(date_str, timezone_str="America/Vancouver"): + # Convert the string to a datetime object + if isinstance(date_str, str): + dt = datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S") + else: # assuming date_str is a datetime object + dt = date_str + + # Adjust timezone + timezone = pytz.timezone(timezone_str) + dt = dt.replace(tzinfo=pytz.UTC) + dt_aware = dt.astimezone(timezone) + + # Format the string with the desired output + formatted = dt_aware.strftime("%Y-%m-%d %H:%M:%S.%f %z") + + return formatted + + +def set_time(date_str, hour=0, minute=0, second=0): + # Replace the time of the given date + if isinstance(date_str, str): + dt = datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S.%f %z") + else: # assuming date_str is a datetime object + dt = date_str + + dt = dt.replace(hour=hour, minute=minute, second=second, microsecond=0) + return dt.strftime("%Y-%m-%d %H:%M:%S.%f %z") diff --git a/bin/migrate-oats-data/noi/notice_of_intent_migration.py b/bin/migrate-oats-data/noi/notice_of_intent_migration.py index 49ec3d712c..baa07aa05c 100644 --- a/bin/migrate-oats-data/noi/notice_of_intent_migration.py +++ b/bin/migrate-oats-data/noi/notice_of_intent_migration.py @@ -9,6 +9,9 @@ process_alcs_notice_of_intent_soil_fields, process_notice_of_intent_adjacent_land_use, process_notice_of_intent_empty_adjacent_land_use, + init_notice_of_intent_statuses, + process_alcs_notice_of_intent_in_progress_status, + clean_notice_of_intent_submission_statuses, ) from .oats_to_alcs_notice_of_intent_table_etl.notice_of_intent_decision_date import ( process_alcs_notice_of_intent_decision_date, @@ -23,6 +26,7 @@ def init_notice_of_intent(batch_size): def clean_notice_of_intent(): + clean_notice_of_intent_submission_statuses() clean_notice_of_intent_submissions() clean_notice_of_intents() @@ -43,4 +47,9 @@ def process_notice_of_intent(batch_size): process_alcs_notice_of_intent_proposal_fields(batch_size) + init_notice_of_intent_statuses() + + process_alcs_notice_of_intent_in_progress_status(batch_size) + + # this script must be the last one process_notice_of_intent_submission_status_emails() diff --git a/bin/migrate-oats-data/noi/notice_of_intent_submissions/__init__.py b/bin/migrate-oats-data/noi/notice_of_intent_submissions/__init__.py index d6591c20bc..e0fa1aca89 100644 --- a/bin/migrate-oats-data/noi/notice_of_intent_submissions/__init__.py +++ b/bin/migrate-oats-data/noi/notice_of_intent_submissions/__init__.py @@ -10,4 +10,15 @@ ) from .notice_of_intent_soil_fields import process_alcs_notice_of_intent_soil_fields -from .notice_of_intent_proposal_fields import process_alcs_notice_of_intent_proposal_fields +from .notice_of_intent_proposal_fields import ( + process_alcs_notice_of_intent_proposal_fields, +) + +from .statuses.notice_of_intent_statuses_base_insert import ( + init_notice_of_intent_statuses, + clean_notice_of_intent_submission_statuses, +) + +from .statuses.notice_of_intent_status_in_progress import ( + process_alcs_notice_of_intent_in_progress_status, +) diff --git a/bin/migrate-oats-data/noi/notice_of_intent_submissions/statuses/notice_of_intent_status_in_progress.py b/bin/migrate-oats-data/noi/notice_of_intent_submissions/statuses/notice_of_intent_status_in_progress.py new file mode 100644 index 0000000000..12c8dff273 --- /dev/null +++ b/bin/migrate-oats-data/noi/notice_of_intent_submissions/statuses/notice_of_intent_status_in_progress.py @@ -0,0 +1,121 @@ +from common import BATCH_UPLOAD_SIZE, setup_and_get_logger, convert_timezone, set_time +from db import inject_conn_pool +from psycopg2.extras import RealDictCursor, execute_batch + +etl_name = "process_alcs_notice_of_intent_in_progress_status" +logger = setup_and_get_logger(etl_name) + + +@inject_conn_pool +def process_alcs_notice_of_intent_in_progress_status( + conn=None, batch_size=BATCH_UPLOAD_SIZE +): + """ + This function is responsible for populating In Progress status of Notice of Intent in ALCS. + + Args: + conn (psycopg2.extensions.connection): PostgreSQL database connection. Provided by the decorator. + batch_size (int): The number of items to process at once. Defaults to BATCH_UPLOAD_SIZE. + """ + + logger.info(f"Start {etl_name}") + with conn.cursor(cursor_factory=RealDictCursor) as cursor: + with open( + "noi/sql/notice_of_intent_submission/statuses/in_progress/notice_of_intent_status_in_progress_count.sql", + "r", + encoding="utf-8", + ) as sql_file: + count_query = sql_file.read() + cursor.execute(count_query) + count_total = dict(cursor.fetchone())["count"] + logger.info(f"Total Notice of Intents data to update: {count_total}") + + failed_inserts = 0 + successful_updates_count = 0 + last_application_id = 0 + + with open( + "noi/sql/notice_of_intent_submission/statuses/in_progress/notice_of_intent_status_in_progress.sql", + "r", + encoding="utf-8", + ) as sql_file: + application_sql = sql_file.read() + while True: + cursor.execute( + f"{application_sql} WHERE oats_in_prog.alr_application_id > {last_application_id} ORDER BY oats_in_prog.alr_application_id;" + ) + + rows = cursor.fetchmany(batch_size) + + if not rows: + break + try: + records_to_be_updated_count = len(rows) + + _update_records(conn, batch_size, cursor, rows) + + successful_updates_count = ( + successful_updates_count + records_to_be_updated_count + ) + last_application_id = dict(rows[-1])["alr_application_id"] + + logger.debug( + f"retrieved/updated items count: {records_to_be_updated_count}; total successfully updated notice of intents so far {successful_updates_count}; last updated alr_application_id: {last_application_id}" + ) + except Exception as err: + # this is NOT going to be caused by actual data update failure. This code is only executed when the code error appears or connection to DB is lost + logger.exception() + conn.rollback() + failed_inserts = count_total - successful_updates_count + last_application_id = last_application_id + 1 + + logger.info( + f"Finished {etl_name}: total amount of successful updates {successful_updates_count}, total failed updates {failed_inserts}" + ) + + +def _update_records(conn, batch_size, cursor, rows): + parsed_data_list = _prepare_oats_data(rows) + + if len(parsed_data_list) > 0: + execute_batch( + cursor, + _update_query, + parsed_data_list, + page_size=batch_size, + ) + + conn.commit() + + +_update_query = """ + UPDATE alcs.notice_of_intent_submission_to_submission_status + SET effective_date = %(date)s + WHERE submission_uuid = %(uuid)s and status_type_code = 'PROG' +""" + + +def _prepare_oats_data(row_data_list): + data_list = [] + for row in row_data_list: + data = map_fields(dict(row)) + data_list.append(data) + return data_list + + +def map_fields(data): + status_effective_date = None + + if data: + if data["completion_date"]: + status_effective_date = data["completion_date"] + elif data["created_date"]: + status_effective_date = data["created_date"] + elif data["submitted_to_alc_date"]: + status_effective_date = data["submitted_to_alc_date"] + + if status_effective_date: + date = convert_timezone(status_effective_date, "US/Pacific") + data["date"] = set_time(date) + + return data diff --git a/bin/migrate-oats-data/noi/notice_of_intent_submissions/statuses/notice_of_intent_statuses_base_insert.py b/bin/migrate-oats-data/noi/notice_of_intent_submissions/statuses/notice_of_intent_statuses_base_insert.py new file mode 100644 index 0000000000..489d97d994 --- /dev/null +++ b/bin/migrate-oats-data/noi/notice_of_intent_submissions/statuses/notice_of_intent_statuses_base_insert.py @@ -0,0 +1,69 @@ +from common import OATS_ETL_USER, setup_and_get_logger +from db import inject_conn_pool +from psycopg2.extras import RealDictCursor + +etl_name = "init_notice_of_intent_statuses" +logger = setup_and_get_logger(etl_name) + + +@inject_conn_pool +def init_notice_of_intent_statuses(conn=None): + """ + This function is responsible for initializing notice of intent statuses. + Initializing means inserting status_to_submission record without the effective_date. + + Args: + conn (psycopg2.extensions.connection): PostgreSQL database connection. Provided by the decorator. + batch_size (int): The number of items to process at once. Defaults to BATCH_UPLOAD_SIZE. + """ + + logger.info(f"Start {etl_name}") + with conn.cursor(cursor_factory=RealDictCursor) as cursor: + with open( + "noi/sql/notice_of_intent_submission/statuses/init_notice_of_intent_submission_statuses_count.sql", + "r", + encoding="utf-8", + ) as sql_file: + count_query = sql_file.read() + cursor.execute(count_query) + count_total = dict(cursor.fetchone())["count"] + logger.info(f"Total Notice of Intents data to insert: {count_total}") + + failed_inserts = 0 + successful_inserts_count = 0 + last_application_id = 0 + + with open( + "noi/sql/notice_of_intent_submission/statuses/init_notice_of_intent_submission_statuses.sql", + "r", + encoding="utf-8", + ) as sql_file: + query = sql_file.read() + + try: + cursor.execute(query) + conn.commit() + successful_inserts_count = cursor.rowcount + except Exception as err: + logger.exception() + conn.rollback() + failed_inserts = count_total - successful_inserts_count + last_application_id = last_application_id + 1 + + logger.info( + f"Finished {etl_name}: total amount of successful inserts {successful_inserts_count}, total failed updates {failed_inserts}" + ) + + +@inject_conn_pool +def clean_notice_of_intent_submission_statuses(conn=None): + logger.info("Start init_notice_of_intent_statuses cleaning") + with conn.cursor() as cursor: + cursor.execute( + f"""DELETE FROM alcs.notice_of_intent_submission_to_submission_status noi_st + USING alcs.notice_of_intent_submission nois + WHERE noi_st.submission_uuid = nois.uuid AND nois.audit_created_by = '{OATS_ETL_USER}';""" + ) + logger.info(f"Deleted items count = {cursor.rowcount}") + + conn.commit() diff --git a/bin/migrate-oats-data/noi/sql/notice_of_intent_submission/statuses/in_progress/notice_of_intent_status_in_progress.sql b/bin/migrate-oats-data/noi/sql/notice_of_intent_submission/statuses/in_progress/notice_of_intent_status_in_progress.sql new file mode 100644 index 0000000000..53581a480e --- /dev/null +++ b/bin/migrate-oats-data/noi/sql/notice_of_intent_submission/statuses/in_progress/notice_of_intent_status_in_progress.sql @@ -0,0 +1,33 @@ +WITH latest_in_progress_accomplishment_per_file_number AS ( + SELECT DISTINCT ON (alr_application_id, accomplishment_code) * + FROM oats.oats_accomplishments oa + WHERE accomplishment_code = 'INP' + ORDER BY alr_application_id, + accomplishment_code, + completion_date DESC +), +latest_in_progress_accomplishments_for_noi_only AS ( + SELECT DISTINCT ON ( + latest_in_prog.accomplishment_code, + latest_in_prog.completion_date, + oaa.alr_application_id, + oaa.created_date, + oaa.submitted_to_alc_date + ) latest_in_prog.accomplishment_code, + latest_in_prog.completion_date, + oaa.alr_application_id, + oaa.created_date, + oaa.submitted_to_alc_date + FROM oats.oats_alr_applications oaa + LEFT JOIN latest_in_progress_accomplishment_per_file_number AS latest_in_prog ON latest_in_prog.alr_application_id = oaa.alr_application_id + WHERE oaa.application_class_code = 'NOI' +) +SELECT DISTINCT ON (oats_in_prog.alr_application_id) oats_in_prog.alr_application_id, + oats_in_prog.accomplishment_code, + oats_in_prog.completion_date, + oats_in_prog.created_date, + oats_in_prog.submitted_to_alc_date, + nois.uuid +FROM alcs.notice_of_intent_submission_to_submission_status noistss + JOIN alcs.notice_of_intent_submission nois ON nois.uuid = noistss.submission_uuid + JOIN latest_in_progress_accomplishments_for_noi_only AS oats_in_prog ON oats_in_prog.alr_application_id::TEXT = nois.file_number \ No newline at end of file diff --git a/bin/migrate-oats-data/noi/sql/notice_of_intent_submission/statuses/in_progress/notice_of_intent_status_in_progress_count.sql b/bin/migrate-oats-data/noi/sql/notice_of_intent_submission/statuses/in_progress/notice_of_intent_status_in_progress_count.sql new file mode 100644 index 0000000000..dd3aea55a2 --- /dev/null +++ b/bin/migrate-oats-data/noi/sql/notice_of_intent_submission/statuses/in_progress/notice_of_intent_status_in_progress_count.sql @@ -0,0 +1,30 @@ +WITH latest_in_progress_accomplishment_per_file_number AS ( + SELECT DISTINCT ON (alr_application_id, accomplishment_code) * + FROM oats.oats_accomplishments oa + WHERE accomplishment_code = 'INP' +), +latest_in_progress_accomplishments_for_noi_only AS ( + SELECT DISTINCT ON ( + latest_in_prog.accomplishment_code, + latest_in_prog.completion_date, + oaa.alr_application_id, + oaa.created_date, + oaa.submitted_to_alc_date + ) latest_in_prog.accomplishment_code, + latest_in_prog.completion_date, + oaa.alr_application_id, + oaa.created_date, + oaa.submitted_to_alc_date + FROM oats.oats_alr_applications oaa + LEFT JOIN latest_in_progress_accomplishment_per_file_number AS latest_in_prog ON latest_in_prog.alr_application_id = oaa.alr_application_id + WHERE oaa.application_class_code = 'NOI' +), +submission_statuses_to_update AS ( + SELECT count(*) + FROM alcs.notice_of_intent_submission_to_submission_status noistss + JOIN alcs.notice_of_intent_submission nois ON nois.uuid = noistss.submission_uuid + JOIN latest_in_progress_accomplishments_for_noi_only AS oats_in_prog ON oats_in_prog.alr_application_id::TEXT = nois.file_number + GROUP BY noistss.submission_uuid +) +SELECT count(*) +FROM submission_statuses_to_update \ No newline at end of file diff --git a/bin/migrate-oats-data/noi/sql/notice_of_intent_submission/statuses/init_notice_of_intent_submission_statuses.sql b/bin/migrate-oats-data/noi/sql/notice_of_intent_submission/statuses/init_notice_of_intent_submission_statuses.sql new file mode 100644 index 0000000000..6eebc25f2a --- /dev/null +++ b/bin/migrate-oats-data/noi/sql/notice_of_intent_submission/statuses/init_notice_of_intent_submission_statuses.sql @@ -0,0 +1,30 @@ +INSERT INTO alcs.notice_of_intent_submission_to_submission_status(submission_uuid, status_type_code) -- retrieve Notice of Intents from OATS that have only 1 proposal component + WITH noi_components_grouped AS ( + SELECT oaac.alr_application_id + FROM oats.oats_alr_appl_components oaac + JOIN oats.oats_alr_applications oaa ON oaa.alr_application_id = oaac.alr_application_id + WHERE oaa.application_class_code = 'NOI' + GROUP BY oaac.alr_application_id + HAVING count(oaac.alr_application_id) < 2 -- ignore notice of intents with multiple components + ), + -- alcs_submissions_with_statuses is required for development environment only. Production environment does not have submissions. + alcs_submissions_with_statuses AS ( + SELECT nois.file_number + FROM alcs.notice_of_intent_submission nois + JOIN alcs.notice_of_intent_submission_to_submission_status noistss ON nois.uuid = noistss.submission_uuid + GROUP BY nois.file_number + ), + -- retrieve submission_uuid from ALCS that were imported with ETL + alcs_submission_uuids_to_populate AS ( + SELECT oaa.alr_application_id, + nois.uuid + FROM noi_components_grouped + JOIN oats.oats_alr_applications oaa ON noi_components_grouped.alr_application_id = oaa.alr_application_id + JOIN alcs.notice_of_intent_submission nois ON oaa.alr_application_id::TEXT = nois.file_number -- make sure TO WORK ONLY with the ones that were imported TO ALCS + LEFT JOIN alcs_submissions_with_statuses ON alcs_submissions_with_statuses.file_number = nois.file_number + WHERE alcs_submissions_with_statuses.file_number IS NULL -- filter out all submissions that have statuses populated before the ETL; + ) +SELECT uuid, + noisst.code +FROM alcs_submission_uuids_to_populate + CROSS JOIN alcs.notice_of_intent_submission_status_type noisst \ No newline at end of file diff --git a/bin/migrate-oats-data/noi/sql/notice_of_intent_submission/statuses/init_notice_of_intent_submission_statuses_count.sql b/bin/migrate-oats-data/noi/sql/notice_of_intent_submission/statuses/init_notice_of_intent_submission_statuses_count.sql new file mode 100644 index 0000000000..7536ae4157 --- /dev/null +++ b/bin/migrate-oats-data/noi/sql/notice_of_intent_submission/statuses/init_notice_of_intent_submission_statuses_count.sql @@ -0,0 +1,27 @@ +WITH noi_components_grouped AS ( + SELECT oaac.alr_application_id + FROM oats.oats_alr_appl_components oaac + JOIN oats.oats_alr_applications oaa ON oaa.alr_application_id = oaac.alr_application_id + WHERE oaa.application_class_code = 'NOI' + GROUP BY oaac.alr_application_id + HAVING count(oaac.alr_application_id) < 2 -- ignore notice of intents with multiple components +), +alcs_submissions_with_statuses AS ( + SELECT nois.file_number + FROM alcs.notice_of_intent_submission nois + JOIN alcs.notice_of_intent_submission_to_submission_status noistss ON nois.uuid = noistss.submission_uuid + GROUP BY nois.file_number +), +-- alcs_submissions_with_statuses is required for developement environment only. Production environment does not have submissions. +alcs_submission_uuids_to_populate AS ( + SELECT oaa.alr_application_id, + nois.uuid + FROM noi_components_grouped + JOIN oats.oats_alr_applications oaa ON noi_components_grouped.alr_application_id = oaa.alr_application_id + JOIN alcs.notice_of_intent_submission nois ON oaa.alr_application_id::TEXT = nois.file_number -- make sure TO WORK ONLY with the ones that were imported TO ALCS + LEFT JOIN alcs_submissions_with_statuses ON alcs_submissions_with_statuses.file_number = nois.file_number + WHERE alcs_submissions_with_statuses.file_number IS NULL -- filter out all submissons that have statuses populated before the ETL; +) +SELECT count(*) +FROM alcs_submission_uuids_to_populate + CROSS JOIN alcs.notice_of_intent_submission_status_type noisst \ No newline at end of file diff --git a/bin/migrate-oats-data/requirements.txt b/bin/migrate-oats-data/requirements.txt index 8ccb3d4a10..6749ce4202 100644 --- a/bin/migrate-oats-data/requirements.txt +++ b/bin/migrate-oats-data/requirements.txt @@ -2,4 +2,5 @@ psycopg2==2.9.5 tqdm==4.64.1 python-dotenv==0.21.0 pyyaml==6.0 -rich==13.3.1 \ No newline at end of file +rich==13.3.1 +pytz==2023.3 \ No newline at end of file