NOI "In progress" status from OATS to ALCS (#1042)

introduce date_time helper
base status import for NOI
import NOI "In progress" status from OATS to ALCS

bin/migrate-oats-data/common/__init__.py

@@ -5,4 +5,5 @@
 from .oats_to_alcs_adjacent_land_use_type_enum import *
 from .json_encoder import *
 from .oats_to_alcs_soil_change_code_enum import *
+from .date_time_helper import *
 from .oats_to_alcs_naru_code_enum import *

bin/migrate-oats-data/common/date_time_helper.py

@@ -0,0 +1,31 @@
+from datetime import datetime
+import pytz
+
+
+def convert_timezone(date_str, timezone_str="America/Vancouver"):
+    # Convert the string to a datetime object
+    if isinstance(date_str, str):
+        dt = datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S")
+    else:  # assuming date_str is a datetime object
+        dt = date_str
+
+    # Adjust timezone
+    timezone = pytz.timezone(timezone_str)
+    dt = dt.replace(tzinfo=pytz.UTC)
+    dt_aware = dt.astimezone(timezone)
+
+    # Format the string with the desired output
+    formatted = dt_aware.strftime("%Y-%m-%d %H:%M:%S.%f %z")
+
+    return formatted
+
+
+def set_time(date_str, hour=0, minute=0, second=0):
+    # Replace the time of the given date
+    if isinstance(date_str, str):
+        dt = datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S.%f %z")
+    else:  # assuming date_str is a datetime object
+        dt = date_str
+
+    dt = dt.replace(hour=hour, minute=minute, second=second, microsecond=0)
+    return dt.strftime("%Y-%m-%d %H:%M:%S.%f %z")

bin/migrate-oats-data/noi/notice_of_intent_migration.py

@@ -9,6 +9,9 @@
     process_alcs_notice_of_intent_soil_fields,
     process_notice_of_intent_adjacent_land_use,
     process_notice_of_intent_empty_adjacent_land_use,
+    init_notice_of_intent_statuses,
+    process_alcs_notice_of_intent_in_progress_status,
+    clean_notice_of_intent_submission_statuses,
 )
 from .oats_to_alcs_notice_of_intent_table_etl.notice_of_intent_decision_date import (
     process_alcs_notice_of_intent_decision_date,
@@ -23,6 +26,7 @@ def init_notice_of_intent(batch_size):
 
 
 def clean_notice_of_intent():
+    clean_notice_of_intent_submission_statuses()
     clean_notice_of_intent_submissions()
     clean_notice_of_intents()
 
@@ -43,4 +47,9 @@ def process_notice_of_intent(batch_size):
 
     process_alcs_notice_of_intent_proposal_fields(batch_size)
 
+    init_notice_of_intent_statuses()
+
+    process_alcs_notice_of_intent_in_progress_status(batch_size)
+
+    # this script must be the last one
     process_notice_of_intent_submission_status_emails()

bin/migrate-oats-data/noi/notice_of_intent_submissions/__init__.py

@@ -10,4 +10,15 @@
 )
 from .notice_of_intent_soil_fields import process_alcs_notice_of_intent_soil_fields
 
-from .notice_of_intent_proposal_fields import process_alcs_notice_of_intent_proposal_fields
+from .notice_of_intent_proposal_fields import (
+    process_alcs_notice_of_intent_proposal_fields,
+)
+
+from .statuses.notice_of_intent_statuses_base_insert import (
+    init_notice_of_intent_statuses,
+    clean_notice_of_intent_submission_statuses,
+)
+
+from .statuses.notice_of_intent_status_in_progress import (
+    process_alcs_notice_of_intent_in_progress_status,
+)

bin/migrate-oats-data/noi/notice_of_intent_submissions/statuses/notice_of_intent_status_in_progress.py

@@ -0,0 +1,121 @@
+from common import BATCH_UPLOAD_SIZE, setup_and_get_logger, convert_timezone, set_time
+from db import inject_conn_pool
+from psycopg2.extras import RealDictCursor, execute_batch
+
+etl_name = "process_alcs_notice_of_intent_in_progress_status"
+logger = setup_and_get_logger(etl_name)
+
+
+@inject_conn_pool
+def process_alcs_notice_of_intent_in_progress_status(
+    conn=None, batch_size=BATCH_UPLOAD_SIZE
+):
+    """
+    This function is responsible for populating In Progress status of Notice of Intent in ALCS.
+
+    Args:
+    conn (psycopg2.extensions.connection): PostgreSQL database connection. Provided by the decorator.
+    batch_size (int): The number of items to process at once. Defaults to BATCH_UPLOAD_SIZE.
+    """
+
+    logger.info(f"Start {etl_name}")
+    with conn.cursor(cursor_factory=RealDictCursor) as cursor:
+        with open(
+            "noi/sql/notice_of_intent_submission/statuses/in_progress/notice_of_intent_status_in_progress_count.sql",
+            "r",
+            encoding="utf-8",
+        ) as sql_file:
+            count_query = sql_file.read()
+            cursor.execute(count_query)
+            count_total = dict(cursor.fetchone())["count"]
+        logger.info(f"Total Notice of Intents data to update: {count_total}")
+
+        failed_inserts = 0
+        successful_updates_count = 0
+        last_application_id = 0
+
+        with open(
+            "noi/sql/notice_of_intent_submission/statuses/in_progress/notice_of_intent_status_in_progress.sql",
+            "r",
+            encoding="utf-8",
+        ) as sql_file:
+            application_sql = sql_file.read()
+            while True:
+                cursor.execute(
+                    f"{application_sql} WHERE oats_in_prog.alr_application_id > {last_application_id} ORDER BY oats_in_prog.alr_application_id;"
+                )
+
+                rows = cursor.fetchmany(batch_size)
+
+                if not rows:
+                    break
+                try:
+                    records_to_be_updated_count = len(rows)
+
+                    _update_records(conn, batch_size, cursor, rows)
+
+                    successful_updates_count = (
+                        successful_updates_count + records_to_be_updated_count
+                    )
+                    last_application_id = dict(rows[-1])["alr_application_id"]
+
+                    logger.debug(
+                        f"retrieved/updated items count: {records_to_be_updated_count}; total successfully updated notice of intents so far {successful_updates_count}; last updated alr_application_id: {last_application_id}"
+                    )
+                except Exception as err:
+                    # this is NOT going to be caused by actual data update failure. This code is only executed when the code error appears or connection to DB is lost
+                    logger.exception()
+                    conn.rollback()
+                    failed_inserts = count_total - successful_updates_count
+                    last_application_id = last_application_id + 1
+
+    logger.info(
+        f"Finished {etl_name}: total amount of successful updates {successful_updates_count}, total failed updates {failed_inserts}"
+    )
+
+
+def _update_records(conn, batch_size, cursor, rows):
+    parsed_data_list = _prepare_oats_data(rows)
+
+    if len(parsed_data_list) > 0:
+        execute_batch(
+            cursor,
+            _update_query,
+            parsed_data_list,
+            page_size=batch_size,
+        )
+
+    conn.commit()
+
+
+_update_query = """
+                    UPDATE alcs.notice_of_intent_submission_to_submission_status 
+                            SET effective_date  = %(date)s
+                    WHERE submission_uuid = %(uuid)s and status_type_code = 'PROG'
+"""
+
+
+def _prepare_oats_data(row_data_list):
+    data_list = []
+    for row in row_data_list:
+        data = map_fields(dict(row))
+        data_list.append(data)
+    return data_list
+
+
+def map_fields(data):
+    status_effective_date = None
+
+    if data:
+        if data["completion_date"]:
+            status_effective_date = data["completion_date"]
+        elif data["created_date"]:
+            status_effective_date = data["created_date"]
+        elif data["submitted_to_alc_date"]:
+            status_effective_date = data["submitted_to_alc_date"]
+
+    if status_effective_date:
+        date = convert_timezone(status_effective_date, "US/Pacific")
+        data["date"] = set_time(date)
+
+    return data

bin/migrate-oats-data/noi/notice_of_intent_submissions/statuses/notice_of_intent_statuses_base_insert.py

@@ -0,0 +1,69 @@
+from common import OATS_ETL_USER, setup_and_get_logger
+from db import inject_conn_pool
+from psycopg2.extras import RealDictCursor
+
+etl_name = "init_notice_of_intent_statuses"
+logger = setup_and_get_logger(etl_name)
+
+
+@inject_conn_pool
+def init_notice_of_intent_statuses(conn=None):
+    """
+    This function is responsible for initializing notice of intent statuses.
+    Initializing means inserting status_to_submission record without the effective_date.
+
+    Args:
+    conn (psycopg2.extensions.connection): PostgreSQL database connection. Provided by the decorator.
+    batch_size (int): The number of items to process at once. Defaults to BATCH_UPLOAD_SIZE.
+    """
+
+    logger.info(f"Start {etl_name}")
+    with conn.cursor(cursor_factory=RealDictCursor) as cursor:
+        with open(
+            "noi/sql/notice_of_intent_submission/statuses/init_notice_of_intent_submission_statuses_count.sql",
+            "r",
+            encoding="utf-8",
+        ) as sql_file:
+            count_query = sql_file.read()
+            cursor.execute(count_query)
+            count_total = dict(cursor.fetchone())["count"]
+        logger.info(f"Total Notice of Intents data to insert: {count_total}")
+
+        failed_inserts = 0
+        successful_inserts_count = 0
+        last_application_id = 0
+
+        with open(
+            "noi/sql/notice_of_intent_submission/statuses/init_notice_of_intent_submission_statuses.sql",
+            "r",
+            encoding="utf-8",
+        ) as sql_file:
+            query = sql_file.read()
+
+            try:
+                cursor.execute(query)
+                conn.commit()
+                successful_inserts_count = cursor.rowcount
+            except Exception as err:
+                logger.exception()
+                conn.rollback()
+                failed_inserts = count_total - successful_inserts_count
+                last_application_id = last_application_id + 1
+
+    logger.info(
+        f"Finished {etl_name}: total amount of successful inserts {successful_inserts_count}, total failed updates {failed_inserts}"
+    )
+
+
+@inject_conn_pool
+def clean_notice_of_intent_submission_statuses(conn=None):
+    logger.info("Start init_notice_of_intent_statuses cleaning")
+    with conn.cursor() as cursor:
+        cursor.execute(
+            f"""DELETE FROM alcs.notice_of_intent_submission_to_submission_status noi_st 
+                USING alcs.notice_of_intent_submission nois
+                WHERE noi_st.submission_uuid = nois.uuid AND nois.audit_created_by = '{OATS_ETL_USER}';"""
+        )
+        logger.info(f"Deleted items count = {cursor.rowcount}")
+
+    conn.commit()

bin/migrate-oats-data/noi/sql/notice_of_intent_submission/statuses/in_progress/notice_of_intent_status_in_progress.sql

@@ -0,0 +1,33 @@
+WITH latest_in_progress_accomplishment_per_file_number AS (
+    SELECT DISTINCT ON (alr_application_id, accomplishment_code) *
+    FROM oats.oats_accomplishments oa
+    WHERE accomplishment_code = 'INP'
+    ORDER BY alr_application_id,
+        accomplishment_code,
+        completion_date DESC
+),
+latest_in_progress_accomplishments_for_noi_only AS (
+    SELECT DISTINCT ON (
+            latest_in_prog.accomplishment_code,
+            latest_in_prog.completion_date,
+            oaa.alr_application_id,
+            oaa.created_date,
+            oaa.submitted_to_alc_date
+        ) latest_in_prog.accomplishment_code,
+        latest_in_prog.completion_date,
+        oaa.alr_application_id,
+        oaa.created_date,
+        oaa.submitted_to_alc_date
+    FROM oats.oats_alr_applications oaa
+        LEFT JOIN latest_in_progress_accomplishment_per_file_number AS latest_in_prog ON latest_in_prog.alr_application_id = oaa.alr_application_id
+    WHERE oaa.application_class_code = 'NOI'
+)
+SELECT DISTINCT ON (oats_in_prog.alr_application_id) oats_in_prog.alr_application_id,
+    oats_in_prog.accomplishment_code,
+    oats_in_prog.completion_date,
+    oats_in_prog.created_date,
+    oats_in_prog.submitted_to_alc_date,
+    nois.uuid
+FROM alcs.notice_of_intent_submission_to_submission_status noistss
+    JOIN alcs.notice_of_intent_submission nois ON nois.uuid = noistss.submission_uuid
+    JOIN latest_in_progress_accomplishments_for_noi_only AS oats_in_prog ON oats_in_prog.alr_application_id::TEXT = nois.file_number

bin/migrate-oats-data/noi/sql/notice_of_intent_submission/statuses/in_progress/notice_of_intent_status_in_progress_count.sql

@@ -0,0 +1,30 @@
+WITH latest_in_progress_accomplishment_per_file_number AS (
+    SELECT DISTINCT ON (alr_application_id, accomplishment_code) *
+    FROM oats.oats_accomplishments oa
+    WHERE accomplishment_code = 'INP'
+),
+latest_in_progress_accomplishments_for_noi_only AS (
+    SELECT DISTINCT ON (
+            latest_in_prog.accomplishment_code,
+            latest_in_prog.completion_date,
+            oaa.alr_application_id,
+            oaa.created_date,
+            oaa.submitted_to_alc_date
+        ) latest_in_prog.accomplishment_code,
+        latest_in_prog.completion_date,
+        oaa.alr_application_id,
+        oaa.created_date,
+        oaa.submitted_to_alc_date
+    FROM oats.oats_alr_applications oaa
+        LEFT JOIN latest_in_progress_accomplishment_per_file_number AS latest_in_prog ON latest_in_prog.alr_application_id = oaa.alr_application_id
+    WHERE oaa.application_class_code = 'NOI'
+),
+submission_statuses_to_update AS (
+    SELECT count(*)
+    FROM alcs.notice_of_intent_submission_to_submission_status noistss
+        JOIN alcs.notice_of_intent_submission nois ON nois.uuid = noistss.submission_uuid
+        JOIN latest_in_progress_accomplishments_for_noi_only AS oats_in_prog ON oats_in_prog.alr_application_id::TEXT = nois.file_number
+    GROUP BY noistss.submission_uuid
+)
+SELECT count(*)
+FROM submission_statuses_to_update

bin/migrate-oats-data/noi/sql/notice_of_intent_submission/statuses/init_notice_of_intent_submission_statuses.sql

@@ -0,0 +1,30 @@
+INSERT INTO alcs.notice_of_intent_submission_to_submission_status(submission_uuid, status_type_code) -- retrieve Notice of Intents from OATS that have only 1 proposal component
+    WITH noi_components_grouped AS (
+        SELECT oaac.alr_application_id
+        FROM oats.oats_alr_appl_components oaac
+            JOIN oats.oats_alr_applications oaa ON oaa.alr_application_id = oaac.alr_application_id
+        WHERE oaa.application_class_code = 'NOI'
+        GROUP BY oaac.alr_application_id
+        HAVING count(oaac.alr_application_id) < 2 -- ignore notice of intents with multiple components
+    ),
+    -- alcs_submissions_with_statuses is required for development environment only. Production environment does not have submissions.
+    alcs_submissions_with_statuses AS (
+        SELECT nois.file_number
+        FROM alcs.notice_of_intent_submission nois
+            JOIN alcs.notice_of_intent_submission_to_submission_status noistss ON nois.uuid = noistss.submission_uuid
+        GROUP BY nois.file_number
+    ),
+    -- retrieve submission_uuid from ALCS that were imported with ETL
+    alcs_submission_uuids_to_populate AS (
+        SELECT oaa.alr_application_id,
+            nois.uuid
+        FROM noi_components_grouped
+            JOIN oats.oats_alr_applications oaa ON noi_components_grouped.alr_application_id = oaa.alr_application_id
+            JOIN alcs.notice_of_intent_submission nois ON oaa.alr_application_id::TEXT = nois.file_number -- make sure TO WORK ONLY with the ones that were imported TO ALCS
+            LEFT JOIN alcs_submissions_with_statuses ON alcs_submissions_with_statuses.file_number = nois.file_number
+        WHERE alcs_submissions_with_statuses.file_number IS NULL -- filter out all submissions that have statuses populated before the ETL;
+    )
+SELECT uuid,
+    noisst.code
+FROM alcs_submission_uuids_to_populate
+    CROSS JOIN alcs.notice_of_intent_submission_status_type noisst