From f4e68d790318b73782681408b92862301af35fe4 Mon Sep 17 00:00:00 2001 From: Beverly Nguyen Date: Thu, 19 Dec 2024 17:53:04 -0800 Subject: [PATCH 01/24] add pending stats --- app/enums.py | 1 + app/service/statistics.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/app/enums.py b/app/enums.py index a0dfbb467..37b3b6892 100644 --- a/app/enums.py +++ b/app/enums.py @@ -211,3 +211,4 @@ class StatisticsType(StrEnum): REQUESTED = "requested" DELIVERED = "delivered" FAILURE = "failure" + PENDING = "pending" diff --git a/app/service/statistics.py b/app/service/statistics.py index a6b58e067..042927c3f 100644 --- a/app/service/statistics.py +++ b/app/service/statistics.py @@ -96,6 +96,8 @@ def _update_statuses_from_row(update_dict, row): NotificationStatus.VIRUS_SCAN_FAILED, ): update_dict[StatisticsType.FAILURE] += row.count + elif row.status == NotificationStatus.PENDING: + update_dict[StatisticsType.PENDING] += row.count def create_empty_monthly_notification_status_stats_dict(year): From 67a56626a4ecf6d412883c93460da895b5682c7e Mon Sep 17 00:00:00 2001 From: Beverly Nguyen Date: Mon, 30 Dec 2024 20:18:56 -0800 Subject: [PATCH 02/24] update pending count --- app/service/statistics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/service/statistics.py b/app/service/statistics.py index 042927c3f..12814b970 100644 --- a/app/service/statistics.py +++ b/app/service/statistics.py @@ -96,7 +96,7 @@ def _update_statuses_from_row(update_dict, row): NotificationStatus.VIRUS_SCAN_FAILED, ): update_dict[StatisticsType.FAILURE] += row.count - elif row.status == NotificationStatus.PENDING: + elif row.status in (NotificationStatus.PENDING, NotificationStatus.CREATED, NotificationStatus.SENDING): update_dict[StatisticsType.PENDING] += row.count From 05055aa60a986c157cf980de9afdca095ebea1b3 Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Wed, 8 Jan 2025 15:41:26 -0500 Subject: [PATCH 03/24] Updated query Signed-off-by: Cliff Hill --- app/dao/services_dao.py | 21 +++++++++++++++++++-- app/service/statistics.py | 6 +++++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 260008193..b81e54bc2 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -455,24 +455,41 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): start_date = get_midnight_in_utc(start_date) end_date = get_midnight_in_utc(end_date + timedelta(days=1)) - stmt = ( + sub_stmt = ( select( + Job.id, + Job.notification_count, NotificationAllTimeView.notification_type, NotificationAllTimeView.status, func.date_trunc("day", NotificationAllTimeView.created_at).label("day"), func.count(NotificationAllTimeView.id).label("count"), ) - .filter( + .join_from( + Notification, + Job, + ) + .where( NotificationAllTimeView.service_id == service_id, NotificationAllTimeView.key_type != KeyType.TEST, NotificationAllTimeView.created_at >= start_date, NotificationAllTimeView.created_at < end_date, ) .group_by( + Job.id, + Job.notification_count, NotificationAllTimeView.notification_type, NotificationAllTimeView.status, func.date_trunc("day", NotificationAllTimeView.created_at), ) + .subquery() + ) + + stmt = select( + func.sum(sub_stmt.notification_count).label("total_notifications"), + sub_stmt.notification_type, + sub_stmt.status, + sub_stmt.day, + func.sum(sub_stmt.count).label("count"), ) return db.session.execute(stmt).all() diff --git a/app/service/statistics.py b/app/service/statistics.py index 12814b970..8bd41da25 100644 --- a/app/service/statistics.py +++ b/app/service/statistics.py @@ -96,7 +96,11 @@ def _update_statuses_from_row(update_dict, row): NotificationStatus.VIRUS_SCAN_FAILED, ): update_dict[StatisticsType.FAILURE] += row.count - elif row.status in (NotificationStatus.PENDING, NotificationStatus.CREATED, NotificationStatus.SENDING): + elif row.status in ( + NotificationStatus.PENDING, + NotificationStatus.CREATED, + NotificationStatus.SENDING, + ): update_dict[StatisticsType.PENDING] += row.count From 37fa1187acdae5838a8bf18f115511a0a692f4f0 Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Thu, 9 Jan 2025 08:29:36 -0500 Subject: [PATCH 04/24] Adding group by clause to outer query for function. Signed-off-by: Cliff Hill --- app/dao/services_dao.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index b81e54bc2..099cb0da1 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -484,12 +484,19 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): .subquery() ) - stmt = select( - func.sum(sub_stmt.notification_count).label("total_notifications"), - sub_stmt.notification_type, - sub_stmt.status, - sub_stmt.day, - func.sum(sub_stmt.count).label("count"), + stmt = ( + select( + func.sum(sub_stmt.notification_count).label("total_notifications"), + sub_stmt.notification_type, + sub_stmt.status, + sub_stmt.day, + func.sum(sub_stmt.count).label("count"), + ) + .group_by( + sub_stmt.notification_type, + sub_stmt.status, + sub_stmt.day, + ) ) return db.session.execute(stmt).all() From 8656c44757855d3f2943739384ee0f7b5dd09850 Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Thu, 9 Jan 2025 13:02:21 -0500 Subject: [PATCH 05/24] Make sure join is correct table. Signed-off-by: Cliff Hill --- app/dao/services_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 099cb0da1..2aa810504 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -465,7 +465,7 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): func.count(NotificationAllTimeView.id).label("count"), ) .join_from( - Notification, + NotificationAllTimeView, Job, ) .where( From b6337ad8074d4764ac77cea612d8800c4df49326 Mon Sep 17 00:00:00 2001 From: Beverly Nguyen Date: Fri, 10 Jan 2025 10:28:28 -0800 Subject: [PATCH 06/24] updating pending count --- app/dao/services_dao.py | 30 ++++++++++++++---------------- app/service/statistics.py | 21 ++++++++++++++------- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 2aa810504..841434a62 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -2,7 +2,7 @@ from datetime import timedelta from flask import current_app -from sqlalchemy import Float, cast, delete, select +from sqlalchemy import Float, cast, delete, select, Integer from sqlalchemy.orm import joinedload from sqlalchemy.sql.expression import and_, asc, case, func @@ -458,16 +458,13 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): sub_stmt = ( select( Job.id, - Job.notification_count, + cast(Job.notification_count, Integer).label("notification_count"), # <-- i added cast here NotificationAllTimeView.notification_type, NotificationAllTimeView.status, func.date_trunc("day", NotificationAllTimeView.created_at).label("day"), - func.count(NotificationAllTimeView.id).label("count"), - ) - .join_from( - NotificationAllTimeView, - Job, + cast(func.count(NotificationAllTimeView.id), Integer).label("count"), # <-- i added cast here ) + .join_from(NotificationAllTimeView, Job, NotificationAllTimeView.job_id == Job.id) # <-- i changed this to NotificationAllTimeView from notifications .where( NotificationAllTimeView.service_id == service_id, NotificationAllTimeView.key_type != KeyType.TEST, @@ -486,21 +483,22 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): stmt = ( select( - func.sum(sub_stmt.notification_count).label("total_notifications"), - sub_stmt.notification_type, - sub_stmt.status, - sub_stmt.day, - func.sum(sub_stmt.count).label("count"), + cast(func.sum(sub_stmt.c.notification_count), Integer).label("total_notifications"), # <-- i added cast here + sub_stmt.c.notification_type, + sub_stmt.c.status, + sub_stmt.c.day, + cast(func.sum(sub_stmt.c.count), Integer).label("count"), # <-- i added cast here ) - .group_by( - sub_stmt.notification_type, - sub_stmt.status, - sub_stmt.day, + .group_by( # <-- i added this group here + sub_stmt.c.notification_type, + sub_stmt.c.status, + sub_stmt.c.day ) ) return db.session.execute(stmt).all() + def dao_fetch_stats_for_service_from_days_for_user( service_id, start_date, end_date, user_id ): diff --git a/app/service/statistics.py b/app/service/statistics.py index 8bd41da25..d2e6f9976 100644 --- a/app/service/statistics.py +++ b/app/service/statistics.py @@ -83,11 +83,21 @@ def create_zeroed_stats_dicts(): def _update_statuses_from_row(update_dict, row): + # Initialize pending_count to total_notifications + pending_count = row.total_notifications + + # Update requested count if row.status != NotificationStatus.CANCELLED: update_dict[StatisticsType.REQUESTED] += row.count + pending_count -= row.count # Subtract from pending_count + + # Update delivered count if row.status in (NotificationStatus.DELIVERED, NotificationStatus.SENT): update_dict[StatisticsType.DELIVERED] += row.count - elif row.status in ( + pending_count -= row.count # Subtract from pending_count + + # Update failure count + if row.status in ( NotificationStatus.FAILED, NotificationStatus.TECHNICAL_FAILURE, NotificationStatus.TEMPORARY_FAILURE, @@ -96,13 +106,10 @@ def _update_statuses_from_row(update_dict, row): NotificationStatus.VIRUS_SCAN_FAILED, ): update_dict[StatisticsType.FAILURE] += row.count - elif row.status in ( - NotificationStatus.PENDING, - NotificationStatus.CREATED, - NotificationStatus.SENDING, - ): - update_dict[StatisticsType.PENDING] += row.count + pending_count -= row.count # Subtract from pending_count + # Update pending count directly + update_dict[StatisticsType.PENDING] = pending_count def create_empty_monthly_notification_status_stats_dict(year): utc_month_starts = get_months_for_financial_year(year) From f1cc6aa400d4963313edd3be87159a4890261ed0 Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Fri, 10 Jan 2025 13:47:43 -0500 Subject: [PATCH 07/24] estructuring how to get total_notifications. Signed-off-by: Cliff Hill --- app/dao/services_dao.py | 26 ++++++++++++++++++++++---- app/service/rest.py | 7 ++++--- app/service/statistics.py | 23 ++++++++++++++--------- 3 files changed, 40 insertions(+), 16 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 841434a62..c4eba56a0 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -457,7 +457,7 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): sub_stmt = ( select( - Job.id, + Job.id.label("job_id"), cast(Job.notification_count, Integer).label("notification_count"), # <-- i added cast here NotificationAllTimeView.notification_type, NotificationAllTimeView.status, @@ -481,6 +481,24 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): .subquery() ) + # Getting the total notifications through this query. + + total_stmt = ( + select( + sub_stmt.c.job_id, + sub_stmt.c.notification_count, + ) + .group_by( + sub_stmt.c.job_id, + sub_stmt.c.notification_count, + ) + ) + + total_notifications = sum( + count + for __, count in db.session.execute(total_stmt).all() + ) + stmt = ( select( cast(func.sum(sub_stmt.c.notification_count), Integer).label("total_notifications"), # <-- i added cast here @@ -495,7 +513,7 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): sub_stmt.c.day ) ) - return db.session.execute(stmt).all() + return total_notifications, db.session.execute(stmt).all() @@ -742,7 +760,7 @@ def fetch_notification_stats_for_service_by_month_by_user( return db.session.execute(stmt).all() -def get_specific_days_stats(data, start_date, days=None, end_date=None): +def get_specific_days_stats(data, start_date, days=None, end_date=None, total_notifications=None): if days is not None and end_date is not None: raise ValueError("Only set days OR set end_date, not both.") elif days is not None: @@ -758,7 +776,7 @@ def get_specific_days_stats(data, start_date, days=None, end_date=None): } stats = { - day.strftime("%Y-%m-%d"): statistics.format_statistics(rows) + day.strftime("%Y-%m-%d"): statistics.format_statistics(rows, total_notifications=total_notifications,) for day, rows in grouped_data.items() } diff --git a/app/service/rest.py b/app/service/rest.py index 7dd614058..3bc27ccb3 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -230,9 +230,9 @@ def get_service_statistics_for_specific_days(service_id, start, days=1): end_date = datetime.strptime(start, "%Y-%m-%d") start_date = end_date - timedelta(days=days - 1) - results = dao_fetch_stats_for_service_from_days(service_id, start_date, end_date) + total_notifications, results = dao_fetch_stats_for_service_from_days(service_id, start_date, end_date,) - stats = get_specific_days_stats(results, start_date, days=days) + stats = get_specific_days_stats(results, start_date, days=days, total_notifications=total_notifications,) return stats @@ -678,7 +678,8 @@ def get_single_month_notification_stats_for_service(service_id): month_year = datetime(year, month, 10, 00, 00, 00) start_date, end_date = get_month_start_and_end_date_in_utc(month_year) - results = dao_fetch_stats_for_service_from_days(service_id, start_date, end_date) + # First element is total notifications used elsewhere. + __, results = dao_fetch_stats_for_service_from_days(service_id, start_date, end_date) stats = get_specific_days_stats(results, start_date, end_date=end_date) return jsonify(stats) diff --git a/app/service/statistics.py b/app/service/statistics.py index d2e6f9976..4103daba8 100644 --- a/app/service/statistics.py +++ b/app/service/statistics.py @@ -5,7 +5,7 @@ from app.enums import KeyType, NotificationStatus, StatisticsType, TemplateType -def format_statistics(statistics): +def format_statistics(statistics, total_notifications=None): # statistics come in a named tuple with uniqueness from 'notification_type', 'status' - however missing # statuses/notification types won't be represented and the status types need to be simplified/summed up # so we can return emails/sms * created, sent, and failed @@ -14,7 +14,7 @@ def format_statistics(statistics): # any row could be null, if the service either has no notifications in the notifications table, # or no historical data in the ft_notification_status table. if row.notification_type: - _update_statuses_from_row(counts[row.notification_type], row) + _update_statuses_from_row(counts[row.notification_type], row, total_notifications=total_notifications,) return counts @@ -82,19 +82,22 @@ def create_zeroed_stats_dicts(): } -def _update_statuses_from_row(update_dict, row): +def _update_statuses_from_row(update_dict, row, total_notifications=None): # Initialize pending_count to total_notifications - pending_count = row.total_notifications + if total_notifications is not None: + pending_count = total_notifications # Update requested count if row.status != NotificationStatus.CANCELLED: update_dict[StatisticsType.REQUESTED] += row.count - pending_count -= row.count # Subtract from pending_count + if total_notifications is not None: + pending_count -= row.count # Subtract from pending_count # Update delivered count if row.status in (NotificationStatus.DELIVERED, NotificationStatus.SENT): update_dict[StatisticsType.DELIVERED] += row.count - pending_count -= row.count # Subtract from pending_count + if total_notifications is not None: + pending_count -= row.count # Subtract from pending_count # Update failure count if row.status in ( @@ -106,10 +109,12 @@ def _update_statuses_from_row(update_dict, row): NotificationStatus.VIRUS_SCAN_FAILED, ): update_dict[StatisticsType.FAILURE] += row.count - pending_count -= row.count # Subtract from pending_count + if total_notifications is not None: + pending_count -= row.count # Subtract from pending_count - # Update pending count directly - update_dict[StatisticsType.PENDING] = pending_count + if total_notifications is not None: + # Update pending count directly + update_dict[StatisticsType.PENDING] = pending_count def create_empty_monthly_notification_status_stats_dict(year): utc_month_starts = get_months_for_financial_year(year) From ef61069101a37546d4be379a0f7ed3e5762c7894 Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Fri, 10 Jan 2025 13:55:01 -0500 Subject: [PATCH 08/24] black cleanup. Signed-off-by: Cliff Hill --- app/dao/services_dao.py | 64 ++++++++++++++++++++------------------- app/service/rest.py | 17 +++++++++-- app/service/statistics.py | 7 ++++- 3 files changed, 53 insertions(+), 35 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index c4eba56a0..16d16d4c6 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -2,7 +2,7 @@ from datetime import timedelta from flask import current_app -from sqlalchemy import Float, cast, delete, select, Integer +from sqlalchemy import Float, Integer, cast, delete, select from sqlalchemy.orm import joinedload from sqlalchemy.sql.expression import and_, asc, case, func @@ -458,13 +458,19 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): sub_stmt = ( select( Job.id.label("job_id"), - cast(Job.notification_count, Integer).label("notification_count"), # <-- i added cast here + cast(Job.notification_count, Integer).label( + "notification_count" + ), # <-- i added cast here NotificationAllTimeView.notification_type, NotificationAllTimeView.status, func.date_trunc("day", NotificationAllTimeView.created_at).label("day"), - cast(func.count(NotificationAllTimeView.id), Integer).label("count"), # <-- i added cast here + cast(func.count(NotificationAllTimeView.id), Integer).label( + "count" + ), # <-- i added cast here ) - .join_from(NotificationAllTimeView, Job, NotificationAllTimeView.job_id == Job.id) # <-- i changed this to NotificationAllTimeView from notifications + .join_from( + NotificationAllTimeView, Job, NotificationAllTimeView.job_id == Job.id + ) # <-- i changed this to NotificationAllTimeView from notifications .where( NotificationAllTimeView.service_id == service_id, NotificationAllTimeView.key_type != KeyType.TEST, @@ -483,40 +489,31 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): # Getting the total notifications through this query. - total_stmt = ( - select( - sub_stmt.c.job_id, - sub_stmt.c.notification_count, - ) - .group_by( - sub_stmt.c.job_id, - sub_stmt.c.notification_count, - ) + total_stmt = select( + sub_stmt.c.job_id, + sub_stmt.c.notification_count, + ).group_by( + sub_stmt.c.job_id, + sub_stmt.c.notification_count, ) total_notifications = sum( - count - for __, count in db.session.execute(total_stmt).all() + count for __, count in db.session.execute(total_stmt).all() ) - stmt = ( - select( - cast(func.sum(sub_stmt.c.notification_count), Integer).label("total_notifications"), # <-- i added cast here - sub_stmt.c.notification_type, - sub_stmt.c.status, - sub_stmt.c.day, - cast(func.sum(sub_stmt.c.count), Integer).label("count"), # <-- i added cast here - ) - .group_by( # <-- i added this group here - sub_stmt.c.notification_type, - sub_stmt.c.status, - sub_stmt.c.day - ) + stmt = select( + sub_stmt.c.notification_type, + sub_stmt.c.status, + sub_stmt.c.day, + cast(func.sum(sub_stmt.c.count), Integer).label( + "count" + ), # <-- i added cast here + ).group_by( # <-- i added this group here + sub_stmt.c.notification_type, sub_stmt.c.status, sub_stmt.c.day ) return total_notifications, db.session.execute(stmt).all() - def dao_fetch_stats_for_service_from_days_for_user( service_id, start_date, end_date, user_id ): @@ -760,7 +757,9 @@ def fetch_notification_stats_for_service_by_month_by_user( return db.session.execute(stmt).all() -def get_specific_days_stats(data, start_date, days=None, end_date=None, total_notifications=None): +def get_specific_days_stats( + data, start_date, days=None, end_date=None, total_notifications=None +): if days is not None and end_date is not None: raise ValueError("Only set days OR set end_date, not both.") elif days is not None: @@ -776,7 +775,10 @@ def get_specific_days_stats(data, start_date, days=None, end_date=None, total_no } stats = { - day.strftime("%Y-%m-%d"): statistics.format_statistics(rows, total_notifications=total_notifications,) + day.strftime("%Y-%m-%d"): statistics.format_statistics( + rows, + total_notifications=total_notifications, + ) for day, rows in grouped_data.items() } diff --git a/app/service/rest.py b/app/service/rest.py index 3bc27ccb3..748da7df1 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -230,9 +230,18 @@ def get_service_statistics_for_specific_days(service_id, start, days=1): end_date = datetime.strptime(start, "%Y-%m-%d") start_date = end_date - timedelta(days=days - 1) - total_notifications, results = dao_fetch_stats_for_service_from_days(service_id, start_date, end_date,) + total_notifications, results = dao_fetch_stats_for_service_from_days( + service_id, + start_date, + end_date, + ) - stats = get_specific_days_stats(results, start_date, days=days, total_notifications=total_notifications,) + stats = get_specific_days_stats( + results, + start_date, + days=days, + total_notifications=total_notifications, + ) return stats @@ -679,7 +688,9 @@ def get_single_month_notification_stats_for_service(service_id): start_date, end_date = get_month_start_and_end_date_in_utc(month_year) # First element is total notifications used elsewhere. - __, results = dao_fetch_stats_for_service_from_days(service_id, start_date, end_date) + __, results = dao_fetch_stats_for_service_from_days( + service_id, start_date, end_date + ) stats = get_specific_days_stats(results, start_date, end_date=end_date) return jsonify(stats) diff --git a/app/service/statistics.py b/app/service/statistics.py index 4103daba8..d359d5f04 100644 --- a/app/service/statistics.py +++ b/app/service/statistics.py @@ -14,7 +14,11 @@ def format_statistics(statistics, total_notifications=None): # any row could be null, if the service either has no notifications in the notifications table, # or no historical data in the ft_notification_status table. if row.notification_type: - _update_statuses_from_row(counts[row.notification_type], row, total_notifications=total_notifications,) + _update_statuses_from_row( + counts[row.notification_type], + row, + total_notifications=total_notifications, + ) return counts @@ -116,6 +120,7 @@ def _update_statuses_from_row(update_dict, row, total_notifications=None): # Update pending count directly update_dict[StatisticsType.PENDING] = pending_count + def create_empty_monthly_notification_status_stats_dict(year): utc_month_starts = get_months_for_financial_year(year) # nested dicts - data[month][template type][status] = count From bc8084121227353399d4dd6df5a83efdc126ab55 Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Fri, 10 Jan 2025 14:29:46 -0500 Subject: [PATCH 09/24] Changing the queries again. Signed-off-by: Cliff Hill --- app/dao/services_dao.py | 57 ++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 32 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 16d16d4c6..41e920cf5 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -455,12 +455,33 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): start_date = get_midnight_in_utc(start_date) end_date = get_midnight_in_utc(end_date + timedelta(days=1)) - sub_stmt = ( + # Getting the total notifications through this query. + + total_stmt = select( select( - Job.id.label("job_id"), cast(Job.notification_count, Integer).label( "notification_count" ), # <-- i added cast here + ) + .join_from( + NotificationAllTimeView, Job, NotificationAllTimeView.job_id == Job.id + ) # <-- i changed this to NotificationAllTimeView from notifications + .where( + NotificationAllTimeView.service_id == service_id, + NotificationAllTimeView.key_type != KeyType.TEST, + NotificationAllTimeView.created_at >= start_date, + NotificationAllTimeView.created_at < end_date, + ) + .group_by( + Job.id, + Job.notification_count, + ) + ) + + total_notifications = sum(db.session.execute(total_stmt).scalars()) + + stmt = ( + select( NotificationAllTimeView.notification_type, NotificationAllTimeView.status, func.date_trunc("day", NotificationAllTimeView.created_at).label("day"), @@ -468,9 +489,6 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): "count" ), # <-- i added cast here ) - .join_from( - NotificationAllTimeView, Job, NotificationAllTimeView.job_id == Job.id - ) # <-- i changed this to NotificationAllTimeView from notifications .where( NotificationAllTimeView.service_id == service_id, NotificationAllTimeView.key_type != KeyType.TEST, @@ -478,40 +496,15 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): NotificationAllTimeView.created_at < end_date, ) .group_by( - Job.id, - Job.notification_count, NotificationAllTimeView.notification_type, NotificationAllTimeView.status, func.date_trunc("day", NotificationAllTimeView.created_at), ) - .subquery() ) - # Getting the total notifications through this query. - - total_stmt = select( - sub_stmt.c.job_id, - sub_stmt.c.notification_count, - ).group_by( - sub_stmt.c.job_id, - sub_stmt.c.notification_count, - ) - - total_notifications = sum( - count for __, count in db.session.execute(total_stmt).all() - ) + data = db.session.execute(stmt).all() - stmt = select( - sub_stmt.c.notification_type, - sub_stmt.c.status, - sub_stmt.c.day, - cast(func.sum(sub_stmt.c.count), Integer).label( - "count" - ), # <-- i added cast here - ).group_by( # <-- i added this group here - sub_stmt.c.notification_type, sub_stmt.c.status, sub_stmt.c.day - ) - return total_notifications, db.session.execute(stmt).all() + return total_notifications, data def dao_fetch_stats_for_service_from_days_for_user( From 162823e59bf896b9b10393881f02e899b3ac35d5 Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Fri, 10 Jan 2025 14:46:59 -0500 Subject: [PATCH 10/24] Query fixing. Signed-off-by: Cliff Hill --- app/dao/services_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 41e920cf5..e9171d1fc 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -457,7 +457,7 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): # Getting the total notifications through this query. - total_stmt = select( + total_stmt = ( select( cast(Job.notification_count, Integer).label( "notification_count" From 92190491504d7a0000a9a3bc28fd0d076fdceed9 Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Fri, 10 Jan 2025 15:08:44 -0500 Subject: [PATCH 11/24] Cleaning up function a bit. Signed-off-by: Cliff Hill --- app/service/statistics.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/app/service/statistics.py b/app/service/statistics.py index d359d5f04..11e19f16b 100644 --- a/app/service/statistics.py +++ b/app/service/statistics.py @@ -87,21 +87,19 @@ def create_zeroed_stats_dicts(): def _update_statuses_from_row(update_dict, row, total_notifications=None): - # Initialize pending_count to total_notifications - if total_notifications is not None: - pending_count = total_notifications + requested_count = 0 + delivered_count = 0 + failed_count = 0 # Update requested count if row.status != NotificationStatus.CANCELLED: update_dict[StatisticsType.REQUESTED] += row.count - if total_notifications is not None: - pending_count -= row.count # Subtract from pending_count + requested_count += row.count # Update delivered count if row.status in (NotificationStatus.DELIVERED, NotificationStatus.SENT): update_dict[StatisticsType.DELIVERED] += row.count - if total_notifications is not None: - pending_count -= row.count # Subtract from pending_count + delivered_count += row.count # Update failure count if row.status in ( @@ -113,11 +111,11 @@ def _update_statuses_from_row(update_dict, row, total_notifications=None): NotificationStatus.VIRUS_SCAN_FAILED, ): update_dict[StatisticsType.FAILURE] += row.count - if total_notifications is not None: - pending_count -= row.count # Subtract from pending_count + failed_count += row.count if total_notifications is not None: # Update pending count directly + pending_count = total_notifications - (requested_count + delivered_count + failed_count) update_dict[StatisticsType.PENDING] = pending_count From b543db474fc33bca9a2efa7994a4fb7799927c94 Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Fri, 10 Jan 2025 16:07:49 -0500 Subject: [PATCH 12/24] Moving where the pending calculation is done. Signed-off-by: Cliff Hill --- app/dao/services_dao.py | 11 +++++++++-- app/service/statistics.py | 19 +++++++++++-------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index e9171d1fc..93b2c93ac 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -457,8 +457,9 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): # Getting the total notifications through this query. - total_stmt = ( + total_substmt = ( select( + func.date_trunc("day", NotificationAllTimeView.created_at).label("day"), cast(Job.notification_count, Integer).label( "notification_count" ), # <-- i added cast here @@ -475,10 +476,16 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): .group_by( Job.id, Job.notification_count, + func.date_trunc("day", NotificationAllTimeView.created_at), ) + .subquery() + ) + + total_stmt = select( + func.sum(total_substmt.c.notification_count).label("total_notifications") ) - total_notifications = sum(db.session.execute(total_stmt).scalars()) + total_notifications = db.session.execute(total_stmt).scalar_one() stmt = ( select( diff --git a/app/service/statistics.py b/app/service/statistics.py index 11e19f16b..593067745 100644 --- a/app/service/statistics.py +++ b/app/service/statistics.py @@ -2,7 +2,7 @@ from datetime import datetime from app.dao.date_util import get_months_for_financial_year -from app.enums import KeyType, NotificationStatus, StatisticsType, TemplateType +from app.enums import KeyType, NotificationStatus, NotificationType, StatisticsType, TemplateType def format_statistics(statistics, total_notifications=None): @@ -17,9 +17,17 @@ def format_statistics(statistics, total_notifications=None): _update_statuses_from_row( counts[row.notification_type], row, - total_notifications=total_notifications, ) + # Update pending count directly + if NotificationType.SMS in counts and total_notifications is not None: + sms_dict = counts[NotificationType.SMS] + requested_count = sms_dict[StatisticsType.REQUESTED] + delivered_count = sms_dict[StatisticsType.DELIVERED] + failed_count = sms_dict[StatisticsType.FAILURE] + pending_count = total_notifications - (requested_count + delivered_count + failed_count) + sms_dict[StatisticsType.PENDING] = pending_count + return counts @@ -86,7 +94,7 @@ def create_zeroed_stats_dicts(): } -def _update_statuses_from_row(update_dict, row, total_notifications=None): +def _update_statuses_from_row(update_dict, row): requested_count = 0 delivered_count = 0 failed_count = 0 @@ -113,11 +121,6 @@ def _update_statuses_from_row(update_dict, row, total_notifications=None): update_dict[StatisticsType.FAILURE] += row.count failed_count += row.count - if total_notifications is not None: - # Update pending count directly - pending_count = total_notifications - (requested_count + delivered_count + failed_count) - update_dict[StatisticsType.PENDING] = pending_count - def create_empty_monthly_notification_status_stats_dict(year): utc_month_starts = get_months_for_financial_year(year) From 467357c0319fff965b300be6ee61e9680c43eb6d Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Fri, 10 Jan 2025 16:21:15 -0500 Subject: [PATCH 13/24] Getting total notifications per day made. Signed-off-by: Cliff Hill --- app/dao/services_dao.py | 13 ++++++++++--- app/service/statistics.py | 12 ++++++++++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 93b2c93ac..739f385ac 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -482,10 +482,13 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): ) total_stmt = select( - func.sum(total_substmt.c.notification_count).label("total_notifications") + func.date_trunc("day", NotificationAllTimeView.created_at).label("day"), + func.sum(total_substmt.c.notification_count).label("total_notifications"), + ).group_by( + func.date_trunc("day", NotificationAllTimeView.created_at), ) - total_notifications = db.session.execute(total_stmt).scalar_one() + total_notifications = {day: count for day, count in db.session.execute(total_stmt)} stmt = ( select( @@ -777,7 +780,11 @@ def get_specific_days_stats( stats = { day.strftime("%Y-%m-%d"): statistics.format_statistics( rows, - total_notifications=total_notifications, + total_notifications=( + total_notifications.get(day, 0) + if total_notifications is not None + else None + ), ) for day, rows in grouped_data.items() } diff --git a/app/service/statistics.py b/app/service/statistics.py index 593067745..68ba4f3ca 100644 --- a/app/service/statistics.py +++ b/app/service/statistics.py @@ -2,7 +2,13 @@ from datetime import datetime from app.dao.date_util import get_months_for_financial_year -from app.enums import KeyType, NotificationStatus, NotificationType, StatisticsType, TemplateType +from app.enums import ( + KeyType, + NotificationStatus, + NotificationType, + StatisticsType, + TemplateType, +) def format_statistics(statistics, total_notifications=None): @@ -25,7 +31,9 @@ def format_statistics(statistics, total_notifications=None): requested_count = sms_dict[StatisticsType.REQUESTED] delivered_count = sms_dict[StatisticsType.DELIVERED] failed_count = sms_dict[StatisticsType.FAILURE] - pending_count = total_notifications - (requested_count + delivered_count + failed_count) + pending_count = total_notifications - ( + requested_count + delivered_count + failed_count + ) sms_dict[StatisticsType.PENDING] = pending_count return counts From 3f8c49d829483bdb661c7a94ab8f0da7b3436906 Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Fri, 10 Jan 2025 16:52:30 -0500 Subject: [PATCH 14/24] Removing total_notification calculations. Signed-off-by: Cliff Hill --- app/dao/services_dao.py | 50 +++------------------------------------ app/service/rest.py | 9 ++----- app/service/statistics.py | 14 +---------- 3 files changed, 6 insertions(+), 67 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 739f385ac..7fdac4213 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -455,41 +455,6 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): start_date = get_midnight_in_utc(start_date) end_date = get_midnight_in_utc(end_date + timedelta(days=1)) - # Getting the total notifications through this query. - - total_substmt = ( - select( - func.date_trunc("day", NotificationAllTimeView.created_at).label("day"), - cast(Job.notification_count, Integer).label( - "notification_count" - ), # <-- i added cast here - ) - .join_from( - NotificationAllTimeView, Job, NotificationAllTimeView.job_id == Job.id - ) # <-- i changed this to NotificationAllTimeView from notifications - .where( - NotificationAllTimeView.service_id == service_id, - NotificationAllTimeView.key_type != KeyType.TEST, - NotificationAllTimeView.created_at >= start_date, - NotificationAllTimeView.created_at < end_date, - ) - .group_by( - Job.id, - Job.notification_count, - func.date_trunc("day", NotificationAllTimeView.created_at), - ) - .subquery() - ) - - total_stmt = select( - func.date_trunc("day", NotificationAllTimeView.created_at).label("day"), - func.sum(total_substmt.c.notification_count).label("total_notifications"), - ).group_by( - func.date_trunc("day", NotificationAllTimeView.created_at), - ) - - total_notifications = {day: count for day, count in db.session.execute(total_stmt)} - stmt = ( select( NotificationAllTimeView.notification_type, @@ -514,7 +479,7 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): data = db.session.execute(stmt).all() - return total_notifications, data + return data def dao_fetch_stats_for_service_from_days_for_user( @@ -760,9 +725,7 @@ def fetch_notification_stats_for_service_by_month_by_user( return db.session.execute(stmt).all() -def get_specific_days_stats( - data, start_date, days=None, end_date=None, total_notifications=None -): +def get_specific_days_stats(data, start_date, days=None, end_date=None): if days is not None and end_date is not None: raise ValueError("Only set days OR set end_date, not both.") elif days is not None: @@ -778,14 +741,7 @@ def get_specific_days_stats( } stats = { - day.strftime("%Y-%m-%d"): statistics.format_statistics( - rows, - total_notifications=( - total_notifications.get(day, 0) - if total_notifications is not None - else None - ), - ) + day.strftime("%Y-%m-%d"): statistics.format_statistics(rows) for day, rows in grouped_data.items() } diff --git a/app/service/rest.py b/app/service/rest.py index 748da7df1..d49142788 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -230,18 +230,13 @@ def get_service_statistics_for_specific_days(service_id, start, days=1): end_date = datetime.strptime(start, "%Y-%m-%d") start_date = end_date - timedelta(days=days - 1) - total_notifications, results = dao_fetch_stats_for_service_from_days( + results = dao_fetch_stats_for_service_from_days( service_id, start_date, end_date, ) - stats = get_specific_days_stats( - results, - start_date, - days=days, - total_notifications=total_notifications, - ) + stats = get_specific_days_stats(results, start_date, days=days) return stats diff --git a/app/service/statistics.py b/app/service/statistics.py index 68ba4f3ca..68d137876 100644 --- a/app/service/statistics.py +++ b/app/service/statistics.py @@ -5,13 +5,12 @@ from app.enums import ( KeyType, NotificationStatus, - NotificationType, StatisticsType, TemplateType, ) -def format_statistics(statistics, total_notifications=None): +def format_statistics(statistics): # statistics come in a named tuple with uniqueness from 'notification_type', 'status' - however missing # statuses/notification types won't be represented and the status types need to be simplified/summed up # so we can return emails/sms * created, sent, and failed @@ -25,17 +24,6 @@ def format_statistics(statistics, total_notifications=None): row, ) - # Update pending count directly - if NotificationType.SMS in counts and total_notifications is not None: - sms_dict = counts[NotificationType.SMS] - requested_count = sms_dict[StatisticsType.REQUESTED] - delivered_count = sms_dict[StatisticsType.DELIVERED] - failed_count = sms_dict[StatisticsType.FAILURE] - pending_count = total_notifications - ( - requested_count + delivered_count + failed_count - ) - sms_dict[StatisticsType.PENDING] = pending_count - return counts From 7ef808196eacd2eb32081df08c9ed666eed6124a Mon Sep 17 00:00:00 2001 From: Beverly Nguyen Date: Fri, 10 Jan 2025 18:18:20 -0800 Subject: [PATCH 15/24] pending count --- app/dao/services_dao.py | 60 +++++++++++++++++++++++++++++++++------ app/enums.py | 2 ++ app/service/rest.py | 9 ++++-- app/service/statistics.py | 17 ++++++++--- 4 files changed, 73 insertions(+), 15 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 7fdac4213..a2ec31f34 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -455,14 +455,49 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): start_date = get_midnight_in_utc(start_date) end_date = get_midnight_in_utc(end_date + timedelta(days=1)) + # Subquery for daily total notifications + total_substmt = ( + select( + func.date_trunc("day", NotificationAllTimeView.created_at).label("day"), + cast(Job.notification_count, Integer).label("notification_count") + ) + .join( + Job, NotificationAllTimeView.job_id == Job.id + ) + .where( + NotificationAllTimeView.service_id == service_id, + NotificationAllTimeView.key_type != KeyType.TEST, + NotificationAllTimeView.created_at >= start_date, + NotificationAllTimeView.created_at < end_date, + ) + .group_by( + Job.id, + Job.notification_count, + func.date_trunc("day", NotificationAllTimeView.created_at), + ) + .subquery() + ) + + # Query for daily total notifications + total_stmt = select( + func.date_trunc("day", total_substmt.c.day).label("day"), + func.sum(total_substmt.c.notification_count).label("total_notifications"), + ).group_by( + total_substmt.c.day + ) + + # Execute both queries + total_notifications = { + row.day: row.total_notifications for row in db.session.execute(total_stmt).all() + } + + # Query for breakdown by notification type and status stmt = ( select( NotificationAllTimeView.notification_type, NotificationAllTimeView.status, func.date_trunc("day", NotificationAllTimeView.created_at).label("day"), - cast(func.count(NotificationAllTimeView.id), Integer).label( - "count" - ), # <-- i added cast here + cast(func.count(NotificationAllTimeView.id), Integer).label("count"), ) .where( NotificationAllTimeView.service_id == service_id, @@ -479,8 +514,9 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): data = db.session.execute(stmt).all() - return data + print("Daily Total Notifications:", total_notifications) + return total_notifications, data def dao_fetch_stats_for_service_from_days_for_user( service_id, start_date, end_date, user_id @@ -725,7 +761,7 @@ def fetch_notification_stats_for_service_by_month_by_user( return db.session.execute(stmt).all() -def get_specific_days_stats(data, start_date, days=None, end_date=None): +def get_specific_days_stats(data, start_date, days=None, end_date=None,total_notifications=None): if days is not None and end_date is not None: raise ValueError("Only set days OR set end_date, not both.") elif days is not None: @@ -736,13 +772,19 @@ def get_specific_days_stats(data, start_date, days=None, end_date=None): raise ValueError("Either days or end_date must be set.") grouped_data = {date: [] for date in gen_range} | { - day: [row for row in data if row.day.date() == day] - for day in {item.day.date() for item in data} + day: [row for row in data if row.day == day] + for day in {item.day for item in data} } stats = { - day.strftime("%Y-%m-%d"): statistics.format_statistics(rows) + day.strftime("%Y-%m-%d"): statistics.format_statistics( + rows, + total_notifications=( + total_notifications.get(day, 0) + if total_notifications is not None + else None + ), + ) for day, rows in grouped_data.items() } - return stats diff --git a/app/enums.py b/app/enums.py index 37b3b6892..e69678671 100644 --- a/app/enums.py +++ b/app/enums.py @@ -212,3 +212,5 @@ class StatisticsType(StrEnum): DELIVERED = "delivered" FAILURE = "failure" PENDING = "pending" + SENDING = "sending" + CREATED = "created" diff --git a/app/service/rest.py b/app/service/rest.py index d49142788..748da7df1 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -230,13 +230,18 @@ def get_service_statistics_for_specific_days(service_id, start, days=1): end_date = datetime.strptime(start, "%Y-%m-%d") start_date = end_date - timedelta(days=days - 1) - results = dao_fetch_stats_for_service_from_days( + total_notifications, results = dao_fetch_stats_for_service_from_days( service_id, start_date, end_date, ) - stats = get_specific_days_stats(results, start_date, days=days) + stats = get_specific_days_stats( + results, + start_date, + days=days, + total_notifications=total_notifications, + ) return stats diff --git a/app/service/statistics.py b/app/service/statistics.py index 68d137876..41ced13ec 100644 --- a/app/service/statistics.py +++ b/app/service/statistics.py @@ -7,10 +7,11 @@ NotificationStatus, StatisticsType, TemplateType, + NotificationType ) -def format_statistics(statistics): +def format_statistics(statistics, total_notifications=None): # statistics come in a named tuple with uniqueness from 'notification_type', 'status' - however missing # statuses/notification types won't be represented and the status types need to be simplified/summed up # so we can return emails/sms * created, sent, and failed @@ -24,8 +25,18 @@ def format_statistics(statistics): row, ) - return counts + if NotificationType.SMS in counts and total_notifications is not None: + sms_dict = counts[NotificationType.SMS] + delivered_count = sms_dict[StatisticsType.DELIVERED] + failed_count = sms_dict[StatisticsType.FAILURE] + print('total_notifications',total_notifications) + pending_count = total_notifications - (delivered_count + failed_count) + + pending_count = max(0, pending_count) + sms_dict[StatisticsType.PENDING] = pending_count + + return counts def format_admin_stats(statistics): counts = create_stats_dict() @@ -98,12 +109,10 @@ def _update_statuses_from_row(update_dict, row): # Update requested count if row.status != NotificationStatus.CANCELLED: update_dict[StatisticsType.REQUESTED] += row.count - requested_count += row.count # Update delivered count if row.status in (NotificationStatus.DELIVERED, NotificationStatus.SENT): update_dict[StatisticsType.DELIVERED] += row.count - delivered_count += row.count # Update failure count if row.status in ( From 30dfc6a571113be60ebc12f1336a82d3ad3f0692 Mon Sep 17 00:00:00 2001 From: Beverly Nguyen Date: Fri, 10 Jan 2025 18:29:16 -0800 Subject: [PATCH 16/24] pending count --- app/dao/services_dao.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index a2ec31f34..f480b4852 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -2,7 +2,7 @@ from datetime import timedelta from flask import current_app -from sqlalchemy import Float, Integer, cast, delete, select +from sqlalchemy import Float, cast, delete, select from sqlalchemy.orm import joinedload from sqlalchemy.sql.expression import and_, asc, case, func @@ -459,7 +459,7 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): total_substmt = ( select( func.date_trunc("day", NotificationAllTimeView.created_at).label("day"), - cast(Job.notification_count, Integer).label("notification_count") + Job.notification_count.label("notification_count") ) .join( Job, NotificationAllTimeView.job_id == Job.id @@ -497,7 +497,7 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): NotificationAllTimeView.notification_type, NotificationAllTimeView.status, func.date_trunc("day", NotificationAllTimeView.created_at).label("day"), - cast(func.count(NotificationAllTimeView.id), Integer).label("count"), + func.count(NotificationAllTimeView.id).label("count"), ) .where( NotificationAllTimeView.service_id == service_id, From af46a671f9dbc5217ab214106a75b6d946195de6 Mon Sep 17 00:00:00 2001 From: Beverly Nguyen Date: Fri, 10 Jan 2025 18:41:09 -0800 Subject: [PATCH 17/24] cleaningu up pending --- app/dao/services_dao.py | 4 +--- app/enums.py | 2 -- app/service/statistics.py | 13 +------------ 3 files changed, 2 insertions(+), 17 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index f480b4852..09322a464 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -480,7 +480,7 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): # Query for daily total notifications total_stmt = select( - func.date_trunc("day", total_substmt.c.day).label("day"), + total_substmt.c.day, func.sum(total_substmt.c.notification_count).label("total_notifications"), ).group_by( total_substmt.c.day @@ -514,8 +514,6 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): data = db.session.execute(stmt).all() - print("Daily Total Notifications:", total_notifications) - return total_notifications, data def dao_fetch_stats_for_service_from_days_for_user( diff --git a/app/enums.py b/app/enums.py index e69678671..37b3b6892 100644 --- a/app/enums.py +++ b/app/enums.py @@ -212,5 +212,3 @@ class StatisticsType(StrEnum): DELIVERED = "delivered" FAILURE = "failure" PENDING = "pending" - SENDING = "sending" - CREATED = "created" diff --git a/app/service/statistics.py b/app/service/statistics.py index 41ced13ec..a6d87da9f 100644 --- a/app/service/statistics.py +++ b/app/service/statistics.py @@ -29,7 +29,6 @@ def format_statistics(statistics, total_notifications=None): sms_dict = counts[NotificationType.SMS] delivered_count = sms_dict[StatisticsType.DELIVERED] failed_count = sms_dict[StatisticsType.FAILURE] - print('total_notifications',total_notifications) pending_count = total_notifications - (delivered_count + failed_count) pending_count = max(0, pending_count) @@ -102,20 +101,11 @@ def create_zeroed_stats_dicts(): def _update_statuses_from_row(update_dict, row): - requested_count = 0 - delivered_count = 0 - failed_count = 0 - - # Update requested count if row.status != NotificationStatus.CANCELLED: update_dict[StatisticsType.REQUESTED] += row.count - - # Update delivered count if row.status in (NotificationStatus.DELIVERED, NotificationStatus.SENT): update_dict[StatisticsType.DELIVERED] += row.count - - # Update failure count - if row.status in ( + elif row.status in ( NotificationStatus.FAILED, NotificationStatus.TECHNICAL_FAILURE, NotificationStatus.TEMPORARY_FAILURE, @@ -124,7 +114,6 @@ def _update_statuses_from_row(update_dict, row): NotificationStatus.VIRUS_SCAN_FAILED, ): update_dict[StatisticsType.FAILURE] += row.count - failed_count += row.count def create_empty_monthly_notification_status_stats_dict(year): From 50a183f05defac5efe89c321bfe9b44353b47dea Mon Sep 17 00:00:00 2001 From: Beverly Nguyen Date: Fri, 10 Jan 2025 18:42:43 -0800 Subject: [PATCH 18/24] flake --- app/dao/services_dao.py | 3 ++- app/service/statistics.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 09322a464..a29dd464a 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -516,6 +516,7 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): return total_notifications, data + def dao_fetch_stats_for_service_from_days_for_user( service_id, start_date, end_date, user_id ): @@ -759,7 +760,7 @@ def fetch_notification_stats_for_service_by_month_by_user( return db.session.execute(stmt).all() -def get_specific_days_stats(data, start_date, days=None, end_date=None,total_notifications=None): +def get_specific_days_stats(data, start_date, days=None, end_date=None, total_notifications=None): if days is not None and end_date is not None: raise ValueError("Only set days OR set end_date, not both.") elif days is not None: diff --git a/app/service/statistics.py b/app/service/statistics.py index a6d87da9f..a1c34ea65 100644 --- a/app/service/statistics.py +++ b/app/service/statistics.py @@ -37,6 +37,7 @@ def format_statistics(statistics, total_notifications=None): return counts + def format_admin_stats(statistics): counts = create_stats_dict() From 2eef7ab206a2d4ce4ba598a5b7c55b6b3af882f5 Mon Sep 17 00:00:00 2001 From: Beverly Nguyen Date: Fri, 10 Jan 2025 18:45:05 -0800 Subject: [PATCH 19/24] remove comments --- app/dao/services_dao.py | 4 ---- app/service/rest.py | 1 - 2 files changed, 5 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index a29dd464a..ba3dbf717 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -455,7 +455,6 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): start_date = get_midnight_in_utc(start_date) end_date = get_midnight_in_utc(end_date + timedelta(days=1)) - # Subquery for daily total notifications total_substmt = ( select( func.date_trunc("day", NotificationAllTimeView.created_at).label("day"), @@ -478,7 +477,6 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): .subquery() ) - # Query for daily total notifications total_stmt = select( total_substmt.c.day, func.sum(total_substmt.c.notification_count).label("total_notifications"), @@ -486,12 +484,10 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): total_substmt.c.day ) - # Execute both queries total_notifications = { row.day: row.total_notifications for row in db.session.execute(total_stmt).all() } - # Query for breakdown by notification type and status stmt = ( select( NotificationAllTimeView.notification_type, diff --git a/app/service/rest.py b/app/service/rest.py index 748da7df1..718e3bd33 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -687,7 +687,6 @@ def get_single_month_notification_stats_for_service(service_id): month_year = datetime(year, month, 10, 00, 00, 00) start_date, end_date = get_month_start_and_end_date_in_utc(month_year) - # First element is total notifications used elsewhere. __, results = dao_fetch_stats_for_service_from_days( service_id, start_date, end_date ) From fa00bd14bf352a5a7fdb7d876dcca12def3e9254 Mon Sep 17 00:00:00 2001 From: Beverly Nguyen Date: Fri, 10 Jan 2025 18:47:39 -0800 Subject: [PATCH 20/24] isort --- app/service/statistics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/service/statistics.py b/app/service/statistics.py index a1c34ea65..b67107ab1 100644 --- a/app/service/statistics.py +++ b/app/service/statistics.py @@ -5,9 +5,9 @@ from app.enums import ( KeyType, NotificationStatus, + NotificationType, StatisticsType, TemplateType, - NotificationType ) From dd3c2779858e561e13ca8dd8fc09773bd03129cf Mon Sep 17 00:00:00 2001 From: Beverly Nguyen Date: Fri, 10 Jan 2025 19:26:46 -0800 Subject: [PATCH 21/24] fix testing --- app/dao/services_dao.py | 14 ++++++-------- tests/app/service/test_rest.py | 23 +++++++++++++++++++++-- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index ba3dbf717..6bd2e8620 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -458,11 +458,9 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): total_substmt = ( select( func.date_trunc("day", NotificationAllTimeView.created_at).label("day"), - Job.notification_count.label("notification_count") - ) - .join( - Job, NotificationAllTimeView.job_id == Job.id + Job.notification_count.label("notification_count"), ) + .join(Job, NotificationAllTimeView.job_id == Job.id) .where( NotificationAllTimeView.service_id == service_id, NotificationAllTimeView.key_type != KeyType.TEST, @@ -480,9 +478,7 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): total_stmt = select( total_substmt.c.day, func.sum(total_substmt.c.notification_count).label("total_notifications"), - ).group_by( - total_substmt.c.day - ) + ).group_by(total_substmt.c.day) total_notifications = { row.day: row.total_notifications for row in db.session.execute(total_stmt).all() @@ -756,7 +752,9 @@ def fetch_notification_stats_for_service_by_month_by_user( return db.session.execute(stmt).all() -def get_specific_days_stats(data, start_date, days=None, end_date=None, total_notifications=None): +def get_specific_days_stats( + data, start_date, days=None, end_date=None, total_notifications=None +): if days is not None and end_date is not None: raise ValueError("Only set days OR set end_date, not both.") elif days is not None: diff --git a/tests/app/service/test_rest.py b/tests/app/service/test_rest.py index 132de48e9..f4057d0db 100644 --- a/tests/app/service/test_rest.py +++ b/tests/app/service/test_rest.py @@ -2200,6 +2200,7 @@ def test_set_sms_prefixing_for_service_cant_be_none( StatisticsType.REQUESTED: 2, StatisticsType.DELIVERED: 1, StatisticsType.FAILURE: 0, + StatisticsType.PENDING: 0, }, ), ( @@ -2208,6 +2209,7 @@ def test_set_sms_prefixing_for_service_cant_be_none( StatisticsType.REQUESTED: 1, StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, + StatisticsType.PENDING: 0, }, ), ], @@ -2256,11 +2258,13 @@ def test_get_services_with_detailed_flag(client, sample_template): NotificationType.EMAIL: { StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, + StatisticsType.PENDING: 0, StatisticsType.REQUESTED: 0, }, NotificationType.SMS: { StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, + StatisticsType.PENDING: 0, StatisticsType.REQUESTED: 3, }, } @@ -2287,11 +2291,13 @@ def test_get_services_with_detailed_flag_excluding_from_test_key( NotificationType.EMAIL: { StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, + StatisticsType.PENDING: 0, StatisticsType.REQUESTED: 0, }, NotificationType.SMS: { StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, + StatisticsType.PENDING: 0, StatisticsType.REQUESTED: 2, }, } @@ -2363,11 +2369,13 @@ def test_get_detailed_services_groups_by_service(notify_db_session): NotificationType.EMAIL: { StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, + StatisticsType.PENDING: 0, StatisticsType.REQUESTED: 0, }, NotificationType.SMS: { StatisticsType.DELIVERED: 1, StatisticsType.FAILURE: 0, + StatisticsType.PENDING: 0, StatisticsType.REQUESTED: 3, }, } @@ -2376,11 +2384,13 @@ def test_get_detailed_services_groups_by_service(notify_db_session): NotificationType.EMAIL: { StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, + StatisticsType.PENDING: 0, StatisticsType.REQUESTED: 0, }, NotificationType.SMS: { StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, + StatisticsType.PENDING: 0, StatisticsType.REQUESTED: 1, }, } @@ -2406,11 +2416,13 @@ def test_get_detailed_services_includes_services_with_no_notifications( NotificationType.EMAIL: { StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, + StatisticsType.PENDING: 0, StatisticsType.REQUESTED: 0, }, NotificationType.SMS: { StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, + StatisticsType.PENDING: 0, StatisticsType.REQUESTED: 1, }, } @@ -2419,11 +2431,13 @@ def test_get_detailed_services_includes_services_with_no_notifications( NotificationType.EMAIL: { StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, + StatisticsType.PENDING: 0, StatisticsType.REQUESTED: 0, }, NotificationType.SMS: { StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, + StatisticsType.PENDING: 0, StatisticsType.REQUESTED: 0, }, } @@ -2448,12 +2462,15 @@ def test_get_detailed_services_only_includes_todays_notifications(sample_templat NotificationType.EMAIL: { StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, - StatisticsType.REQUESTED: 0, + StatisticsType.PENDING: 0, + StatisticsType.REQUESTED: 0 + }, NotificationType.SMS: { StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, - StatisticsType.REQUESTED: 3, + StatisticsType.PENDING: 0, + StatisticsType.REQUESTED: 3 }, } @@ -2501,11 +2518,13 @@ def test_get_detailed_services_for_date_range( assert data[0]["statistics"][NotificationType.EMAIL] == { StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, + StatisticsType.PENDING: 0, StatisticsType.REQUESTED: 0, } assert data[0]["statistics"][NotificationType.SMS] == { StatisticsType.DELIVERED: 2, StatisticsType.FAILURE: 0, + StatisticsType.PENDING: 0, StatisticsType.REQUESTED: 2, } From b2240659ce2092c473a73af3c7313a9d99297f85 Mon Sep 17 00:00:00 2001 From: Beverly Nguyen Date: Mon, 13 Jan 2025 10:16:38 -0800 Subject: [PATCH 22/24] fix testing --- app/enums.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/enums.py b/app/enums.py index 37b3b6892..a0dfbb467 100644 --- a/app/enums.py +++ b/app/enums.py @@ -211,4 +211,3 @@ class StatisticsType(StrEnum): REQUESTED = "requested" DELIVERED = "delivered" FAILURE = "failure" - PENDING = "pending" From 88c3da5579099600e66ae869a29a033b27b34036 Mon Sep 17 00:00:00 2001 From: Beverly Nguyen Date: Mon, 13 Jan 2025 13:39:34 -0800 Subject: [PATCH 23/24] add testing --- app/enums.py | 1 + app/service/statistics.py | 11 +++-- tests/app/dao/test_services_dao.py | 38 ++++++++++++++- tests/app/service/test_rest.py | 5 +- tests/app/service/test_statistics.py | 59 ++++++++++++++++++----- tests/app/service/test_statistics_rest.py | 4 ++ 6 files changed, 99 insertions(+), 19 deletions(-) diff --git a/app/enums.py b/app/enums.py index a0dfbb467..37b3b6892 100644 --- a/app/enums.py +++ b/app/enums.py @@ -211,3 +211,4 @@ class StatisticsType(StrEnum): REQUESTED = "requested" DELIVERED = "delivered" FAILURE = "failure" + PENDING = "pending" diff --git a/app/service/statistics.py b/app/service/statistics.py index b67107ab1..d6d776539 100644 --- a/app/service/statistics.py +++ b/app/service/statistics.py @@ -29,13 +29,16 @@ def format_statistics(statistics, total_notifications=None): sms_dict = counts[NotificationType.SMS] delivered_count = sms_dict[StatisticsType.DELIVERED] failed_count = sms_dict[StatisticsType.FAILURE] - pending_count = total_notifications - (delivered_count + failed_count) + sms_dict[StatisticsType.PENDING] = calculate_pending_stats( + delivered_count, failed_count, total_notifications + ) - pending_count = max(0, pending_count) + return counts - sms_dict[StatisticsType.PENDING] = pending_count - return counts +def calculate_pending_stats(delivered_count, failed_count, total_notifications): + pending_count = total_notifications - (delivered_count + failed_count) + return max(0, pending_count) def format_admin_stats(statistics): diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index 61fe99419..8cd8a11fd 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -1638,11 +1638,13 @@ def test_get_live_services_with_organization(sample_organization): StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, StatisticsType.REQUESTED: 0, + StatisticsType.PENDING: 0, }, TemplateType.SMS: { StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, StatisticsType.REQUESTED: 2, + StatisticsType.PENDING: 2, }, }, (_this_date.date() + timedelta(days=1)).strftime("%Y-%m-%d"): { @@ -1650,11 +1652,13 @@ def test_get_live_services_with_organization(sample_organization): StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, StatisticsType.REQUESTED: 0, + StatisticsType.PENDING: 0, }, TemplateType.SMS: { StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, StatisticsType.REQUESTED: 1, + StatisticsType.PENDING: 0, }, }, (_this_date.date() + timedelta(days=2)).strftime("%Y-%m-%d"): { @@ -1662,11 +1666,13 @@ def test_get_live_services_with_organization(sample_organization): StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, StatisticsType.REQUESTED: 0, + StatisticsType.PENDING: 0, }, TemplateType.SMS: { StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, StatisticsType.REQUESTED: 1, + StatisticsType.PENDING: 0, }, }, (_this_date.date() + timedelta(days=3)).strftime("%Y-%m-%d"): { @@ -1674,11 +1680,13 @@ def test_get_live_services_with_organization(sample_organization): StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, StatisticsType.REQUESTED: 0, + StatisticsType.PENDING: 0, }, TemplateType.SMS: { StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, StatisticsType.REQUESTED: 0, + StatisticsType.PENDING: 0, }, }, (_this_date.date() + timedelta(days=4)).strftime("%Y-%m-%d"): { @@ -1686,11 +1694,13 @@ def test_get_live_services_with_organization(sample_organization): StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, StatisticsType.REQUESTED: 0, + StatisticsType.PENDING: 0, }, TemplateType.SMS: { StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, StatisticsType.REQUESTED: 1, + StatisticsType.PENDING: 0, }, }, }, @@ -1713,11 +1723,13 @@ def test_get_live_services_with_organization(sample_organization): StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, StatisticsType.REQUESTED: 0, + StatisticsType.PENDING: 0, }, TemplateType.SMS: { StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, StatisticsType.REQUESTED: 2, + StatisticsType.PENDING: 2, }, }, (_this_date.date() + timedelta(days=1)).strftime("%Y-%m-%d"): { @@ -1725,11 +1737,13 @@ def test_get_live_services_with_organization(sample_organization): StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, StatisticsType.REQUESTED: 0, + StatisticsType.PENDING: 0, }, TemplateType.SMS: { StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, StatisticsType.REQUESTED: 1, + StatisticsType.PENDING: 0, }, }, (_this_date.date() + timedelta(days=2)).strftime("%Y-%m-%d"): { @@ -1737,11 +1751,13 @@ def test_get_live_services_with_organization(sample_organization): StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, StatisticsType.REQUESTED: 0, + StatisticsType.PENDING: 0, }, TemplateType.SMS: { StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, StatisticsType.REQUESTED: 1, + StatisticsType.PENDING: 0, }, }, (_this_date.date() + timedelta(days=3)).strftime("%Y-%m-%d"): { @@ -1749,11 +1765,13 @@ def test_get_live_services_with_organization(sample_organization): StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, StatisticsType.REQUESTED: 0, + StatisticsType.PENDING: 0, }, TemplateType.SMS: { StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, StatisticsType.REQUESTED: 0, + StatisticsType.PENDING: 0, }, }, (_this_date.date() + timedelta(days=4)).strftime("%Y-%m-%d"): { @@ -1761,11 +1779,13 @@ def test_get_live_services_with_organization(sample_organization): StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, StatisticsType.REQUESTED: 0, + StatisticsType.PENDING: 0, }, TemplateType.SMS: { StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, StatisticsType.REQUESTED: 1, + StatisticsType.PENDING: 0, }, }, }, @@ -1786,5 +1806,21 @@ def test_get_specific_days(data, start_date, days, end_date, expected, is_error) new_line.count = 1 new_line.something = line["something"] new_data.append(new_line) - results = get_specific_days_stats(new_data, start_date, days, end_date) + + total_notifications = None + + date_key = _this_date.date().strftime("%Y-%m-%d") + if expected and date_key in expected: + sms_stats = expected[date_key].get(TemplateType.SMS, {}) + requested = sms_stats.get(StatisticsType.REQUESTED, 0) + if requested > 0: + total_notifications = {_this_date: requested} + + results = get_specific_days_stats( + new_data, + start_date, + days, + end_date, + total_notifications=total_notifications, + ) assert results == expected diff --git a/tests/app/service/test_rest.py b/tests/app/service/test_rest.py index f4057d0db..4dc48140e 100644 --- a/tests/app/service/test_rest.py +++ b/tests/app/service/test_rest.py @@ -2463,14 +2463,13 @@ def test_get_detailed_services_only_includes_todays_notifications(sample_templat StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, StatisticsType.PENDING: 0, - StatisticsType.REQUESTED: 0 - + StatisticsType.REQUESTED: 0, }, NotificationType.SMS: { StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, StatisticsType.PENDING: 0, - StatisticsType.REQUESTED: 3 + StatisticsType.REQUESTED: 3, }, } diff --git a/tests/app/service/test_statistics.py b/tests/app/service/test_statistics.py index b3534fed3..a16625361 100644 --- a/tests/app/service/test_statistics.py +++ b/tests/app/service/test_statistics.py @@ -9,6 +9,7 @@ from app.enums import KeyType, NotificationStatus, NotificationType, StatisticsType from app.service.statistics import ( add_monthly_notification_status_stats, + calculate_pending_stats, create_empty_monthly_notification_status_stats_dict, create_stats_dict, create_zeroed_stats_dicts, @@ -27,22 +28,22 @@ @pytest.mark.idparametrize( "stats, email_counts, sms_counts", { - "empty": ([], [0, 0, 0], [0, 0, 0]), + "empty": ([], [0, 0, 0, 0], [0, 0, 0, 0]), "always_increment_requested": ( [ StatsRow(NotificationType.EMAIL, NotificationStatus.DELIVERED, 1), StatsRow(NotificationType.EMAIL, NotificationStatus.FAILED, 1), ], - [2, 1, 1], - [0, 0, 0], + [2, 1, 1, 0], + [0, 0, 0, 0], ), "dont_mix_template_types": ( [ StatsRow(NotificationType.EMAIL, NotificationStatus.DELIVERED, 1), StatsRow(NotificationType.SMS, NotificationStatus.DELIVERED, 1), ], - [1, 1, 0], - [1, 1, 0], + [1, 1, 0, 0], + [1, 1, 0, 0], ), "convert_fail_statuses_to_failed": ( [ @@ -57,8 +58,8 @@ NotificationType.EMAIL, NotificationStatus.PERMANENT_FAILURE, 1 ), ], - [4, 0, 4], - [0, 0, 0], + [4, 0, 4, 0], + [0, 0, 0, 0], ), "convert_sent_to_delivered": ( [ @@ -66,16 +67,16 @@ StatsRow(NotificationType.SMS, NotificationStatus.DELIVERED, 1), StatsRow(NotificationType.SMS, NotificationStatus.SENT, 1), ], - [0, 0, 0], - [3, 2, 0], + [0, 0, 0, 0], + [3, 2, 0, 0], ), "handles_none_rows": ( [ StatsRow(NotificationType.SMS, NotificationStatus.SENDING, 1), StatsRow(None, None, None), ], - [0, 0, 0], - [1, 0, 0], + [0, 0, 0, 0], + [1, 0, 0, 0], ), }, ) @@ -89,6 +90,7 @@ def test_format_statistics(stats, email_counts, sms_counts): StatisticsType.REQUESTED, StatisticsType.DELIVERED, StatisticsType.FAILURE, + StatisticsType.PENDING, ], email_counts, ) @@ -101,23 +103,58 @@ def test_format_statistics(stats, email_counts, sms_counts): StatisticsType.REQUESTED, StatisticsType.DELIVERED, StatisticsType.FAILURE, + StatisticsType.PENDING, ], sms_counts, ) } +def test_format_statistics_with_pending(): + stats = [ + StatsRow(NotificationType.SMS, NotificationStatus.DELIVERED, 10), + StatsRow(NotificationType.SMS, NotificationStatus.FAILED, 2), + ] + + total_notifications_for_sms = 20 + + result = format_statistics(stats, total_notifications=total_notifications_for_sms) + + expected_sms_counts = { + StatisticsType.REQUESTED: 12, + StatisticsType.DELIVERED: 10, + StatisticsType.FAILURE: 2, + StatisticsType.PENDING: 8, + } + + assert result[NotificationType.SMS] == expected_sms_counts + + +@pytest.mark.parametrize( + "delivered, failed, total, expected", + [ + (10, 2, 20, 8), + (10, 10, 20, 0), + (15, 10, 20, 0), + ], +) +def test_calculate_pending(delivered, failed, total, expected): + assert calculate_pending_stats(delivered, failed, total) == expected + + def test_create_zeroed_stats_dicts(): assert create_zeroed_stats_dicts() == { NotificationType.SMS: { StatisticsType.REQUESTED: 0, StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, + StatisticsType.PENDING: 0, }, NotificationType.EMAIL: { StatisticsType.REQUESTED: 0, StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, + StatisticsType.PENDING: 0, }, } diff --git a/tests/app/service/test_statistics_rest.py b/tests/app/service/test_statistics_rest.py index 6d20cacc3..254736bc9 100644 --- a/tests/app/service/test_statistics_rest.py +++ b/tests/app/service/test_statistics_rest.py @@ -119,6 +119,7 @@ def test_get_template_usage_by_month_returns_two_templates( StatisticsType.REQUESTED: 2, StatisticsType.DELIVERED: 1, StatisticsType.FAILURE: 0, + StatisticsType.PENDING: 0, }, ), ( @@ -127,6 +128,7 @@ def test_get_template_usage_by_month_returns_two_templates( StatisticsType.REQUESTED: 1, StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, + StatisticsType.PENDING: 0, }, ), ], @@ -163,11 +165,13 @@ def test_get_service_notification_statistics_with_unknown_service(admin_request) StatisticsType.REQUESTED: 0, StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, + StatisticsType.PENDING: 0, }, NotificationType.EMAIL: { StatisticsType.REQUESTED: 0, StatisticsType.DELIVERED: 0, StatisticsType.FAILURE: 0, + StatisticsType.PENDING: 0, }, } From 880237f55d55f78599910f282a248b6a27088f1d Mon Sep 17 00:00:00 2001 From: Beverly Nguyen Date: Tue, 14 Jan 2025 16:57:43 -0800 Subject: [PATCH 24/24] fix tuples --- app/dao/services_dao.py | 38 +++++++++++++++++++++++++++++++++++--- app/service/rest.py | 14 +++++++++----- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 6bd2e8620..7a8d73578 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -515,6 +515,36 @@ def dao_fetch_stats_for_service_from_days_for_user( start_date = get_midnight_in_utc(start_date) end_date = get_midnight_in_utc(end_date + timedelta(days=1)) + total_substmt = ( + select( + func.date_trunc("day", NotificationAllTimeView.created_at).label("day"), + Job.notification_count.label("notification_count"), + ) + .join(Job, NotificationAllTimeView.job_id == Job.id) + .where( + NotificationAllTimeView.service_id == service_id, + NotificationAllTimeView.key_type != KeyType.TEST, + NotificationAllTimeView.created_at >= start_date, + NotificationAllTimeView.created_at < end_date, + NotificationAllTimeView.created_by_id == user_id, + ) + .group_by( + Job.id, + Job.notification_count, + func.date_trunc("day", NotificationAllTimeView.created_at), + ) + .subquery() + ) + + total_stmt = select( + total_substmt.c.day, + func.sum(total_substmt.c.notification_count).label("total_notifications"), + ).group_by(total_substmt.c.day) + + total_notifications = { + row.day: row.total_notifications for row in db.session.execute(total_stmt).all() + } + stmt = ( select( NotificationAllTimeView.notification_type, @@ -522,8 +552,7 @@ def dao_fetch_stats_for_service_from_days_for_user( func.date_trunc("day", NotificationAllTimeView.created_at).label("day"), func.count(NotificationAllTimeView.id).label("count"), ) - .select_from(NotificationAllTimeView) - .filter( + .where( NotificationAllTimeView.service_id == service_id, NotificationAllTimeView.key_type != KeyType.TEST, NotificationAllTimeView.created_at >= start_date, @@ -536,7 +565,10 @@ def dao_fetch_stats_for_service_from_days_for_user( func.date_trunc("day", NotificationAllTimeView.created_at), ) ) - return db.session.execute(stmt).scalars().all() + + data = db.session.execute(stmt).all() + + return total_notifications, data def dao_fetch_todays_stats_for_all_services( diff --git a/app/service/rest.py b/app/service/rest.py index 718e3bd33..81c5eb5c5 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -268,12 +268,16 @@ def get_service_statistics_for_specific_days_by_user( end_date = datetime.strptime(start, "%Y-%m-%d") start_date = end_date - timedelta(days=days - 1) - results = dao_fetch_stats_for_service_from_days_for_user( + total_notifications, results = dao_fetch_stats_for_service_from_days_for_user( service_id, start_date, end_date, user_id ) - stats = get_specific_days_stats(results, start_date, days=days) - + stats = get_specific_days_stats( + results, + start_date, + days=days, + total_notifications=total_notifications, + ) return stats @@ -663,11 +667,11 @@ def get_single_month_notification_stats_by_user(service_id, user_id): month_year = datetime(year, month, 10, 00, 00, 00) start_date, end_date = get_month_start_and_end_date_in_utc(month_year) - results = dao_fetch_stats_for_service_from_days_for_user( + total_notifications, results = dao_fetch_stats_for_service_from_days_for_user( service_id, start_date, end_date, user_id ) - stats = get_specific_days_stats(results, start_date, end_date=end_date) + stats = get_specific_days_stats(results, start_date, end_date=end_date, total_notifications=total_notifications,) return jsonify(stats)