From 44575ee198f07d7587a1e8b19ef546119bb882e9 Mon Sep 17 00:00:00 2001 From: Rudo Kemper Date: Mon, 6 Jan 2025 21:18:55 -0500 Subject: [PATCH 01/12] WIP: twilio working --- c_twilio.resource-type.json | 47 ++++++++++++++++++++++ f/connectors/alerts/alerts_gcs.py | 41 ++++++++++++++++++- f/connectors/alerts/alerts_gcs.script.lock | 1 + f/connectors/alerts/alerts_gcs.script.yaml | 9 +++++ 4 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 c_twilio.resource-type.json diff --git a/c_twilio.resource-type.json b/c_twilio.resource-type.json new file mode 100644 index 0000000..9943d19 --- /dev/null +++ b/c_twilio.resource-type.json @@ -0,0 +1,47 @@ +{ + "type": "object", + "order": [ + "account_sid", + "auth_token", + "message_service_sid", + "origin_number", + "forward_numbers" + ], + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "account_sid", + "auth_token", + "origin_number", + "forward_numbers" + ], + "properties": { + "account_sid": { + "type": "string", + "default": "", + "description": "The 34 letter SID used to represent a Twilio resource." + }, + "auth_token": { + "type": "string", + "default": "", + "description": "The token used to authenticate Twilio API requests." + }, + "message_service_sid": { + "type": "string", + "default": "", + "description": "(Optional) The SID for a messaging service, which is a container that bundle messaging functionality for your specific use cases (such as WhatsApp). It can be found in the Messaging Services menu; each service has their own SID. By including it, you can retrieve usage statistics for the service." + }, + "origin_number": { + "type": "string", + "default": "", + "description": "The phone number from which messages will originate. This number must be initially approved and authenticated in the Twilio UI before it can be activated and used. If you are sending via a WhatsApp number, prefix your phone number with whatsapp:." + }, + "recipients": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "description": "The list of phone numbers to which alerts will be sent. If you are sending to WhatsApp numbers, prefix your phone numbers with whatsapp:." + } + } +} \ No newline at end of file diff --git a/f/connectors/alerts/alerts_gcs.py b/f/connectors/alerts/alerts_gcs.py index 8459687..930860b 100644 --- a/f/connectors/alerts/alerts_gcs.py +++ b/f/connectors/alerts/alerts_gcs.py @@ -6,6 +6,7 @@ # pillow~=10.3 # psycopg2-binary # requests~=2.32 +# twilio~=9.4 import json import logging @@ -19,10 +20,12 @@ from google.oauth2.service_account import Credentials from PIL import Image from psycopg2 import sql +from twilio.rest import Client as TwilioClient # type names that refer to Windmill Resources gcp_service_account = dict postgresql = dict +c_twilio = dict logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -42,6 +45,7 @@ def main( territory_id: int, db: postgresql, db_table_name: str, + twilio: c_twilio, destination_path: str = "/frizzle-persistent-storage/datalake/change_detection/alerts", ): """ @@ -53,7 +57,13 @@ def main( ) return _main( - storage_client, alerts_bucket, territory_id, db, db_table_name, destination_path + storage_client, + alerts_bucket, + territory_id, + db, + db_table_name, + destination_path, + twilio, ) @@ -64,6 +74,7 @@ def _main( db: postgresql, db_table_name: str, destination_path: str, + twilio: c_twilio, ): """Download alerts to warehouse storage and index them in a database. @@ -80,6 +91,8 @@ def _main( The name of the database table to write alerts to. destination_path : str, optional The local directory to save files + twilio : dict, optional + A dictionary containing Twilio configuration parameters. Returns ------- @@ -107,6 +120,9 @@ def _main( f"Alerts data successfully written to database table: [{db_table_name}]" ) + if twilio: + send_twilio_message(twilio) + def _get_rel_filepath(local_file_path, territory_id): """Generate the relative file path for a file based on its blob name and local path. @@ -726,3 +742,26 @@ def handle_output(self, alerts, alerts_metadata): logger.info(f"Total alert rows updated: {updated_count}") cursor.close() conn.close() + + +def send_twilio_message(twilio): + """ + Send a Twilio message to alert the user of the script's completion. + """ + messaging_service_sid = twilio.get("messaging_service_sid") + client = TwilioClient(twilio["account_sid"], twilio["auth_token"]) + + message = "Alerts data successfully written to database table." + + for recipient in twilio["recipients"]: + if messaging_service_sid: + client.messages.create( + messaging_service_sid=messaging_service_sid, + to=recipient, + from_=twilio["origin_number"], + body=message, + ) + else: + client.messages.create( + to=recipient, from_=twilio["origin_number"], body=message + ) diff --git a/f/connectors/alerts/alerts_gcs.script.lock b/f/connectors/alerts/alerts_gcs.script.lock index 8985c59..124491f 100644 --- a/f/connectors/alerts/alerts_gcs.script.lock +++ b/f/connectors/alerts/alerts_gcs.script.lock @@ -24,4 +24,5 @@ requests==2.32.3 rsa==4.9 six==1.17.0 tzdata==2024.2 +twilio==9.4.1 urllib3==2.2.3 \ No newline at end of file diff --git a/f/connectors/alerts/alerts_gcs.script.yaml b/f/connectors/alerts/alerts_gcs.script.yaml index 31d0667..15f65aa 100644 --- a/f/connectors/alerts/alerts_gcs.script.yaml +++ b/f/connectors/alerts/alerts_gcs.script.yaml @@ -13,6 +13,7 @@ schema: - db - db_table_name - destination_path + - twilio properties: alerts_bucket: type: string @@ -53,6 +54,14 @@ schema: will be retrieved. This ID is used to filter and process relevant alerts. default: null + twilio: + type: object + description: >- + (Optional) Twilio account credentials and phone numbers for sending alerts + after ingesting and writing alerts to the database. Should include at minimum: + `account_sid`, `auth_token`, `origin_number`, `recipients`. + default: null + format: resource-c_twilio required: - gcp_service_acct - alerts_bucket From 81d5336dd1b49074d002802bbde0086031ab22fc Mon Sep 17 00:00:00 2001 From: Rudo Kemper Date: Tue, 7 Jan 2025 09:14:29 -0500 Subject: [PATCH 02/12] Twilio working --- c_twilio.resource-type.json | 6 ++++++ f/connectors/alerts/alerts_gcs.py | 36 ++++++++++++++++--------------- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/c_twilio.resource-type.json b/c_twilio.resource-type.json index 9943d19..c5a0520 100644 --- a/c_twilio.resource-type.json +++ b/c_twilio.resource-type.json @@ -4,6 +4,7 @@ "account_sid", "auth_token", "message_service_sid", + "content_sid", "origin_number", "forward_numbers" ], @@ -30,6 +31,11 @@ "default": "", "description": "(Optional) The SID for a messaging service, which is a container that bundle messaging functionality for your specific use cases (such as WhatsApp). It can be found in the Messaging Services menu; each service has their own SID. By including it, you can retrieve usage statistics for the service." }, + "content_sid": { + "type": "string", + "default": "", + "description": "The SID for the message content template." + }, "origin_number": { "type": "string", "default": "", diff --git a/f/connectors/alerts/alerts_gcs.py b/f/connectors/alerts/alerts_gcs.py index 9ea729e..ee5437d 100644 --- a/f/connectors/alerts/alerts_gcs.py +++ b/f/connectors/alerts/alerts_gcs.py @@ -112,7 +112,9 @@ def _main( convert_tiffs_to_jpg(tiff_files) - prepared_alerts_metadata = prepare_alerts_metadata(alerts_metadata, territory_id) + prepared_alerts_metadata, alerts_statistics = prepare_alerts_metadata( + alerts_metadata, territory_id + ) prepared_alerts_data = prepare_alerts_data(destination_path, geojson_files) @@ -123,7 +125,7 @@ def _main( ) if twilio: - send_twilio_message(twilio) + send_twilio_message(twilio, alerts_statistics) def _get_rel_filepath(local_file_path, territory_id): @@ -767,24 +769,24 @@ def handle_output(self, alerts, alerts_metadata): conn.close() -def send_twilio_message(twilio): +def send_twilio_message(twilio, alerts_statistics): """ Send a Twilio message to alert the user of the script's completion. """ - messaging_service_sid = twilio.get("messaging_service_sid") client = TwilioClient(twilio["account_sid"], twilio["auth_token"]) - message = "Alerts data successfully written to database table." - for recipient in twilio["recipients"]: - if messaging_service_sid: - client.messages.create( - messaging_service_sid=messaging_service_sid, - to=recipient, - from_=twilio["origin_number"], - body=message, - ) - else: - client.messages.create( - to=recipient, from_=twilio["origin_number"], body=message - ) + client.messages.create( + content_sid=twilio.get("content_sid"), + content_variables=json.dumps( + { + "1": alerts_statistics.get("total_alerts"), + "2": alerts_statistics.get("month_year"), + "3": alerts_statistics.get("type_alert"), + "4": alerts_statistics.get("alerts_dashboard_url"), + } + ), + messaging_service_sid=twilio.get("messaging_service_sid"), + to=recipient, + from_=twilio["origin_number"], + ) From bd70b3a8cf097510fc7886fee6456104fc244048 Mon Sep 17 00:00:00 2001 From: Rudo Kemper Date: Tue, 7 Jan 2025 09:58:48 -0500 Subject: [PATCH 03/12] Change name of Twilio resource type --- ...pe.json => c_twilio_message_template.resource-type.json | 2 ++ f/connectors/alerts/alerts_gcs.script.yaml | 7 +++++++ 2 files changed, 9 insertions(+) rename c_twilio.resource-type.json => c_twilio_message_template.resource-type.json (97%) diff --git a/c_twilio.resource-type.json b/c_twilio_message_template.resource-type.json similarity index 97% rename from c_twilio.resource-type.json rename to c_twilio_message_template.resource-type.json index c5a0520..a0262ee 100644 --- a/c_twilio.resource-type.json +++ b/c_twilio_message_template.resource-type.json @@ -12,6 +12,8 @@ "required": [ "account_sid", "auth_token", + "message_service_sid", + "content_sid", "origin_number", "forward_numbers" ], diff --git a/f/connectors/alerts/alerts_gcs.script.yaml b/f/connectors/alerts/alerts_gcs.script.yaml index 15f65aa..ec838d2 100644 --- a/f/connectors/alerts/alerts_gcs.script.yaml +++ b/f/connectors/alerts/alerts_gcs.script.yaml @@ -14,6 +14,7 @@ schema: - db_table_name - destination_path - twilio + - territory_name properties: alerts_bucket: type: string @@ -54,6 +55,12 @@ schema: will be retrieved. This ID is used to filter and process relevant alerts. default: null + territory_name: + type: string + description: >- + The name of the territory for which change detection alerts will be + retrieved. This name is used for Twilio. + default: null twilio: type: object description: >- From 7f780dddefbd986d0eb3ced11c27cd05b04056e8 Mon Sep 17 00:00:00 2001 From: Rudo Kemper Date: Tue, 7 Jan 2025 09:59:02 -0500 Subject: [PATCH 04/12] Return last date row of alerts statistics --- f/connectors/alerts/alerts_gcs.py | 44 +++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/f/connectors/alerts/alerts_gcs.py b/f/connectors/alerts/alerts_gcs.py index ee5437d..a7cf1c0 100644 --- a/f/connectors/alerts/alerts_gcs.py +++ b/f/connectors/alerts/alerts_gcs.py @@ -27,7 +27,7 @@ # type names that refer to Windmill Resources gcp_service_account = dict postgresql = dict -c_twilio = dict +c_twilio_message_template = dict logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -47,7 +47,8 @@ def main( territory_id: int, db: postgresql, db_table_name: str, - twilio: c_twilio, + territory_name: str, + twilio: c_twilio_message_template, destination_path: str = "/frizzle-persistent-storage/datalake/change_detection/alerts", ): """ @@ -65,6 +66,7 @@ def main( db, db_table_name, destination_path, + territory_name, twilio, ) @@ -76,7 +78,8 @@ def _main( db: postgresql, db_table_name: str, destination_path: str, - twilio: c_twilio, + territory_name: str, + twilio: c_twilio_message_template, ): """Download alerts to warehouse storage and index them in a database. @@ -93,6 +96,8 @@ def _main( The name of the database table to write alerts to. destination_path : str, optional The local directory to save files + territory_name : str + The name of the territory for which alerts are being processed. twilio : dict, optional A dictionary containing Twilio configuration parameters. @@ -125,7 +130,7 @@ def _main( ) if twilio: - send_twilio_message(twilio, alerts_statistics) + send_twilio_message(twilio, alerts_statistics, territory_name) def _get_rel_filepath(local_file_path, territory_id): @@ -350,6 +355,14 @@ def prepare_alerts_metadata(alerts_metadata, territory_id): # Filter DataFrame based on territory_id filtered_df = df.loc[df["territory_id"] == territory_id] + # Group the DataFrame by month and year, and get the last row for each group + filtered_df = ( + filtered_df.loc[df["territory_id"] == territory_id] + .groupby(["month", "year"]) + .last() + .reset_index() + ) + # Hash each row into a unique UUID; this will be used as the primary key for the metadata table # The hash is based on the most important columns for the metadata table, so that changes in other columns do not affect the hash filtered_df["metadata_uuid"] = pd.util.hash_pandas_object( @@ -374,7 +387,16 @@ def prepare_alerts_metadata(alerts_metadata, territory_id): logger.info("Successfully prepared alerts metadata.") - return prepared_alerts_metadata + # Generate alert statistics + # This is assuming that the last row (sorted by month and year) in the filtered DataFrame is the most recent alert + latest_row = filtered_df.iloc[-1] + alert_statistics = { + "total_alerts": str(latest_row["total_alerts"]), + "month_year": f"{latest_row['month']}/{latest_row['year']}", + "description_alerts": latest_row["description_alerts"].replace("_", " "), + } + + return prepared_alerts_metadata, alert_statistics def prepare_alerts_data(local_directory, geojson_files): @@ -769,12 +791,16 @@ def handle_output(self, alerts, alerts_metadata): conn.close() -def send_twilio_message(twilio, alerts_statistics): +def send_twilio_message(twilio, alerts_statistics, territory_name): """ Send a Twilio message to alert the user of the script's completion. """ client = TwilioClient(twilio["account_sid"], twilio["auth_token"]) + logger.info( + f"Sending Twilio messages to {len(twilio.get('recipients', []))} recipients." + ) + for recipient in twilio["recipients"]: client.messages.create( content_sid=twilio.get("content_sid"), @@ -782,11 +808,13 @@ def send_twilio_message(twilio, alerts_statistics): { "1": alerts_statistics.get("total_alerts"), "2": alerts_statistics.get("month_year"), - "3": alerts_statistics.get("type_alert"), - "4": alerts_statistics.get("alerts_dashboard_url"), + "3": alerts_statistics.get("description_alerts"), + "4": f"https://explorer.{territory_name}.guardianconnector.net/alerts/alerts", } ), messaging_service_sid=twilio.get("messaging_service_sid"), to=recipient, from_=twilio["origin_number"], ) + + logger.info("Twilio messages sent successfully.") From c6eb0014e398d59ab4c5ea72c054d516181f15fd Mon Sep 17 00:00:00 2001 From: Rudo Kemper Date: Tue, 7 Jan 2025 10:24:25 -0500 Subject: [PATCH 05/12] Fix to ensure last month and year are returned, and test for this --- f/connectors/alerts/alerts_gcs.py | 41 +++++++++++++++---- f/connectors/alerts/tests/alerts_gcs_test.py | 20 +++++++-- .../alerts/tests/assets/alerts_history.csv | 1 + 3 files changed, 51 insertions(+), 11 deletions(-) diff --git a/f/connectors/alerts/alerts_gcs.py b/f/connectors/alerts/alerts_gcs.py index a7cf1c0..1716f9b 100644 --- a/f/connectors/alerts/alerts_gcs.py +++ b/f/connectors/alerts/alerts_gcs.py @@ -47,9 +47,9 @@ def main( territory_id: int, db: postgresql, db_table_name: str, - territory_name: str, - twilio: c_twilio_message_template, destination_path: str = "/frizzle-persistent-storage/datalake/change_detection/alerts", + territory_name: str = None, + twilio: c_twilio_message_template = None, ): """ Wrapper around _main() that instantiates the GCP client. @@ -78,8 +78,8 @@ def _main( db: postgresql, db_table_name: str, destination_path: str, - territory_name: str, - twilio: c_twilio_message_template, + territory_name: str = None, + twilio: c_twilio_message_template = None, ): """Download alerts to warehouse storage and index them in a database. @@ -96,7 +96,7 @@ def _main( The name of the database table to write alerts to. destination_path : str, optional The local directory to save files - territory_name : str + territory_name : str, optional The name of the territory for which alerts are being processed. twilio : dict, optional A dictionary containing Twilio configuration parameters. @@ -335,6 +335,12 @@ def prepare_alerts_metadata(alerts_metadata, territory_id): a unique UUID for the metadata based on the content hash and includes a placeholder geolocation. + The alert statistics dictionary is generated based on the assumption that + the first row (after sorting by month and year in descending order) in the + filtered DataFrame represents the most recent alert. In other words, it is + assumed that the latest alerts posted by the provider are always for the + latest month and year in the dataset. + Parameters ---------- alerts_metadata : str @@ -348,6 +354,9 @@ def prepare_alerts_metadata(alerts_metadata, territory_id): A list of dictionaries representing the filtered and processed alerts metadata, including additional columns for geolocation, metadata UUID, and alert source. + dict + A dictionary containing alert statistics such as total alerts, month/year, + and description of alerts. """ # Convert CSV bytes to DataFrame df = pd.read_csv(StringIO(alerts_metadata)) @@ -363,6 +372,11 @@ def prepare_alerts_metadata(alerts_metadata, territory_id): .reset_index() ) + # Sort the DataFrame by year and month in descending order + filtered_df = filtered_df.sort_values( + by=["year", "month"], ascending=[False, False] + ) + # Hash each row into a unique UUID; this will be used as the primary key for the metadata table # The hash is based on the most important columns for the metadata table, so that changes in other columns do not affect the hash filtered_df["metadata_uuid"] = pd.util.hash_pandas_object( @@ -388,8 +402,7 @@ def prepare_alerts_metadata(alerts_metadata, territory_id): logger.info("Successfully prepared alerts metadata.") # Generate alert statistics - # This is assuming that the last row (sorted by month and year) in the filtered DataFrame is the most recent alert - latest_row = filtered_df.iloc[-1] + latest_row = filtered_df.iloc[0] alert_statistics = { "total_alerts": str(latest_row["total_alerts"]), "month_year": f"{latest_row['month']}/{latest_row['year']}", @@ -793,7 +806,19 @@ def handle_output(self, alerts, alerts_metadata): def send_twilio_message(twilio, alerts_statistics, territory_name): """ - Send a Twilio message to alert the user of the script's completion. + Send a Twilio SMS message with alerts processing completion details. + + Parameters + ---------- + twilio : dict + A dictionary containing Twilio configuration parameters, including + account credentials, messaging service details, and recipient phone + numbers. + alerts_statistics : dict + A dictionary containing statistics about the processed alerts, such as + the total number of alerts, month and year, and a description. + territory_name : str + The name of the territory for which the alerts were processed. """ client = TwilioClient(twilio["account_sid"], twilio["auth_token"]) diff --git a/f/connectors/alerts/tests/alerts_gcs_test.py b/f/connectors/alerts/tests/alerts_gcs_test.py index 2601921..1de2413 100644 --- a/f/connectors/alerts/tests/alerts_gcs_test.py +++ b/f/connectors/alerts/tests/alerts_gcs_test.py @@ -5,14 +5,29 @@ from pathlib import Path import google.api_core.exceptions +import pandas as pd import psycopg2 import pytest -from f.connectors.alerts.alerts_gcs import _main +from f.connectors.alerts.alerts_gcs import _main, prepare_alerts_metadata logger = logging.getLogger(__name__) MOCK_BUCKET_NAME = "test-bucket" +assets_directory = "f/connectors/alerts/tests/assets/" + + +def test_prepare_alerts_metadata(): + alerts_history_csv = Path(assets_directory, "alerts_history.csv") + alerts_metadata = pd.read_csv(alerts_history_csv).to_csv(index=False) + + prepared_alerts_metadata, alert_statistics = prepare_alerts_metadata( + alerts_metadata, 100 + ) + + assert alert_statistics["month_year"] == "2/2024" + assert alert_statistics["total_alerts"] == "1" + assert alert_statistics["description_alerts"] == "fake alert" @pytest.fixture @@ -32,7 +47,6 @@ def _upload_blob(bucket, source_file_name, destination_blob_name): logger.info(f"Uploaded {source_file_name} -> {destination_blob_name}") # Upload test files to the emulator - assets_directory = "f/connectors/alerts/tests/assets/" alerts_filenames = [ "alerts_history.csv", "100/vector/2023/09/alert_202309900112345671.geojson", @@ -74,7 +88,7 @@ def test_script_e2e(pg_database, mock_alerts_storage_client, tmp_path): # Count of unique rows in alerts_history.csv based on UUID # The last row in the CSV is a duplicate of the one before it, but updates the confidence field, hence shares the same UUID cursor.execute("SELECT COUNT(*) FROM fake_alerts__metadata") - assert cursor.fetchone()[0] == 5 + assert cursor.fetchone()[0] == 6 # Check that the confidence field is NULL if it is not defined in the CSV cursor.execute( diff --git a/f/connectors/alerts/tests/assets/alerts_history.csv b/f/connectors/alerts/tests/assets/alerts_history.csv index f0eec56..98b565f 100644 --- a/f/connectors/alerts/tests/assets/alerts_history.csv +++ b/f/connectors/alerts/tests/assets/alerts_history.csv @@ -2,6 +2,7 @@ territory_id,type_alert,month,year,total_alerts,description_alerts,confidence 100,001,09,2023,84,fake_alert,1 100,001,10,2023,9,fake_alert,0 100,001,11,2023,3,fake_alert,0.5 +100,001,02,2024,1,fake_alert,1 100,001,12,2022,3,fake_alert, 100,001,01,2024,0,fake_alert,1 100,001,01,2024,0,fake_alert,0 \ No newline at end of file From 1e710ecea2f98009d3d07f2841bd2df9a8fbef9e Mon Sep 17 00:00:00 2001 From: Rudo Kemper Date: Tue, 7 Jan 2025 10:44:11 -0500 Subject: [PATCH 06/12] Fix: var names and documentation --- c_twilio_message_template.resource-type.json | 1 - f/connectors/alerts/alerts_gcs.py | 8 ++++---- f/connectors/alerts/alerts_gcs.script.yaml | 11 ++++++----- f/connectors/alerts/tests/alerts_gcs_test.py | 1 + 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/c_twilio_message_template.resource-type.json b/c_twilio_message_template.resource-type.json index a0262ee..fa92164 100644 --- a/c_twilio_message_template.resource-type.json +++ b/c_twilio_message_template.resource-type.json @@ -12,7 +12,6 @@ "required": [ "account_sid", "auth_token", - "message_service_sid", "content_sid", "origin_number", "forward_numbers" diff --git a/f/connectors/alerts/alerts_gcs.py b/f/connectors/alerts/alerts_gcs.py index 1716f9b..7c239cf 100644 --- a/f/connectors/alerts/alerts_gcs.py +++ b/f/connectors/alerts/alerts_gcs.py @@ -97,7 +97,7 @@ def _main( destination_path : str, optional The local directory to save files territory_name : str, optional - The name of the territory for which alerts are being processed. + The slug of the territory for which alerts are being processed. twilio : dict, optional A dictionary containing Twilio configuration parameters. @@ -129,7 +129,7 @@ def _main( f"Alerts data successfully written to database table: [{db_table_name}]" ) - if twilio: + if twilio and alerts_statistics and territory_name: send_twilio_message(twilio, alerts_statistics, territory_name) @@ -355,7 +355,7 @@ def prepare_alerts_metadata(alerts_metadata, territory_id): metadata, including additional columns for geolocation, metadata UUID, and alert source. dict - A dictionary containing alert statistics such as total alerts, month/year, + A dictionary containing alert statistics: total alerts, month/year, and description of alerts. """ # Convert CSV bytes to DataFrame @@ -818,7 +818,7 @@ def send_twilio_message(twilio, alerts_statistics, territory_name): A dictionary containing statistics about the processed alerts, such as the total number of alerts, month and year, and a description. territory_name : str - The name of the territory for which the alerts were processed. + The slug of the territory for which alerts are being processed. """ client = TwilioClient(twilio["account_sid"], twilio["auth_token"]) diff --git a/f/connectors/alerts/alerts_gcs.script.yaml b/f/connectors/alerts/alerts_gcs.script.yaml index ec838d2..134a6a6 100644 --- a/f/connectors/alerts/alerts_gcs.script.yaml +++ b/f/connectors/alerts/alerts_gcs.script.yaml @@ -58,17 +58,18 @@ schema: territory_name: type: string description: >- - The name of the territory for which change detection alerts will be - retrieved. This name is used for Twilio. + The URL slug for the territory, used to construct a link to the alerts + dashboard. This name is used for Twilio and must be provided for messages to + be sent. default: null twilio: type: object description: >- (Optional) Twilio account credentials and phone numbers for sending alerts - after ingesting and writing alerts to the database. Should include at minimum: - `account_sid`, `auth_token`, `origin_number`, `recipients`. + after ingesting and writing alerts to the database via a Twilio message content + template. default: null - format: resource-c_twilio + format: resource-c_twilio_message_template required: - gcp_service_acct - alerts_bucket diff --git a/f/connectors/alerts/tests/alerts_gcs_test.py b/f/connectors/alerts/tests/alerts_gcs_test.py index 1de2413..ab2b603 100644 --- a/f/connectors/alerts/tests/alerts_gcs_test.py +++ b/f/connectors/alerts/tests/alerts_gcs_test.py @@ -25,6 +25,7 @@ def test_prepare_alerts_metadata(): alerts_metadata, 100 ) + # Check that alerts statistics is the latest month and year in the CSV assert alert_statistics["month_year"] == "2/2024" assert alert_statistics["total_alerts"] == "1" assert alert_statistics["description_alerts"] == "fake alert" From 248182fd63ddd9de83c69e6a65c9e4ad1bddc8c4 Mon Sep 17 00:00:00 2001 From: Rudo Kemper Date: Tue, 7 Jan 2025 11:00:10 -0500 Subject: [PATCH 07/12] Add Twilio message option to readme --- f/connectors/alerts/README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/f/connectors/alerts/README.md b/f/connectors/alerts/README.md index 449cd5b..0443baf 100644 --- a/f/connectors/alerts/README.md +++ b/f/connectors/alerts/README.md @@ -1,8 +1,10 @@ # Google Cloud Alerts Change Detection Integration -This script fetches change detection alerts and images from a storage bucket on Google Cloud Platform. The script transforms the data for SQL compatibility and stores it in a PostgreSQL database. Additionally, it saves before-and-after images -- as TIF and JPEG -- to a specified directory. +This script fetches change detection alerts and images from a storage bucket on Google Cloud Platform. The script transforms the data for SQL compatibility and stores it in a PostgreSQL database. Additionally, it saves before-and-after images -- as TIF and JPEG -- to a specified directory. -## API Queries +Optionally, the script can send a Twilio WhatsApp message with a summary of the latest processed alerts. + +## GCP API Queries Change Detection alerts can be stored in a Google Cloud storage bucket. @@ -14,8 +16,7 @@ Google Cloud storage has a built-in API solution, [GCS JSON API](https://cloud.g Change detection alert files are currently stored on GCP in this format: -**Vector:** -``` +**Vector:**``` /vector///alert_.geojson ``` @@ -42,4 +43,4 @@ Currently, we are assuming there to be only four raster images for each change d ////images/_T0_.tif ////images/_T0_.jpg ... -``` \ No newline at end of file +``` From 71550de2931c79cb8bd000b698ad0ce2441d0cb3 Mon Sep 17 00:00:00 2001 From: Rudo Kemper Date: Tue, 7 Jan 2025 11:14:34 -0500 Subject: [PATCH 08/12] Expand twilio function docstring --- f/connectors/alerts/alerts_gcs.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/f/connectors/alerts/alerts_gcs.py b/f/connectors/alerts/alerts_gcs.py index 7c239cf..4033572 100644 --- a/f/connectors/alerts/alerts_gcs.py +++ b/f/connectors/alerts/alerts_gcs.py @@ -808,6 +808,14 @@ def send_twilio_message(twilio, alerts_statistics, territory_name): """ Send a Twilio SMS message with alerts processing completion details. + The message template is defined in the Twilio console, and is structured as follows: + {{1}} new change detection alert(s) have been published on your alerts dashboard for + the date of {{2}}. The following activities have been detected in your region: {{3}}. + Visit your alerts dashboard here: {{4}} + + In the content_variables below, the placeholders {{1}}, {{2}}, {{3}}, and {{4}} are + replaced with the corresponding values from the alerts_statistics dictionary. + Parameters ---------- twilio : dict @@ -822,6 +830,7 @@ def send_twilio_message(twilio, alerts_statistics, territory_name): """ client = TwilioClient(twilio["account_sid"], twilio["auth_token"]) + # Send a message to each recipient logger.info( f"Sending Twilio messages to {len(twilio.get('recipients', []))} recipients." ) From e5a4ac14c820d6bfe9d7512b4e10508ca7b96a41 Mon Sep 17 00:00:00 2001 From: Rudo Kemper Date: Thu, 16 Jan 2025 14:34:37 -0500 Subject: [PATCH 09/12] Refactor: turn twilio into its own script to be used in Flow --- f/connectors/alerts/README.md | 6 +- f/connectors/alerts/alerts_gcs.py | 80 ++----------------- f/connectors/alerts/alerts_gcs.script.lock | 1 - f/connectors/alerts/alerts_gcs.script.yaml | 17 ---- f/connectors/alerts/alerts_twilio.py | 70 ++++++++++++++++ f/connectors/alerts/alerts_twilio.script.lock | 1 + f/connectors/alerts/alerts_twilio.script.yaml | 38 +++++++++ f/connectors/alerts/tests/alerts_gcs_test.py | 6 +- 8 files changed, 126 insertions(+), 93 deletions(-) create mode 100644 f/connectors/alerts/alerts_twilio.py create mode 100644 f/connectors/alerts/alerts_twilio.script.lock create mode 100644 f/connectors/alerts/alerts_twilio.script.yaml diff --git a/f/connectors/alerts/README.md b/f/connectors/alerts/README.md index 0443baf..54787b1 100644 --- a/f/connectors/alerts/README.md +++ b/f/connectors/alerts/README.md @@ -2,8 +2,6 @@ This script fetches change detection alerts and images from a storage bucket on Google Cloud Platform. The script transforms the data for SQL compatibility and stores it in a PostgreSQL database. Additionally, it saves before-and-after images -- as TIF and JPEG -- to a specified directory. -Optionally, the script can send a Twilio WhatsApp message with a summary of the latest processed alerts. - ## GCP API Queries Change Detection alerts can be stored in a Google Cloud storage bucket. @@ -44,3 +42,7 @@ Currently, we are assuming there to be only four raster images for each change d ////images/_T0_.jpg ... ``` + +# Send a Twilio Message + +This script can send a Twilio WhatsApp message with a summary of the latest processed alerts. diff --git a/f/connectors/alerts/alerts_gcs.py b/f/connectors/alerts/alerts_gcs.py index ab6712c..dc9722e 100644 --- a/f/connectors/alerts/alerts_gcs.py +++ b/f/connectors/alerts/alerts_gcs.py @@ -6,7 +6,6 @@ # pillow~=10.3 # psycopg2-binary # requests~=2.32 -# twilio~=9.4 import base64 import hashlib @@ -22,12 +21,10 @@ from google.oauth2.service_account import Credentials from PIL import Image from psycopg2 import sql -from twilio.rest import Client as TwilioClient # type names that refer to Windmill Resources gcp_service_account = dict postgresql = dict -c_twilio_message_template = dict logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -48,8 +45,6 @@ def main( db: postgresql, db_table_name: str, destination_path: str = "/frizzle-persistent-storage/datalake/change_detection/alerts", - territory_name: str = None, - twilio: c_twilio_message_template = None, ): """ Wrapper around _main() that instantiates the GCP client. @@ -66,8 +61,6 @@ def main( db, db_table_name, destination_path, - territory_name, - twilio, ) @@ -78,8 +71,6 @@ def _main( db: postgresql, db_table_name: str, destination_path: str, - territory_name: str = None, - twilio: c_twilio_message_template = None, ): """Download alerts to warehouse storage and index them in a database. @@ -96,10 +87,6 @@ def _main( The name of the database table to write alerts to. destination_path : str, optional The local directory to save files - territory_name : str, optional - The slug of the territory for which alerts are being processed. - twilio : dict, optional - A dictionary containing Twilio configuration parameters. Returns ------- @@ -129,8 +116,7 @@ def _main( f"Alerts data successfully written to database table: [{db_table_name}]" ) - if twilio and alerts_statistics and territory_name: - send_twilio_message(twilio, alerts_statistics, territory_name) + return alerts_statistics def _get_rel_filepath(local_file_path, territory_id): @@ -211,9 +197,9 @@ def sync_gcs_to_local( prefix = f"{territory_id}/" files_to_download = set(blob.name for blob in bucket.list_blobs(prefix=prefix)) - assert ( - len(files_to_download) > 0 - ), f"No files found to download in bucket '{bucket_name}' with that prefix." + assert len(files_to_download) > 0, ( + f"No files found to download in bucket '{bucket_name}' with that prefix." + ) logger.info( f"Found {len(files_to_download)} files to download from bucket '{bucket_name}'." @@ -350,11 +336,11 @@ def prepare_alerts_metadata(alerts_metadata, territory_id): Returns ------- - list of dict + prepared_alerts_metadata : list of dict A list of dictionaries representing the filtered and processed alerts metadata, including additional columns for geolocation, metadata UUID, and alert source. - dict + alerts_statistics : dict A dictionary containing alert statistics: total alerts, month/year, and description of alerts. """ @@ -403,13 +389,13 @@ def prepare_alerts_metadata(alerts_metadata, territory_id): # Generate alert statistics latest_row = filtered_df.iloc[0] - alert_statistics = { + alerts_statistics = { "total_alerts": str(latest_row["total_alerts"]), "month_year": f"{latest_row['month']}/{latest_row['year']}", "description_alerts": latest_row["description_alerts"].replace("_", " "), } - return prepared_alerts_metadata, alert_statistics + return prepared_alerts_metadata, alerts_statistics def prepare_alerts_data(local_directory, geojson_files): @@ -802,53 +788,3 @@ def handle_output(self, alerts, alerts_metadata): logger.info(f"Total alert rows updated: {updated_count}") cursor.close() conn.close() - - -def send_twilio_message(twilio, alerts_statistics, territory_name): - """ - Send a Twilio SMS message with alerts processing completion details. - - The message template is defined in the Twilio console, and is structured as follows: - {{1}} new change detection alert(s) have been published on your alerts dashboard for - the date of {{2}}. The following activities have been detected in your region: {{3}}. - Visit your alerts dashboard here: {{4}} - - In the content_variables below, the placeholders {{1}}, {{2}}, {{3}}, and {{4}} are - replaced with the corresponding values from the alerts_statistics dictionary. - - Parameters - ---------- - twilio : dict - A dictionary containing Twilio configuration parameters, including - account credentials, messaging service details, and recipient phone - numbers. - alerts_statistics : dict - A dictionary containing statistics about the processed alerts, such as - the total number of alerts, month and year, and a description. - territory_name : str - The slug of the territory for which alerts are being processed. - """ - client = TwilioClient(twilio["account_sid"], twilio["auth_token"]) - - # Send a message to each recipient - logger.info( - f"Sending Twilio messages to {len(twilio.get('recipients', []))} recipients." - ) - - for recipient in twilio["recipients"]: - client.messages.create( - content_sid=twilio.get("content_sid"), - content_variables=json.dumps( - { - "1": alerts_statistics.get("total_alerts"), - "2": alerts_statistics.get("month_year"), - "3": alerts_statistics.get("description_alerts"), - "4": f"https://explorer.{territory_name}.guardianconnector.net/alerts/alerts", - } - ), - messaging_service_sid=twilio.get("messaging_service_sid"), - to=recipient, - from_=twilio["origin_number"], - ) - - logger.info("Twilio messages sent successfully.") diff --git a/f/connectors/alerts/alerts_gcs.script.lock b/f/connectors/alerts/alerts_gcs.script.lock index 124491f..8985c59 100644 --- a/f/connectors/alerts/alerts_gcs.script.lock +++ b/f/connectors/alerts/alerts_gcs.script.lock @@ -24,5 +24,4 @@ requests==2.32.3 rsa==4.9 six==1.17.0 tzdata==2024.2 -twilio==9.4.1 urllib3==2.2.3 \ No newline at end of file diff --git a/f/connectors/alerts/alerts_gcs.script.yaml b/f/connectors/alerts/alerts_gcs.script.yaml index 134a6a6..31d0667 100644 --- a/f/connectors/alerts/alerts_gcs.script.yaml +++ b/f/connectors/alerts/alerts_gcs.script.yaml @@ -13,8 +13,6 @@ schema: - db - db_table_name - destination_path - - twilio - - territory_name properties: alerts_bucket: type: string @@ -55,21 +53,6 @@ schema: will be retrieved. This ID is used to filter and process relevant alerts. default: null - territory_name: - type: string - description: >- - The URL slug for the territory, used to construct a link to the alerts - dashboard. This name is used for Twilio and must be provided for messages to - be sent. - default: null - twilio: - type: object - description: >- - (Optional) Twilio account credentials and phone numbers for sending alerts - after ingesting and writing alerts to the database via a Twilio message content - template. - default: null - format: resource-c_twilio_message_template required: - gcp_service_acct - alerts_bucket diff --git a/f/connectors/alerts/alerts_twilio.py b/f/connectors/alerts/alerts_twilio.py new file mode 100644 index 0000000..8918080 --- /dev/null +++ b/f/connectors/alerts/alerts_twilio.py @@ -0,0 +1,70 @@ +# twilio~=9.4 + +import json +import logging + +from twilio.rest import Client as TwilioClient + +# type names that refer to Windmill Resources +c_twilio_message_template = dict + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def main( + alerts_statistics: dict, + territory_name: str, + twilio: c_twilio_message_template, +): + send_twilio_message(twilio, alerts_statistics, territory_name) + + +def send_twilio_message(twilio, alerts_statistics, territory_name): + """ + Send a Twilio SMS message with alerts processing completion details. + + The message template is defined in the Twilio console, and is structured as follows: + {{1}} new change detection alert(s) have been published on your alerts dashboard for + the date of {{2}}. The following activities have been detected in your region: {{3}}. + Visit your alerts dashboard here: {{4}} + + In the content_variables below, the placeholders {{1}}, {{2}}, {{3}}, and {{4}} are + replaced with the corresponding values from the alerts_statistics dictionary. + + Parameters + ---------- + twilio : dict + A dictionary containing Twilio configuration parameters, including + account credentials, messaging service details, and recipient phone + numbers. + alerts_statistics : dict + A dictionary containing statistics about the processed alerts, such as + the total number of alerts, month and year, and a description. + territory_name : str + The slug of the territory for which alerts are being processed. + """ + client = TwilioClient(twilio["account_sid"], twilio["auth_token"]) + + # Send a message to each recipient + logger.info( + f"Sending Twilio messages to {len(twilio.get('recipients', []))} recipients." + ) + + for recipient in twilio["recipients"]: + client.messages.create( + content_sid=twilio.get("content_sid"), + content_variables=json.dumps( + { + "1": alerts_statistics.get("total_alerts"), + "2": alerts_statistics.get("month_year"), + "3": alerts_statistics.get("description_alerts"), + "4": f"https://explorer.{territory_name}.guardianconnector.net/alerts/alerts", + } + ), + messaging_service_sid=twilio.get("messaging_service_sid"), + to=recipient, + from_=twilio["origin_number"], + ) + + logger.info("Twilio messages sent successfully.") diff --git a/f/connectors/alerts/alerts_twilio.script.lock b/f/connectors/alerts/alerts_twilio.script.lock new file mode 100644 index 0000000..c2b1a3d --- /dev/null +++ b/f/connectors/alerts/alerts_twilio.script.lock @@ -0,0 +1 @@ +twilio==9.4.1 diff --git a/f/connectors/alerts/alerts_twilio.script.yaml b/f/connectors/alerts/alerts_twilio.script.yaml new file mode 100644 index 0000000..c588ba3 --- /dev/null +++ b/f/connectors/alerts/alerts_twilio.script.yaml @@ -0,0 +1,38 @@ +summary: 'Alerts: Send a Twilio Message' +description: Send a Twilio WhatsApp message with a summary of the latest processed alerts. +lock: '!inline f/connectors/alerts/alerts_twilio.script.lock' +concurrency_time_window_s: 0 +kind: script +schema: + $schema: 'https://json-schema.org/draft/2020-12/schema' + type: object + order: + - alerts_statistics + - twilio + - territory_name + properties: + alerts_statistics: + type: object + description: >- + An object containing alert statistics: total alerts, month/year, + and description of alerts. + default: {} + territory_name: + type: string + description: >- + The URL slug for the territory, used to construct a link to the alerts + dashboard. This name is used for Twilio and must be provided for messages to + be sent. + default: null + twilio: + type: object + description: >- + Twilio account credentials and phone numbers for sending alerts + after ingesting and writing alerts to the database via a Twilio message content + template. + default: null + format: resource-c_twilio_message_template + required: + - alerts_statistics + - twilio + - territory_name \ No newline at end of file diff --git a/f/connectors/alerts/tests/alerts_gcs_test.py b/f/connectors/alerts/tests/alerts_gcs_test.py index ab2b603..3fca2bc 100644 --- a/f/connectors/alerts/tests/alerts_gcs_test.py +++ b/f/connectors/alerts/tests/alerts_gcs_test.py @@ -66,7 +66,7 @@ def _upload_blob(bucket, source_file_name, destination_blob_name): def test_script_e2e(pg_database, mock_alerts_storage_client, tmp_path): asset_storage = tmp_path / "datalake" - _main( + alerts_metadata = _main( mock_alerts_storage_client, MOCK_BUCKET_NAME, 100, @@ -122,6 +122,10 @@ def test_script_e2e(pg_database, mock_alerts_storage_client, tmp_path): # Alerts metadata is not saved to disk assert not (asset_storage / "alerts_history.csv").exists() + # Check that the alerts metadata is returned by the script. The unit test for prepare_alerts_metadata() + # checks the correctness of the metadata, so here we just check that it is not None + assert alerts_metadata is not None + def test_file_update_logic(pg_database, mock_alerts_storage_client, tmp_path): asset_storage = tmp_path / "datalake" From fe79f593b35aca9bb0c167e3dcdad631f39b9dfc Mon Sep 17 00:00:00 2001 From: Rudo Kemper Date: Thu, 16 Jan 2025 15:29:10 -0500 Subject: [PATCH 10/12] Only return alerts_statistics if new rows have been written --- f/connectors/alerts/alerts_gcs.py | 41 ++++++++++++++------ f/connectors/alerts/tests/alerts_gcs_test.py | 13 +++++++ 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/f/connectors/alerts/alerts_gcs.py b/f/connectors/alerts/alerts_gcs.py index dc9722e..d3ffa14 100644 --- a/f/connectors/alerts/alerts_gcs.py +++ b/f/connectors/alerts/alerts_gcs.py @@ -111,12 +111,12 @@ def _main( prepared_alerts_data = prepare_alerts_data(destination_path, geojson_files) db_writer = AlertsDBWriter(conninfo(db), db_table_name) - db_writer.handle_output(prepared_alerts_data, prepared_alerts_metadata) - logger.info( - f"Alerts data successfully written to database table: [{db_table_name}]" + new_alerts_data = db_writer.handle_output( + prepared_alerts_data, prepared_alerts_metadata ) - return alerts_statistics + if new_alerts_data: + return alerts_statistics def _get_rel_filepath(local_file_path, territory_id): @@ -557,6 +557,9 @@ def _safe_insert(cursor, table_name, columns, values): cursor.execute(select_query, (values[columns.index("_id")],)) existing_row = cursor.fetchone() + logger.debug(f"Existing row: {existing_row}") + logger.debug(f"List(existing_row): {list(existing_row)}") + logger.debug(f"Values: {values}") if existing_row and list(existing_row) == values: # No changes, skip the update return inserted_count, updated_count @@ -602,8 +605,11 @@ def handle_output(self, alerts, alerts_metadata): conn = self._get_conn() cursor = conn.cursor() - inserted_count = 0 - updated_count = 0 + new_alerts_data = False + data_inserted_count = 0 + data_updated_count = 0 + metadata_inserted_count = 0 + metadata_updated_count = 0 try: if alerts: @@ -714,8 +720,8 @@ def handle_output(self, alerts, alerts_metadata): result_inserted_count, result_updated_count = ( self._safe_insert(cursor, table_name, columns, values) ) - inserted_count += result_inserted_count - updated_count += result_updated_count + data_inserted_count += result_inserted_count + data_updated_count += result_updated_count except Exception: logger.exception( @@ -771,9 +777,14 @@ def handle_output(self, alerts, alerts_metadata): data_source, ] - self._safe_insert( - cursor, f"{table_name}__metadata", columns, values + result_inserted_count, result_updated_count = self._safe_insert( + cursor, + f"{table_name}__metadata", + columns, + values, ) + metadata_inserted_count += result_inserted_count + metadata_updated_count += result_updated_count conn.commit() except Exception: @@ -784,7 +795,13 @@ def handle_output(self, alerts, alerts_metadata): raise finally: - logger.info(f"Total alert rows inserted: {inserted_count}") - logger.info(f"Total alert rows updated: {updated_count}") + logger.info(f"Total alert rows inserted: {data_inserted_count}") + logger.info(f"Total alert rows updated: {data_updated_count}") + logger.info(f"Total metadata rows inserted: {metadata_inserted_count}") + logger.info(f"Total metadata rows updated: {metadata_updated_count}") cursor.close() conn.close() + + if data_inserted_count > 0 or metadata_inserted_count > 0: + new_alerts_data = True + return new_alerts_data diff --git a/f/connectors/alerts/tests/alerts_gcs_test.py b/f/connectors/alerts/tests/alerts_gcs_test.py index 3fca2bc..afc5888 100644 --- a/f/connectors/alerts/tests/alerts_gcs_test.py +++ b/f/connectors/alerts/tests/alerts_gcs_test.py @@ -126,6 +126,19 @@ def test_script_e2e(pg_database, mock_alerts_storage_client, tmp_path): # checks the correctness of the metadata, so here we just check that it is not None assert alerts_metadata is not None + # Now, let's run the script again to check if alerts_metadata is returned (it should be None, + # since no new alerts data or metadata has been inserted into the database) + alerts_metadata = _main( + mock_alerts_storage_client, + MOCK_BUCKET_NAME, + 100, + pg_database, + "fake_alerts", + asset_storage, + ) + + assert alerts_metadata is None + def test_file_update_logic(pg_database, mock_alerts_storage_client, tmp_path): asset_storage = tmp_path / "datalake" From e142a64b7162374fe21592d7bced1db1b7f61dcd Mon Sep 17 00:00:00 2001 From: Rudo Kemper Date: Thu, 16 Jan 2025 15:29:17 -0500 Subject: [PATCH 11/12] Improve readme --- f/connectors/alerts/README.md | 41 ++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/f/connectors/alerts/README.md b/f/connectors/alerts/README.md index 54787b1..053f3f7 100644 --- a/f/connectors/alerts/README.md +++ b/f/connectors/alerts/README.md @@ -1,4 +1,4 @@ -# Google Cloud Alerts Change Detection Integration +# `alerts_gcs`: Google Cloud Alerts Change Detection Integration This script fetches change detection alerts and images from a storage bucket on Google Cloud Platform. The script transforms the data for SQL compatibility and stores it in a PostgreSQL database. Additionally, it saves before-and-after images -- as TIF and JPEG -- to a specified directory. @@ -14,35 +14,36 @@ Google Cloud storage has a built-in API solution, [GCS JSON API](https://cloud.g Change detection alert files are currently stored on GCP in this format: -**Vector:**``` -/vector///alert_.geojson -``` +**Vector:** + + /vector///alert_.geojson + **Raster:** -``` -/raster///_T0_.tif -/raster///_T1_.tif -/raster///_T0_.tif -/raster///_T1_.tif -``` + + /raster///_T0_.tif + /raster///_T1_.tif + /raster///_T0_.tif + /raster///_T1_.tif ### Warehouse **Vector:** -``` -////alert_.geojson -``` + + ////alert_.geojson **Raster:** Currently, we are assuming there to be only four raster images for each change detection alert: a 'before' and 'after' used for detection and visualization, respectively. Each of these is saved in both TIFF and JPEG format in the following way: -``` -////images/_T0_.tif -////images/_T0_.jpg -... -``` + ////images/_T0_.tif + ////images/_T0_.jpg + ... + +# `alerts_twilio`: Send a Twilio Message -# Send a Twilio Message +This script leverages Twilio to send a WhatsApp message to recipients with a summary of the latest processed alerts. Below is the message template, with values from an `alerts_statistics` object: -This script can send a Twilio WhatsApp message with a summary of the latest processed alerts. +```javascript +`${total_alerts} new change detection alert(s) have been published on your alerts dashboard for the date of ${month_year}. The following activities have been detected in your region: ${description_alerts}. Visit your alerts dashboard here: https://explorer.${territory_name}.guardianconnector.net/alerts/alerts` +``` \ No newline at end of file From cc01ad4ed3f80766a170d3001dd8a5328c8ff2bc Mon Sep 17 00:00:00 2001 From: Rudo Kemper Date: Thu, 16 Jan 2025 15:32:21 -0500 Subject: [PATCH 12/12] Remove logs --- f/connectors/alerts/alerts_gcs.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/f/connectors/alerts/alerts_gcs.py b/f/connectors/alerts/alerts_gcs.py index d3ffa14..86e454f 100644 --- a/f/connectors/alerts/alerts_gcs.py +++ b/f/connectors/alerts/alerts_gcs.py @@ -557,9 +557,6 @@ def _safe_insert(cursor, table_name, columns, values): cursor.execute(select_query, (values[columns.index("_id")],)) existing_row = cursor.fetchone() - logger.debug(f"Existing row: {existing_row}") - logger.debug(f"List(existing_row): {list(existing_row)}") - logger.debug(f"Values: {values}") if existing_row and list(existing_row) == values: # No changes, skip the update return inserted_count, updated_count