Add message parts column to notifications table

Makefile

@@ -18,7 +18,6 @@ bootstrap: ## Set up everything to run the app
 	poetry install --sync --no-root
 	poetry run pre-commit install
 	createdb notification_api || true
-	createdb test_notification_api || true
 	(poetry run flask db upgrade) || true
 
 .PHONY: bootstrap-with-docker

app/dao/notifications_dao.py

@@ -15,6 +15,7 @@
 from app.utils import (
     escape_special_characters,
     get_midnight_in_utc,
+    hilite,
     midnight_n_days_ago,
 )
 from notifications_utils.international_billing_rates import INTERNATIONAL_BILLING_RATES
@@ -53,6 +54,7 @@ def dao_get_last_date_template_was_used(template_id, service_id):
 
 @autocommit
 def dao_create_notification(notification):
+    print(hilite("about to add noti to db"))
     if not notification.id:
         # need to populate defaulted fields before we create the notification history object
         notification.id = create_uuid()
@@ -71,6 +73,7 @@ def dao_create_notification(notification):
     notification.to = "1"
     notification.normalised_to = "1"
     db.session.add(notification)
+    print(hilite("added noti to db"))
 
 
 def country_records_delivery(phone_prefix):
@@ -671,3 +674,10 @@ def get_service_ids_with_notifications_on_date(notification_type, date):
             union(notification_table_query, ft_status_table_query).subquery()
         ).distinct()
     }
+
+
+def dao_get_notification_message_parts_by_job_id(job_id):
+    total_message_parts = db.session.query(
+        functions.sum(Notification.message_parts.label("message_parts"))
+    ).filter(Notification.job_id == job_id)
+    return total_message_parts

app/job/rest.py

@@ -108,7 +108,6 @@ def get_all_notifications_for_service_job(service_id, job_id):
         notifications = notification_with_template_schema.dump(
             paginated_notifications.items, many=True
         )
-
     return (
         jsonify(
             notifications=notifications,
@@ -204,17 +203,23 @@ def create_job(service_id):
 
     dao_create_job(job)
 
+    job_json = job_schema.dump(job)
+    job_json["statistics"] = []
+
+    return jsonify(data=job_json), 201
+
+
+@job_blueprint.route("/<job_id>/start-job", methods=["POST"])
+def start_job(service_id, job_id):
+    job = dao_get_job_by_service_id_and_job_id(service_id, job_id)
+    data = request.get_json()
     sender_id = data.get("sender_id")
     # Kick off job in tasks.py
     if job.job_status == JobStatus.PENDING:
         process_job.apply_async(
             [str(job.id)], {"sender_id": sender_id}, queue=QueueNames.JOBS
         )
-
-    job_json = job_schema.dump(job)
-    job_json["statistics"] = []
-
-    return jsonify(data=job_json), 201
+    return {}, 201
 
 
 @job_blueprint.route("/scheduled-job-stats", methods=["GET"])

app/models.py

@@ -1503,6 +1503,7 @@ class Notification(db.Model):
     created_at = db.Column(db.DateTime, index=True, unique=False, nullable=False)
     sent_at = db.Column(db.DateTime, index=False, unique=False, nullable=True)
     sent_by = db.Column(db.String, nullable=True)
+    message_parts = db.Column(db.Integer, nullable=True)
     updated_at = db.Column(
         db.DateTime,
         index=False,
@@ -1743,6 +1744,7 @@ def serialize(self):
             "sent_at": get_dt_string_or_none(self.sent_at),
             "completed_at": self.completed_at(),
             "scheduled_for": None,
+            "message_parts": self.message_parts,
         }
 
         return serialized

app/notifications/process_notifications.py

@@ -10,8 +10,10 @@
     dao_create_notification,
     dao_delete_notifications_by_id,
 )
+from app.dao.templates_dao import dao_get_template_by_id_and_service_id
 from app.enums import KeyType, NotificationStatus, NotificationType
 from app.models import Notification
+from app.utils import hilite
 from app.v2.errors import BadRequestError
 from notifications_utils.recipients import (
     format_email_address,
@@ -22,6 +24,7 @@
 
 
 def create_content_for_notification(template, personalisation):
+    print(f"enter create_content_for_notification {template.id} {personalisation}")
     if template.template_type == NotificationType.EMAIL:
         template_object = PlainTextEmailTemplate(
             {
@@ -31,6 +34,7 @@ def create_content_for_notification(template, personalisation):
             },
             personalisation,
         )
+        print("created email template object")
     if template.template_type == NotificationType.SMS:
         template_object = SMSMessageTemplate(
             {
@@ -50,6 +54,7 @@ def check_placeholders(template_object):
         message = "Missing personalisation: {}".format(
             ", ".join(template_object.missing_data)
         )
+        print(f"GOING TO RAISE BAD REQUEST ERROR WITH {message}")
         raise BadRequestError(fields=[{"template": message}], message=message)
 
 
@@ -105,6 +110,8 @@ def persist_notification(
         document_download_count=document_download_count,
         updated_at=updated_at,
     )
+    print(hilite(f"personalisation? {personalisation}"))
+    print(hilite(f"notification? {notification}"))
 
     if notification_type == NotificationType.SMS:
         formatted_recipient = validate_and_format_phone_number(
@@ -117,6 +124,7 @@ def persist_notification(
         notification.rate_multiplier = recipient_info.billable_units
 
     elif notification_type == NotificationType.EMAIL:
+        print(hilite("inside elif for email type"))
         current_app.logger.info(
             f"Persisting notification with type: {NotificationType.EMAIL}"
         )
@@ -125,11 +133,29 @@ def persist_notification(
             format_email_address(notification.to),
             ex=1800,
         )
-
+    print(hilite(f"simulated? {simulated}"))
     # if simulated create a Notification model to return but do not persist the Notification to the dB
     if not simulated:
         current_app.logger.info("Firing dao_create_notification")
-        dao_create_notification(notification)
+        print(
+            f"LOOK UP TEMPLATE WITH TEMPLATE_ID {template_id} and service_id {service.id}"
+        )
+        template = dao_get_template_by_id_and_service_id(template_id, service.id)
+        print(hilite(f"created template?: {template} personalization {notification.personalisation}"))
+        template_object = create_content_for_notification(
+            template, notification.personalisation
+        )
+        print(hilite(f"template object? {template_object}"))
+        if isinstance(template_object, SMSMessageTemplate):
+            print(f"MESSAGE PARTS {template_object.fragment_count}")
+            notification.message_parts = template_object.fragment_count
+        print("GOING TO CREATE NOTIFICATION")
+        try:
+            dao_create_notification(notification)
+        except BaseException as be:
+            print(be)
+
+        print(hilite("created notification?"))
         if key_type != KeyType.TEST and current_app.config["REDIS_ENABLED"]:
             current_app.logger.info(
                 "Redis enabled, querying cache key for service id: {}".format(

app/notifications/rest.py

@@ -233,3 +233,11 @@ def create_template_object_for_notification(template, personalisation):
         errors = {"content": [message]}
         raise InvalidRequest(errors, status_code=400)
     return template_object
+
+
+@notifications.route("/notifications/<uuid:job_id>/message_parts", methods=["GET"])
+def get_batch_notification_message_parts_by_job_id(job_id):
+    total_message_parts = (
+        notifications_dao.dao_get_notification_message_parts_by_job_id(job_id=job_id)
+    )
+    return jsonify(total_message_parts=total_message_parts), 200

app/organization/invite_rest.py

@@ -27,6 +27,7 @@
     post_update_invited_org_user_status_schema,
 )
 from app.schema_validation import validate
+from app.utils import hilite
 from notifications_utils.url_safe_token import check_token, generate_token
 
 organization_invite_blueprint = Blueprint("organization_invite", __name__)
@@ -38,20 +39,21 @@
     "/organization/<uuid:organization_id>/invite", methods=["POST"]
 )
 def invite_user_to_org(organization_id):
+    print(hilite("top of org method"))
     data = request.get_json()
     validate(data, post_create_invited_org_user_status_schema)
-
+    print(hilite(f"got the data: {data}"))
     invited_org_user = InvitedOrganizationUser(
         email_address=data["email_address"],
         invited_by_id=data["invited_by"],
         organization_id=organization_id,
     )
     save_invited_org_user(invited_org_user)
-
+    print(hilite("saved the data"))
     template = dao_get_template_by_id(
         current_app.config["ORGANIZATION_INVITATION_EMAIL_TEMPLATE_ID"]
     )
-
+    print(hilite("got the template"))
     token = generate_token(
         str(invited_org_user.email_address),
         current_app.config["SECRET_KEY"],
@@ -70,27 +72,37 @@ def invite_user_to_org(organization_id):
         "organization_name": invited_org_user.organization.name,
         "url": url,
     }
+    print(hilite(f"personalisation: {personalisation}"))
+    db_personalisation = personalisation
+    if db_personalisation.get("phone_number"):
+        del db_personalisation["phone_number"]
+    if db_personalisation.get("email_address"):
+        del db_personalisation["email_address"]
+    # try:
     saved_notification = persist_notification(
         template_id=template.id,
         template_version=template.version,
         recipient=invited_org_user.email_address,
         service=template.service,
-        personalisation={},
+        personalisation=db_personalisation,
         notification_type=NotificationType.EMAIL,
         api_key_id=None,
         key_type=KeyType.NORMAL,
         reply_to_text=invited_org_user.invited_by.email_address,
     )
-
+    # except BaseException as be:
+    #     print(hilite(be))
+    print(hilite(f"saved notification: {saved_notification}"))
     saved_notification.personalisation = personalisation
+    print(hilite(f"saved notification after personalisation: {saved_notification}"))
     redis_store.set(
         f"email-personalisation-{saved_notification.id}",
         json.dumps(personalisation),
         ex=1800,
     )
 
     send_notification_to_queue(saved_notification, queue=QueueNames.NOTIFY)
-
+    print(hilite("sent notification to queue"))
     return jsonify(data=invited_org_user.serialize()), 201
 
 

app/organization/rest.py

@@ -200,13 +200,17 @@ def send_notifications_on_mou_signed(organization_id):
 
     def _send_notification(template_id, recipient, personalisation):
         template = dao_get_template_by_id(template_id)
-
+        db_personalisation = personalisation
+        if db_personalisation.get("phone_number"):
+            del db_personalisation["phone_number"]
+        if db_personalisation.get("email_address"):
+            del db_personalisation["email_address"]
         saved_notification = persist_notification(
             template_id=template.id,
             template_version=template.version,
             recipient=recipient,
             service=notify_service,
-            personalisation={},
+            personalisation=db_personalisation,
             notification_type=template.template_type,
             api_key_id=None,
             key_type=KeyType.NORMAL,

app/service/sender.py

@@ -11,12 +11,15 @@
     persist_notification,
     send_notification_to_queue,
 )
+from app.utils import hilite
 
 
 def send_notification_to_service_users(
     service_id, template_id, personalisation=None, include_user_fields=None
 ):
+    print(f"SERVICE ID {service_id} TEMPLATE_ID {template_id}")
     personalisation = personalisation or {}
+    print(hilite(f"personalisation {personalisation}"))
     include_user_fields = include_user_fields or []
     template = dao_get_template_by_id(template_id)
     service = dao_fetch_service_by_id(service_id)
@@ -25,6 +28,7 @@ def send_notification_to_service_users(
 
     for user in active_users:
         personalisation = _add_user_fields(user, personalisation, include_user_fields)
+        print(hilite(f"next personalisation {personalisation}"))
         notification = persist_notification(
             template_id=template.id,
             template_version=template.version,
@@ -40,6 +44,7 @@ def send_notification_to_service_users(
             key_type=KeyType.NORMAL,
             reply_to_text=notify_service.get_default_reply_to_email_address(),
         )
+        print(hilite("Persisted notification!"))
         send_notification_to_queue(notification, queue=QueueNames.NOTIFY)
 
 

app/service_invite/rest.py

@@ -75,12 +75,18 @@ def _create_service_invite(invited_user, invite_link_host):
         "url": url,
     }
 
+    db_personalisation = personalisation
+    if db_personalisation.get("phone_number"):
+        del db_personalisation["phone_number"]
+    if db_personalisation.get("email_address"):
+        del db_personalisation["email_address"]
+
     saved_notification = persist_notification(
         template_id=template.id,
         template_version=template.version,
         recipient=invited_user.email_address,
         service=service,
-        personalisation={},
+        personalisation=db_personalisation,
         notification_type=NotificationType.EMAIL,
         api_key_id=None,
         key_type=KeyType.NORMAL,

app/user/rest.py

@@ -386,6 +386,10 @@ def send_user_confirm_new_email(user_id):
         current_app.config["CHANGE_EMAIL_CONFIRMATION_TEMPLATE_ID"]
     )
     service = Service.query.get(current_app.config["NOTIFY_SERVICE_ID"])
+    # TODO writing this personalization to the db adds a little PII
+    # to the notifications table, but it will be purged after 7 days.
+    # If we don't want it to go into the db at all, we would need to
+    # turn this into a job and retrieve it from S3.
     personalisation = {
         "name": user_to_send_to.name,
         "url": _create_confirmation_url(
@@ -398,7 +402,7 @@ def send_user_confirm_new_email(user_id):
         template_version=template.version,
         recipient=email["email"],
         service=service,
-        personalisation={},
+        personalisation=personalisation,
         notification_type=template.template_type,
         api_key_id=None,
         key_type=KeyType.NORMAL,
@@ -437,17 +441,19 @@ def send_new_user_email_verification(user_id):
             base_url=request_json.get("admin_base_url"),
         ),
     }
+    print("TRY TO SAVE NOTI")
     saved_notification = persist_notification(
         template_id=template.id,
         template_version=template.version,
         recipient=user_to_send_to.email_address,
         service=service,
-        personalisation={},
+        personalisation=personalisation,
         notification_type=template.template_type,
         api_key_id=None,
         key_type=KeyType.NORMAL,
         reply_to_text=service.get_default_reply_to_email_address(),
     )
+    print("SAVED THE NOTIFICATION")
     saved_notification.personalisation = personalisation
 
     redis_store.set(
@@ -460,6 +466,7 @@ def send_new_user_email_verification(user_id):
         json.dumps(personalisation),
         ex=60 * 60,
     )
+    print("SET REDIS")
     current_app.logger.info("Sending notification to queue")
 
     send_notification_to_queue(saved_notification, queue=QueueNames.NOTIFY)

migrations/versions/0336_broadcast_msg_content_2.py

@@ -8,10 +8,11 @@
 
 import sqlalchemy as sa
 from alembic import op
-from notifications_utils.template import BroadcastMessageTemplate
 from sqlalchemy.dialects import postgresql
 from sqlalchemy.orm.session import Session
 
+from notifications_utils.template import BroadcastMessageTemplate
+
 revision = "0336_broadcast_msg_content_2"
 down_revision = "0335_broadcast_msg_content"