generate report of unopened unactioned reqeusts and their potential m…

docker-compose.yml

@@ -110,6 +110,10 @@ services:
       - EVENT_QUEUE_PDFSTITCH_LARGE_FILE_STREAMKEY=${EVENT_QUEUE_PDFSTITCH_LARGE_FILE_STREAMKEY}
       - STREAM_SEPARATION_FILE_SIZE_LIMIT=${STREAM_SEPARATION_FILE_SIZE_LIMIT}
       - MUTE_NOTIFICATION=${MUTE_NOTIFICATION}
+      - UNOPENED_REPORT_CUTOFF_DAYS=${UNOPENED_REPORT_CUTOFF_DAYS}
+      - UNOPENED_REPORT_WAIT_DAYS=${UNOPENED_REPORT_WAIT_DAYS}
+      - UNOPENED_REPORT_JARO_CUTOFF=${UNOPENED_REPORT_JARO_CUTOFF}
+      - UNOPENED_REPORT_EMAIL_RECIPIENT=${UNOPENED_REPORT_EMAIL_RECIPIENT}
       #- LOG_ROOT=${LOG_ROOT}
       #- LOG_BASIC=${LOG_BASIC}
       #- LOG_TRACING=${LOG_TRACING}

request-management-api/migrations/versions/d42a1cf67c5c_.py

@@ -0,0 +1,35 @@
+"""empty message
+
+Revision ID: d42a1cf67c5c
+Revises: ba218164248e
+Create Date: 2024-02-08 12:40:33.968711
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'd42a1cf67c5c'
+down_revision = 'ba218164248e'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+        op.create_table('UnopenedReport',
+            sa.Column('id', sa.Integer(), primary_key=True, autoincrement=True, nullable=False),
+            sa.Column('rawrequestid', sa.Integer(), nullable=False),
+            sa.Column('date', sa.DateTime(), nullable=True),
+            sa.Column('rank', sa.Integer(), nullable=False),    
+            sa.Column('potentialmatches', postgresql.JSON(astext_type=sa.Text()), nullable=True)
+        )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+        op.execute('DROP TABLE IF EXISTS public."UnopenedReport";')
+
+    # ### end Alembic commands ###

request-management-api/request_api/models/FOIRawRequests.py

@@ -1098,6 +1098,77 @@ def getlatestsection5pendings(cls):
         finally:
             db.session.close()
         return section5pendings
+
+    @classmethod
+    def getunopenedunactionedrequests(cls, startdate, enddate):
+        try:
+            requests = []
+            sql = '''select rr.created_at, rr.requestrawdata, rr.requestid, p.status from public."FOIRawRequests" rr
+                    join (
+                        select max(version) as version, requestid from public."FOIRawRequests"
+                        group by requestid
+                    ) mv on mv.requestid = rr.requestid and mv.version = rr.version
+                    join (
+                        select request_id, max(payment_id) from public."Payments"
+                        where fee_code_id = 1
+                        group by request_id
+                        order by request_id
+                    ) mp
+                    join public."Payments" p on p.payment_id = mp.max
+                    where status = 'Unopened' and rr.version = 1 and created_at > :cutoffdate and created_at < :enddate
+                    order by rr.requestid '''
+            rs = db.session.execute(text(sql), {'startdate': startdate, 'enddate': enddate})
+            for row in rs:
+                requests.append({
+                    "requestid": row["requestid"],
+                    "created_at": row["created_at"],
+                    "requestrawdata": row["requestrawdata"],
+                    "paymentstatus": row["status"]
+                })
+        except Exception as ex:
+            logging.error(ex)
+            raise ex
+        finally:
+            db.session.close()
+        return requests
+
+    @classmethod
+    def getpotentialactionedmatches(cls, request):
+        try:
+            requests = []
+            sql = '''select rr.created_at, rr.* from public."FOIRawRequests" rr
+                    join (
+                        select max(version) as version, requestid from public."FOIRawRequests"
+                        group by requestid
+                    ) mv on mv.requestid = rr.requestid and mv.version = rr.version
+                    where status = 'Intake in Progress' and (
+                        requestrawdata->>'lastName' ilike :lastName
+                        or requestrawdata->>'firstName' ilike :firstName
+                        or requestrawdata->>'email' ilike :email
+                        or requestrawdata->>'address' ilike :address
+                        or requestrawdata->>'phonePrimary' ilike :phonePrimary
+                        or requestrawdata->>'postal' ilike :postal
+                    ) and substring(rr.requestrawdata->>'receivedDateUF', 1, 10) = :receiveddate
+                    order by requestid desc, version desc '''
+            rs = db.session.execute(text(sql), {
+                'lastName': request['requestrawdata']['contactInfo']['lastName'],
+                'firstName': request['requestrawdata']['contactInfo']['firstName'],
+                'email': request['requestrawdata']['contactInfoOptions']['email'],
+                'address': request['requestrawdata']['contactInfoOptions']['address'],
+                'phonePrimary': request['requestrawdata']['contactInfoOptions']['phonePrimary'],
+                'postal': request['requestrawdata']['contactInfoOptions']['postal'],
+                'receiveddate': request['requestrawdata']['receivedDateUF'][0:10],
+            })
+            for row in rs:
+                requests.append({"requestid": row["requestid"], "created_at": row["created_at"], "requestrawdata": row["requestrawdata"]})
+        except Exception as ex:
+            logging.error(ex)
+            raise ex
+        finally:
+            db.session.close()
+        return requests
+
+
 
 class FOIRawRequestSchema(ma.Schema):
     class Meta:

request-management-api/request_api/models/UnopenedReport.py

@@ -0,0 +1,22 @@
+from .db import  db, ma
+from .default_method_result import DefaultMethodResult
+from sqlalchemy.orm import relationship,backref
+from datetime import datetime
+from sqlalchemy import text
+from sqlalchemy.dialects.postgresql import JSON
+import json
+
+class UnopenedReport(db.Model):
+    __tablename__ = 'UnopenedReport' 
+    # Defining the columns
+    id = db.Column(db.Integer, primary_key=True,autoincrement=True)
+    rawrequestid = db.Column(db.Text, unique=False, nullable=False)
+    date = db.Column(db.Text, unique=False, nullable=False)
+    rank = db.Column(db.Text, unique=False, nullable=False)
+    potentialmatches = db.Column(JSON, unique=False, nullable=False)
+
+    @classmethod
+    def insert(cls, row):
+        db.session.add(row)
+        db.session.commit()
+        return DefaultMethodResult(True,'Report Row added',row.rawrequestid)

request-management-api/request_api/resources/request.py

@@ -25,6 +25,7 @@
 from request_api.services.rawrequestservice import rawrequestservice
 from request_api.services.documentservice import documentservice
 from request_api.services.eventservice import eventservice
+from request_api.services.unopenedreportservice import unopenedreportservice
 from request_api.utils.enums import StateName
 import json
 import asyncio
@@ -323,4 +324,22 @@ def post(requestid=None):
         except ValueError:
             return {'status': 500, 'message':"Invalid Request"}, 400    
         except BusinessException as exception:            
-            return {'status': exception.status_code, 'message':exception.message}, 500  
+            return {'status': exception.status_code, 'message':exception.message}, 500  
+
+@cors_preflight('POST,OPTIONS')
+@API.route('/foirawrequest/unopenedreport')
+class FOIRawRequestReport(Resource):
+    """Generates report of unopened requests that have not been actioned in over X amount of days"""
+
+
+    @staticmethod
+    @TRACER.trace()
+    @cross_origin(origins=allowedorigins())
+    @auth.require
+    def post():
+        try:
+            result = unopenedreportservice().generateunopenedreport()
+            # responsecode = 200 if result.success == True else 500
+            return {'status': True, 'message': result} , 200
+        except BusinessException as exception:
+            return {'status': exception.status_code, 'message':exception.message}, 500

request-management-api/request_api/services/email/senderservice.py

@@ -30,12 +30,12 @@ class senderservice:
 
     """
 
-    def send(self, servicekey, content, _messageattachmentlist, requestjson):
+    def send(self, subject, content, _messageattachmentlist, requestjson):
         logging.debug("Begin: Send email for request = "+json.dumps(requestjson))
         msg = MIMEMultipart()
         msg['From'] = MAIL_FROM_ADDRESS
         msg['To'] = requestjson["email"]
-        msg['Subject'] = templateconfig().getsubject(servicekey, requestjson)
+        msg['Subject'] = subject
         part = MIMEText(content, "html")
         msg.attach(part)
         # Add Attachment and Set mail headers

request-management-api/request_api/services/emailservice.py

@@ -34,7 +34,8 @@ def send(self, servicename, requestid, ministryrequestid, emailschema):
                 servicename = _templatename.upper() if _templatename else ""
             _messageattachmentlist = self.__get_attachments(ministryrequestid, emailschema, servicename)
             self.__pre_send_correspondence_audit(requestid, ministryrequestid,emailschema, content, templateconfig().isnotreceipt(servicename), _messageattachmentlist)
-            return senderservice().send(servicename, _messagepart, _messageattachmentlist, requestjson)
+            subject = templateconfig().getsubject(servicename, requestjson)
+            return senderservice().send(subject, _messagepart, _messageattachmentlist, requestjson)
         except Exception as ex:
             logging.exception(ex)
 

request-management-api/request_api/services/unopenedreportservice.py

@@ -0,0 +1,153 @@
+
+from request_api.models.FOIRawRequests import FOIRawRequest
+from request_api.models.UnopenedReport import UnopenedReport
+from request_api.services.email.senderservice import senderservice
+from os import getenv
+from datetime import timedelta, date
+from jaro import jaro_winkler_metric
+import json
+import logging
+from math import inf
+
+class unopenedreportservice:
+    """
+    This service generates a report of unopened unactioned requests
+
+    """
+
+    dayscutoff = getenv('UNOPENED_REPORT_CUTOFF_DAYS', 10)
+    waitdays = getenv('UNOPENED_REPORT_WAIT_DAYS', 5)
+    jarocutoff = getenv('UNOPENED_REPORT_JARO_CUTOFF', 0.8)
+    reportemail = getenv('UNOPENED_REPORT_EMAIL_RECIPIENT')
+
+
+    def generateunopenedreport(self):
+        startdate = date.today() - timedelta(days=self.dayscutoff)
+        enddate = startdate + timedelta(days=self.waitdays)
+        requests = FOIRawRequest.getunopenedunactionedrequests(str(startdate), str(enddate))
+        alerts = []
+        for request in requests:
+            potentialmatches = FOIRawRequest.getpotentialactionedmatches(request)
+            if len(potentialmatches) == 0:
+                alert = UnopenedReport()
+                alert.rawrequestid = request['requestid']
+                alert.date = date.today()
+                alert.rank = 1
+                UnopenedReport.insert(alert)
+                alerts.append({"request": request, "rank": 1})
+            else:
+                highscore = 0
+                for match in potentialmatches:
+                    match['score'] = jaro_winkler_metric(
+                        request['requestrawdata']['descriptionTimeframe']['description'].replace('\n', ' ').replace('\t', ' '),
+                        match['requestrawdata']['description']
+                    )
+                    if match['score'] > highscore:
+                        highscore = match['score']
+                alert = UnopenedReport()
+                alert.rawrequestid = request['requestid']
+                alert.date = date.today()
+                alert.rank = 2
+                alert.potentialmatches = {"highscore": round(highscore, 2), "matches": [{
+                    "requestid": m["requestrawdata"]['axisRequestId'],
+                    "similarity": round(m['score'], 2)
+                } for m in potentialmatches]}
+                UnopenedReport.insert(alert)
+                alerts.append({"request": request, "rank": 2, "potentialmatches": alert.potentialmatches})
+        alerts.sort(key=lambda a : a.get('potentialmatches', {'highscore': 0})['highscore'])
+        senderservice().send(
+            subject="Intake Unopened Request Report: " + date.today(),
+            content=self.generateemailhtml(alerts),
+            _messageattachmentlist=[],
+            requestjson={"email": self.reportemail, "topic": "Unopened Report"}
+        )
+        return alerts
+
+
+    def generateemailhtml(self, alerts):
+        emailhtml = """
+            <h3>Unopened Report - 2024/02/05</h3>
+
+            <p>This is a report for unopened requests in the past 10 days that have not yet been actioned.</p>
+            <p><b>Rank 1:</b> Very likely to be unactioned — unable to find a request with any matching applicant info</p>
+            <table border='1' style='border-collapse:collapse'>
+            <tr>
+                <th>Unopened ID</th>
+                <th>Applicant First Name</th>
+                <th>Applicant Last Name</th>
+                <th>Payment Status</th>
+                <th>Description</th>
+            </tr>
+        """
+        for alert in alerts:
+            if alert.get('potentialmatches', False):
+                emailhtml += '''
+                    <tr>
+                        <td>U-000''' + alert['request']['requestid'] + '''</td>
+                        <td>''' + alert['request']['requestrawdata']['contactInfo']['lastName'] + '''</td>
+                        <td>''' + alert['request']['requestrawdata']['contactInfo']['firstName'] + '''</td>
+                        <td>''' + alert['request']['paymentstatus'] + '''</td>
+                        <td>''' + alert['request']['requestrawdata']['descriptionTimeframe']['description'][0:99] + '''...</td>
+                    </tr>
+                '''
+            else:
+                firstrank2 = True
+                if firstrank2:
+                    emailhtml += """</table>
+                        <p><b>Rank 2:</b> Possibly unactioned — requests found but some applicant info is mismatching — please double check</p>
+                        <table border='1' style='border-collapse:collapse'>
+                        <tr>
+                            <th>Unopened ID</th>
+                            <th>Applicant First Name</th>
+                            <th>Applicant Last Name</th>
+                            <th>Payment Status</th>
+                            <th>Potential Matches</th>
+                            <th>Description</th>
+                        </tr>
+                    """
+                    firstrank2 = False
+                emailhtml += '''
+                    <tr>
+                        <td>U-000''' + alert['request']['requestid'] + '''</td>
+                        <td>''' + alert['request']['requestrawdata']['contactInfo']['lastName'] + '''</td>
+                        <td>''' + alert['request']['requestrawdata']['contactInfo']['firstName'] + '''</td>
+                        <td>''' + alert['request']['paymentstatus'] + '''</td>
+                        <td>
+                    '''
+                for m in alert['potentialmatches']:
+                    emailhtml += (m['requestid'] + " - similarity: " + m['similarity'] + "<br>")
+                emailhtml = emailhtml[:4]
+                emailhtml += '''</td>
+                        <td>''' + alert['request']['requestrawdata']['descriptionTimeframe']['description'][0:99] + '''...</td>
+                    </tr>
+                '''
+
+
+    # def isrank2match(self, request, match):
+    #     diff = 0
+    #     for k, v in request['requestrawdata']['contactInfo'].items():
+    #         if v and match['requestrawdata'].get(k, False) != v:
+    #             diff += 1
+    #             if diff > self.diffcutoff:
+    #                 return False
+    #     for k, v in request['requestrawdata']['contactInfoOptions'].items():
+    #         if v and match['requestrawdata'].get(k, False) != v:
+    #             diff += 1
+    #             if diff > self.diffcutoff:
+    #                 return False
+    #     print("rank 2 match")
+    #     return True
+
+    # def isrank1match(self, request, match):
+    #     print(request['requestrawdata']['receivedDateUF'][0:10])
+    #     print(match['requestrawdata']['receivedDateUF'][0:10])
+    #     match = (((request['requestrawdata']['contactInfo']['firstName'] == match['requestrawdata']['firstName']
+    #               and request['requestrawdata']['contactInfo']['lastName'] == match['requestrawdata']['lastName'])
+    #              or request['requestrawdata']['contactInfoOptions']['email'] == match['requestrawdata']['email']
+    #              or request['requestrawdata']['contactInfoOptions']['phonePrimary'] == match['requestrawdata']['phonePrimary']
+    #              or request['requestrawdata']['contactInfoOptions']['postal'] == match['requestrawdata']['postal']
+    #              )
+    #         and request['requestrawdata']['receivedDateUF'][0:10] == match['requestrawdata']['receivedDateUF'][0:10])
+    #     if match:
+    #         print("rank 1 match")
+    #     return match

request-management-api/requirements.txt

@@ -36,6 +36,7 @@ gunicorn==20.1.0
 idna==2.10
 itsdangerous==2.0.1
 jaeger-client==4.4.0
+jaro-winkler==2.0.3
 jsonschema==3.2.0
 marshmallow-sqlalchemy==0.23.1
 marshmallow==3.0.0rc7

sample.env

@@ -159,4 +159,9 @@ DISABLE_REDACT_WEBLINK=false
 DISABLE_GATHERINGRECORDS_TAB=false
 KC_SRC_ACC_TOKEN_EXPIRY=60
 
-MUTE_NOTIFICATION={"MCF": {"request_types": ["Personal"], "state_exceptions": ["Call For Records", "Ministry Sign Off"], "type_exceptions":["Reply User Comments", "Tagged User Comments"]}}
+MUTE_NOTIFICATION={"MCF": {"request_types": ["Personal"], "state_exceptions": ["Call For Records", "Ministry Sign Off"], "type_exceptions":["Reply User Comments", "Tagged User Comments"]}}
+
+UNOPENED_REPORT_CUTOFF_DAYS=10
+UNOPENED_REPORT_WAIT_DAYS=5
+UNOPENED_REPORT_JARO_CUTOFF=0.8
+UNOPENED_REPORT_EMAIL_RECIPIENT=