From 3242955ff78bdf4d25611b36783fe581b669d9af Mon Sep 17 00:00:00 2001 From: Abin Antony <78570775+abin-aot@users.noreply.github.com> Date: Fri, 23 Feb 2024 12:01:44 -0800 Subject: [PATCH] Revert "Dev" --- docker-compose.yml | 4 - .../migrations/versions/d42a1cf67c5c_.py | 35 ---- .../request_api/models/FOIMinistryRequests.py | 43 ++--- .../request_api/models/FOIRawRequests.py | 76 --------- .../request_api/models/UnopenedReport.py | 22 --- .../request_api/resources/request.py | 21 +-- .../services/email/senderservice.py | 4 +- .../request_api/services/emailservice.py | 3 +- .../services/unopenedreportservice.py | 150 ------------------ request-management-api/requirements.txt | 1 - sample.env | 7 +- 11 files changed, 30 insertions(+), 336 deletions(-) delete mode 100644 request-management-api/migrations/versions/d42a1cf67c5c_.py delete mode 100644 request-management-api/request_api/models/UnopenedReport.py delete mode 100644 request-management-api/request_api/services/unopenedreportservice.py diff --git a/docker-compose.yml b/docker-compose.yml index 3bcafd591..77423470b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -110,10 +110,6 @@ services: - EVENT_QUEUE_PDFSTITCH_LARGE_FILE_STREAMKEY=${EVENT_QUEUE_PDFSTITCH_LARGE_FILE_STREAMKEY} - STREAM_SEPARATION_FILE_SIZE_LIMIT=${STREAM_SEPARATION_FILE_SIZE_LIMIT} - MUTE_NOTIFICATION=${MUTE_NOTIFICATION} - - UNOPENED_REPORT_CUTOFF_DAYS=${UNOPENED_REPORT_CUTOFF_DAYS} - - UNOPENED_REPORT_WAIT_DAYS=${UNOPENED_REPORT_WAIT_DAYS} - - UNOPENED_REPORT_JARO_CUTOFF=${UNOPENED_REPORT_JARO_CUTOFF} - - UNOPENED_REPORT_EMAIL_RECIPIENT=${UNOPENED_REPORT_EMAIL_RECIPIENT} #- LOG_ROOT=${LOG_ROOT} #- LOG_BASIC=${LOG_BASIC} #- LOG_TRACING=${LOG_TRACING} diff --git a/request-management-api/migrations/versions/d42a1cf67c5c_.py b/request-management-api/migrations/versions/d42a1cf67c5c_.py deleted file mode 100644 index 4a61359be..000000000 --- a/request-management-api/migrations/versions/d42a1cf67c5c_.py +++ /dev/null @@ -1,35 +0,0 @@ -"""empty message - -Revision ID: d42a1cf67c5c -Revises: b4da31675bd0 -Create Date: 2024-02-08 12:40:33.968711 - -""" -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -# revision identifiers, used by Alembic. -revision = 'd42a1cf67c5c' -down_revision = 'b4da31675bd0' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('UnopenedReport', - sa.Column('id', sa.Integer(), primary_key=True, autoincrement=True, nullable=False), - sa.Column('rawrequestid', sa.Integer(), nullable=False), - sa.Column('date', sa.DateTime(), nullable=True), - sa.Column('rank', sa.Integer(), nullable=False), - sa.Column('potentialmatches', postgresql.JSON(astext_type=sa.Text()), nullable=True) - ) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.execute('DROP TABLE IF EXISTS public."UnopenedReport";') - - # ### end Alembic commands ### diff --git a/request-management-api/request_api/models/FOIMinistryRequests.py b/request-management-api/request_api/models/FOIMinistryRequests.py index 565076612..92ec73e7c 100644 --- a/request-management-api/request_api/models/FOIMinistryRequests.py +++ b/request-management-api/request_api/models/FOIMinistryRequests.py @@ -9,6 +9,7 @@ from sqlalchemy import or_, and_, text, func, literal, cast, case, nullslast, nullsfirst, desc, asc from sqlalchemy.sql.sqltypes import String from sqlalchemy.dialects.postgresql import JSON + from .FOIRequestApplicantMappings import FOIRequestApplicantMapping from .FOIRequestApplicants import FOIRequestApplicant from .FOIRequestStatus import FOIRequestStatus @@ -583,8 +584,7 @@ def getrequestssubquery(cls, groups, filterfields, keyword, additionalfilter, us *joincondition_oipc ), isouter=True - ).filter(or_(FOIMinistryRequest.requeststatuslabel != StateName.closed.name, - and_(FOIMinistryRequest.isoipcreview == True, FOIMinistryRequest.requeststatusid == 3, subquery_with_oipc.c.outcomeid == None))) + ).filter(or_(FOIMinistryRequest.requeststatuslabel != StateName.closed.name, and_(FOIMinistryRequest.isoipcreview == True, FOIMinistryRequest.requeststatusid == 3, subquery_with_oipc.c.outcomeid == None))) if(additionalfilter == 'watchingRequests'): #watchby @@ -701,35 +701,42 @@ def getgroupfilters(cls, groups): ministryfilter = FOIMinistryRequest.isactive == True else: groupfilter = [] - statusfilter = None - processinggroups = list(set(groups).intersection(ProcessingTeamWithKeycloackGroup.list())) - if IAOTeamWithKeycloackGroup.intake.value in groups or len(processinggroups) > 0: - groupfilter.append( + for group in groups: + if (group == IAOTeamWithKeycloackGroup.flex.value or group in ProcessingTeamWithKeycloackGroup.list()): + groupfilter.append( + and_( + FOIMinistryRequest.assignedgroup == group + ) + ) + elif (group == IAOTeamWithKeycloackGroup.intake.value): + groupfilter.append( + or_( + FOIMinistryRequest.assignedgroup == group, and_( - FOIMinistryRequest.assignedgroup.in_(tuple(groups)) + FOIMinistryRequest.assignedgroup == IAOTeamWithKeycloackGroup.flex.value, + FOIMinistryRequest.requeststatuslabel.in_([StateName.open.name]) ) ) - statusfilter = FOIMinistryRequest.requeststatuslabel != StateName.closed.name - else: - groupfilter.append( + ) + else: + groupfilter.append( or_( - FOIMinistryRequest.assignedgroup.in_(tuple(groups)), + FOIMinistryRequest.assignedgroup == group, and_( - FOIMinistryRequest.assignedministrygroup.in_(tuple(groups)) + FOIMinistryRequest.assignedministrygroup == group, + FOIMinistryRequest.requeststatuslabel.in_([StateName.callforrecords.name,StateName.recordsreview.name,StateName.feeestimate.name,StateName.consult.name,StateName.ministrysignoff.name,StateName.onhold.name,StateName.deduplication.name,StateName.harmsassessment.name,StateName.response.name,StateName.peerreview.name,StateName.tagging.name,StateName.readytoscan.name]) ) ) ) - statusfilter = FOIMinistryRequest.requeststatuslabel.in_([StateName.callforrecords.name,StateName.recordsreview.name,StateName.feeestimate.name,StateName.consult.name,StateName.ministrysignoff.name,StateName.onhold.name,StateName.deduplication.name,StateName.harmsassessment.name,StateName.response.name,StateName.peerreview.name,StateName.tagging.name,StateName.readytoscan.name]) + ministryfilter = and_( FOIMinistryRequest.isactive == True, FOIRequestStatus.isactive == True, or_(*groupfilter) ) - ministryfilterwithclosedoipc = and_(ministryfilter, - or_(statusfilter, - and_(FOIMinistryRequest.isoipcreview == True, FOIMinistryRequest.requeststatuslabel == StateName.closed.name) - ) - ) + + ministryfilterwithclosedoipc = or_(ministryfilter, and_(FOIMinistryRequest.isoipcreview == True, FOIMinistryRequest.requeststatuslabel == StateName.closed.name)) + return ministryfilterwithclosedoipc @classmethod diff --git a/request-management-api/request_api/models/FOIRawRequests.py b/request-management-api/request_api/models/FOIRawRequests.py index 1957a4c79..8229605be 100644 --- a/request-management-api/request_api/models/FOIRawRequests.py +++ b/request-management-api/request_api/models/FOIRawRequests.py @@ -1073,82 +1073,6 @@ def getlatestsection5pendings(cls): finally: db.session.close() return section5pendings - - @classmethod - def getunopenedunactionedrequests(cls, startdate, enddate): - try: - requests = [] - sql = '''select rr.created_at, rr.requestrawdata, rr.requestid, coalesce(p.status, '') as status, - coalesce(p.transaction_number, '') as txnno, - coalesce(p.total::text, '') as fee - from public."FOIRawRequests" rr - join ( - select max(version) as version, requestid from public."FOIRawRequests" - group by requestid - ) mv on mv.requestid = rr.requestid and mv.version = rr.version - left join ( - select request_id, max(payment_id) from public."Payments" - where fee_code_id = 1 - group by request_id - order by request_id - ) mp on mp.request_id = rr.requestid - left join public."Payments" p on p.payment_id = mp.max - where rr.status = 'Unopened' and rr.version = 1 and created_at > :startdate and created_at < :enddate - order by rr.requestid ''' - rs = db.session.execute(text(sql), {'startdate': startdate, 'enddate': enddate}) - for row in rs: - requests.append({ - "requestid": row["requestid"], - "created_at": row["created_at"], - "requestrawdata": row["requestrawdata"], - "paymentstatus": row["status"], - "fee": row["fee"], - "txnno": row["txnno"] - }) - except Exception as ex: - logging.error(ex) - raise ex - finally: - db.session.close() - return requests - - @classmethod - def getpotentialactionedmatches(cls, request): - try: - requests = [] - sql = '''select rr.created_at, rr.* from public."FOIRawRequests" rr - join ( - select max(version) as version, requestid from public."FOIRawRequests" - group by requestid - ) mv on mv.requestid = rr.requestid and mv.version = rr.version - where status = 'Intake in Progress' and ( - requestrawdata->>'lastName' ilike :lastName - or requestrawdata->>'firstName' ilike :firstName - or requestrawdata->>'email' ilike :email - or requestrawdata->>'address' ilike :address - or requestrawdata->>'phonePrimary' ilike :phonePrimary - or requestrawdata->>'postal' ilike :postal - ) and substring(rr.requestrawdata->>'receivedDateUF', 1, 10) = :receiveddate - order by requestid desc, version desc ''' - rs = db.session.execute(text(sql), { - 'lastName': request['requestrawdata']['contactInfo']['lastName'], - 'firstName': request['requestrawdata']['contactInfo']['firstName'], - 'email': request['requestrawdata']['contactInfoOptions']['email'], - 'address': request['requestrawdata']['contactInfoOptions']['address'], - 'phonePrimary': request['requestrawdata']['contactInfoOptions']['phonePrimary'], - 'postal': request['requestrawdata']['contactInfoOptions']['postal'], - 'receiveddate': request['requestrawdata']['receivedDateUF'][0:10], - }) - for row in rs: - requests.append({"requestid": row["requestid"], "created_at": row["created_at"], "requestrawdata": row["requestrawdata"]}) - except Exception as ex: - logging.error(ex) - raise ex - finally: - db.session.close() - return requests - - class FOIRawRequestSchema(ma.Schema): class Meta: diff --git a/request-management-api/request_api/models/UnopenedReport.py b/request-management-api/request_api/models/UnopenedReport.py deleted file mode 100644 index dd8a75f43..000000000 --- a/request-management-api/request_api/models/UnopenedReport.py +++ /dev/null @@ -1,22 +0,0 @@ -from .db import db, ma -from .default_method_result import DefaultMethodResult -from sqlalchemy.orm import relationship,backref -from datetime import datetime -from sqlalchemy import text -from sqlalchemy.dialects.postgresql import JSON -import json - -class UnopenedReport(db.Model): - __tablename__ = 'UnopenedReport' - # Defining the columns - id = db.Column(db.Integer, primary_key=True,autoincrement=True) - rawrequestid = db.Column(db.Text, unique=False, nullable=False) - date = db.Column(db.Text, unique=False, nullable=False) - rank = db.Column(db.Text, unique=False, nullable=False) - potentialmatches = db.Column(JSON, unique=False, nullable=False) - - @classmethod - def insert(cls, row): - db.session.add(row) - db.session.commit() - return DefaultMethodResult(True,'Report Row added',row.rawrequestid) \ No newline at end of file diff --git a/request-management-api/request_api/resources/request.py b/request-management-api/request_api/resources/request.py index 335a4fac8..fcec8a2e5 100644 --- a/request-management-api/request_api/resources/request.py +++ b/request-management-api/request_api/resources/request.py @@ -25,7 +25,6 @@ from request_api.services.rawrequestservice import rawrequestservice from request_api.services.documentservice import documentservice from request_api.services.eventservice import eventservice -from request_api.services.unopenedreportservice import unopenedreportservice from request_api.utils.enums import StateName import json import asyncio @@ -324,22 +323,4 @@ def post(requestid=None): except ValueError: return {'status': 500, 'message':"Invalid Request"}, 400 except BusinessException as exception: - return {'status': exception.status_code, 'message':exception.message}, 500 - -@cors_preflight('POST,OPTIONS') -@API.route('/foirawrequest/unopenedreport') -class FOIRawRequestReport(Resource): - """Generates report of unopened requests that have not been actioned in over X amount of days""" - - - @staticmethod - @TRACER.trace() - @cross_origin(origins=allowedorigins()) - @auth.require - def post(): - try: - result = unopenedreportservice().generateunopenedreport() - # responsecode = 200 if result.success == True else 500 - return {'status': True, 'message': result} , 200 - except BusinessException as exception: - return {'status': exception.status_code, 'message':exception.message}, 500 \ No newline at end of file + return {'status': exception.status_code, 'message':exception.message}, 500 \ No newline at end of file diff --git a/request-management-api/request_api/services/email/senderservice.py b/request-management-api/request_api/services/email/senderservice.py index 4c5034e4a..d15069ca7 100644 --- a/request-management-api/request_api/services/email/senderservice.py +++ b/request-management-api/request_api/services/email/senderservice.py @@ -30,12 +30,12 @@ class senderservice: """ - def send(self, subject, content, _messageattachmentlist, requestjson): + def send(self, servicekey, content, _messageattachmentlist, requestjson): logging.debug("Begin: Send email for request = "+json.dumps(requestjson)) msg = MIMEMultipart() msg['From'] = MAIL_FROM_ADDRESS msg['To'] = requestjson["email"] - msg['Subject'] = subject + msg['Subject'] = templateconfig().getsubject(servicekey, requestjson) part = MIMEText(content, "html") msg.attach(part) # Add Attachment and Set mail headers diff --git a/request-management-api/request_api/services/emailservice.py b/request-management-api/request_api/services/emailservice.py index 6df55855c..2d16c7c35 100644 --- a/request-management-api/request_api/services/emailservice.py +++ b/request-management-api/request_api/services/emailservice.py @@ -34,8 +34,7 @@ def send(self, servicename, requestid, ministryrequestid, emailschema): servicename = _templatename.upper() if _templatename else "" _messageattachmentlist = self.__get_attachments(ministryrequestid, emailschema, servicename) self.__pre_send_correspondence_audit(requestid, ministryrequestid,emailschema, content, templateconfig().isnotreceipt(servicename), _messageattachmentlist) - subject = templateconfig().getsubject(servicename, requestjson) - return senderservice().send(subject, _messagepart, _messageattachmentlist, requestjson) + return senderservice().send(servicename, _messagepart, _messageattachmentlist, requestjson) except Exception as ex: logging.exception(ex) diff --git a/request-management-api/request_api/services/unopenedreportservice.py b/request-management-api/request_api/services/unopenedreportservice.py deleted file mode 100644 index 2fbaddad7..000000000 --- a/request-management-api/request_api/services/unopenedreportservice.py +++ /dev/null @@ -1,150 +0,0 @@ - -from request_api.models.FOIRawRequests import FOIRawRequest -from request_api.models.UnopenedReport import UnopenedReport -from request_api.services.email.senderservice import senderservice -from os import getenv -from datetime import timedelta, date -from jaro import jaro_winkler_metric -import json -import logging -from math import inf - -class unopenedreportservice: - """ - This service generates a report of unopened unactioned requests - - """ - - dayscutoff = getenv('UNOPENED_REPORT_CUTOFF_DAYS', 10) - waitdays = getenv('UNOPENED_REPORT_WAIT_DAYS', 5) - jarocutoff = getenv('UNOPENED_REPORT_JARO_CUTOFF', 0.8) - reportemail = getenv('UNOPENED_REPORT_EMAIL_RECIPIENT') - - - def generateunopenedreport(self): - startdate = date.today() - timedelta(days=int(self.dayscutoff)) - enddate = date.today() - timedelta(days=int(self.waitdays)) - requests = FOIRawRequest.getunopenedunactionedrequests(str(startdate), str(enddate)) - alerts = [] - for request in requests: - potentialmatches = FOIRawRequest.getpotentialactionedmatches(request) - if len(potentialmatches) == 0: - alert = UnopenedReport() - alert.rawrequestid = request['requestid'] - alert.date = date.today() - alert.rank = 1 - UnopenedReport.insert(alert) - alerts.append({"request": request, "rank": 1}) - else: - highscore = 0 - for match in potentialmatches: - match['score'] = jaro_winkler_metric( - request['requestrawdata']['descriptionTimeframe']['description'].replace('\n', ' ').replace('\t', ' '), - match['requestrawdata']['description'] - ) - if match['score'] > highscore: - highscore = match['score'] - alert = UnopenedReport() - alert.rawrequestid = request['requestid'] - alert.date = date.today() - alert.rank = 2 - alert.potentialmatches = {"highscore": round(highscore, 2), "matches": [{ - "requestid": m["requestrawdata"]['axisRequestId'], - "similarity": round(m['score'], 2) - } for m in potentialmatches]} - UnopenedReport.insert(alert) - alerts.append({"request": request, "rank": 2, "potentialmatches": alert.potentialmatches}) - alerts.sort(key=lambda a : a.get('potentialmatches', {'highscore': 0})['highscore']) - senderservice().send( - subject="Intake Unopened Request Report: " + str(date.today()), - content=self.generateemailhtml(alerts), - _messageattachmentlist=[], - requestjson={"email": self.reportemail, "topic": "Unopened Report"} - ) - return alerts - - - def generateemailhtml(self, alerts): - emailhtml = """ -

Unopened Report - """ + str(date.today()) + """

- -

This is a report for unopened requests in the past """ + self.dayscutoff + """ days that have not yet been actioned.

-

Rank 1: Very likely to be unactioned — unable to find a request with any matching applicant info

- - - - - - - - - - - - - """ - firstrank2 = True - print(alerts) - for alert in alerts: - if alert.get('potentialmatches') == None: - emailhtml += ''' - - - - - - - - - - - - ''' - else: - if firstrank2: - emailhtml += """
Unopened IDDate ReceivedMinistry SelectedApplicant First NameApplicant Last NamePayment StatusReceipt NumberApplication FeeDescription
U-000''' + str(alert['request']['requestid']) + '''''' + alert['request']['requestrawdata']['receivedDate'] + '''''' - for m in alert['request']['requestrawdata']['ministry']['selectedMinistry']: - emailhtml += (m['code'] + ' ') - emailhtml += '''''' + alert['request']['requestrawdata']['contactInfo']['firstName'] + '''''' + alert['request']['requestrawdata']['contactInfo']['lastName'] + '''''' + alert['request']['paymentstatus'] + '''''' + alert['request']['txnno'] + '''''' + alert['request']['fee'] + '''''' + alert['request']['requestrawdata']['descriptionTimeframe']['description'][0:99] + '''...
-

Rank 2: Possibly unactioned — requests found but some applicant info is mismatching — please double check

- - - - - - - - - - - - - - """ - firstrank2 = False - print(alert) - if alert['potentialmatches']['highscore'] > float(self.jarocutoff): - break - emailhtml += ''' - - - - - - - - - - - - - ''' - return emailhtml \ No newline at end of file diff --git a/request-management-api/requirements.txt b/request-management-api/requirements.txt index ddc56c845..abc1dec44 100644 --- a/request-management-api/requirements.txt +++ b/request-management-api/requirements.txt @@ -36,7 +36,6 @@ gunicorn==20.1.0 idna==2.10 itsdangerous==2.0.1 jaeger-client==4.4.0 -jaro-winkler==2.0.3 jsonschema==3.2.0 marshmallow-sqlalchemy==0.23.1 marshmallow==3.0.0rc7 diff --git a/sample.env b/sample.env index d8a2ab0fd..b0f5583d4 100644 --- a/sample.env +++ b/sample.env @@ -159,9 +159,4 @@ DISABLE_REDACT_WEBLINK=false DISABLE_GATHERINGRECORDS_TAB=false KC_SRC_ACC_TOKEN_EXPIRY=60 -MUTE_NOTIFICATION={"MCF": {"request_types": ["Personal"], "state_exceptions": ["Call For Records", "Ministry Sign Off"], "type_exceptions":["Reply User Comments", "Tagged User Comments"]}} - -UNOPENED_REPORT_CUTOFF_DAYS=10 -UNOPENED_REPORT_WAIT_DAYS=5 -UNOPENED_REPORT_JARO_CUTOFF=0.8 -UNOPENED_REPORT_EMAIL_RECIPIENT= \ No newline at end of file +MUTE_NOTIFICATION={"MCF": {"request_types": ["Personal"], "state_exceptions": ["Call For Records", "Ministry Sign Off"], "type_exceptions":["Reply User Comments", "Tagged User Comments"]}} \ No newline at end of file
Unopened IDDate ReceivedMinistry SelectedApplicant First NameApplicant Last NamePayment StatusReceipt NumberApplication FeePotential MatchesDescription
U-000''' + str(alert['request']['requestid']) + '''''' + alert['request']['requestrawdata']['receivedDate'] + '''''' - for m in alert['request']['requestrawdata']['ministry']['selectedMinistry']: - emailhtml += (m['code'] + ' ') - emailhtml += '''''' + alert['request']['requestrawdata']['contactInfo']['firstName'] + '''''' + alert['request']['requestrawdata']['contactInfo']['lastName'] + '''''' + alert['request']['paymentstatus'] + '''''' + alert['request']['txnno'] + '''''' + alert['request']['fee'] + ''' - ''' - for m in alert['potentialmatches']['matches']: - emailhtml += (m['requestid'] + " - similarity: " + str(m['similarity']*100) + "%
") - emailhtml = emailhtml[:-4] - emailhtml += '''		''' + alert['request']['requestrawdata']['descriptionTimeframe']['description'][0:99] + '''...