14618 - Routing Slip job changes for Corrections (#1082)

* Changes for corrections job.

* Changes required for correction job.

* Re-enable tasks.

* Comment update

jobs/payment-jobs/invoke_jobs.py

@@ -122,6 +122,7 @@ def run(job_name):
         RoutingSlipTask.link_routing_slips()
         RoutingSlipTask.process_void()
         RoutingSlipTask.process_nsf()
+        RoutingSlipTask.process_correction()
         RoutingSlipTask.adjust_routing_slips()
         application.logger.info(f'<<<< Completed Routing Slip tasks >>>>')
     elif job_name == 'EJV_PAYMENT':

jobs/payment-jobs/requirements.txt

@@ -1,6 +1,6 @@
--e git+https://github.com/bcgov/sbc-common-components.git@647fb167b7cf1e5966b3515422a7a830bfb9b63b#egg=sbc_common_components&subdirectory=python
--e git+https://github.com/bcgov/sbc-pay.git@25c0e4d9b40402ad5a750bc08f17cceaedcc6c3b#egg=pay_api&subdirectory=pay-api
-Flask-Caching==2.0.1
+-e git+https://github.com/bcgov/sbc-common-components.git@b0a0c7a40670135b4a2b32a4dd87afa8f0cbcfda#egg=sbc_common_components&subdirectory=python
+-e git+https://github.com/bcgov/sbc-pay.git@1874762ca2929976c152b3cfe60ba74b59b8df5b#egg=pay_api&subdirectory=pay-api
+Flask-Caching==2.0.2
 Flask-Migrate==2.7.0
 Flask-Moment==1.0.5
 Flask-OpenTracing==1.1.0
@@ -9,63 +9,68 @@ Flask-Script==2.0.6
 Flask==1.1.2
 Jinja2==3.0.3
 Mako==1.2.4
-MarkupSafe==2.1.1
+MarkupSafe==2.1.2
 PyNaCl==1.5.0
-SQLAlchemy-Continuum==1.3.13
-SQLAlchemy-Utils==0.38.3
+SQLAlchemy-Continuum==1.3.14
+SQLAlchemy-Utils==0.39.0
 SQLAlchemy==1.3.24
 Werkzeug==1.0.1
-alembic==1.9.0
+alembic==1.9.2
 aniso8601==9.0.1
 asyncio-nats-client==0.11.5
 asyncio-nats-streaming==0.4.0
-attrs==22.1.0
+attrs==22.2.0
 bcrypt==4.0.1
 blinker==1.5
 cachelib==0.9.0
 certifi==2022.12.7
 cffi==1.15.1
-charset-normalizer==2.1.1
+charset-normalizer==3.0.1
 click==8.1.3
 croniter==1.3.8
-cryptography==38.0.4
+cryptography==39.0.0
 cx-Oracle==8.3.0
 dataclass-wizard==0.22.2
-dpath==2.1.3
+dpath==2.1.4
 ecdsa==0.18.0
 expiringdict==1.2.2
 flask-jwt-oidc==0.3.0
 flask-marshmallow==0.11.0
-flask-restx==1.0.3
+flask-restx==1.0.5
 gunicorn==20.1.0
 idna==3.4
+importlib-metadata==6.0.0
+importlib-resources==5.10.2
 itsdangerous==2.0.1
 jaeger-client==4.8.0
 jsonschema==4.17.3
-launchdarkly-server-sdk==7.5.1
+launchdarkly-server-sdk==8.0.0
 marshmallow-sqlalchemy==0.25.0
 marshmallow==3.19.0
-minio==7.1.12
+minio==7.1.13
 opentracing==2.4.0
-packaging==22.0
-paramiko==2.12.0
+packaging==23.0
+paramiko==3.0.0
+pkgutil_resolve_name==1.3.10
 protobuf==3.19.6
 psycopg2-binary==2.9.5
 pyRFC3339==1.1
 pyasn1==0.4.8
 pycparser==2.21
-pyrsistent==0.19.2
+pyrsistent==0.19.3
 pysftp==0.2.9
 python-dateutil==2.8.2
-python-dotenv==0.21.0
+python-dotenv==0.21.1
 python-jose==3.3.0
-pytz==2022.7
-requests==2.28.1
+pytz==2022.7.1
+requests==2.28.2
 rsa==4.9
 semver==2.13.0
-sentry-sdk==1.12.1
+sentry-sdk==1.14.0
 six==1.16.0
 threadloop==1.0.2
 thrift==0.16.0
 tornado==6.2
-urllib3==1.26.13
+typing_extensions==4.4.0
+urllib3==1.26.14
+zipp==3.11.0

jobs/payment-jobs/tasks/routing_slip_task.py

@@ -47,12 +47,7 @@ def link_routing_slips(cls):
         1. Find all pending rs with pending status.
         2. Notify mailer
         """
-        routing_slips = db.session.query(RoutingSlipModel) \
-            .join(PaymentAccountModel, PaymentAccountModel.id == RoutingSlipModel.payment_account_id) \
-            .join(CfsAccountModel, CfsAccountModel.account_id == PaymentAccountModel.id) \
-            .filter(RoutingSlipModel.status == RoutingSlipStatus.LINKED.value) \
-            .filter(CfsAccountModel.status == CfsAccountStatus.ACTIVE.value).all()
-
+        routing_slips = cls._get_routing_slip_by_status(RoutingSlipStatus.LINKED.value)
         for routing_slip in routing_slips:
             # 1. Reverse the child routing slip.
             # 2. Create receipt to the parent.
@@ -77,7 +72,7 @@ def link_routing_slips(cls):
                 parent_cfs_account: CfsAccountModel = CfsAccountModel.find_effective_by_account_id(
                     parent_payment_account.id)
                 # For linked routing slip receipts, append 'L' to the number to avoid duplicate error
-                receipt_number = f'{routing_slip.number}L'
+                receipt_number = routing_slip.generate_cas_receipt_number()
                 CFSService.create_cfs_receipt(cfs_account=parent_cfs_account,
                                               rcpt_number=receipt_number,
                                               rcpt_date=routing_slip.routing_slip_date.strftime('%Y-%m-%d'),
@@ -103,6 +98,57 @@ def link_routing_slips(cls):
                 current_app.logger.error(e)
                 continue
 
+    @classmethod
+    def process_correction(cls):
+        """Process CORRECTION routing slips.
+
+        Steps:
+        1. Reverse the routing slip.
+        2. Recreate the routing slip receipt with the corrected amount.
+        3. Reset the invoices.
+        4. Reapply the invoices.
+
+        """
+        routing_slips = cls._get_routing_slip_by_status(RoutingSlipStatus.CORRECTION.value)
+        current_app.logger.info(f'Found {len(routing_slips)} to process CORRECTIONS.')
+        for rs in routing_slips:
+            try:
+                wait_for_create_invoice_job = any(x.invoice_status_code in [
+                                                   InvoiceStatus.APPROVED.value, InvoiceStatus.CREATED.value]
+                                                  for x in rs.invoices)
+                if wait_for_create_invoice_job:
+                    continue
+                current_app.logger.debug(f'Correcting Routing Slip: {rs.number}')
+                payment_account: PaymentAccountModel = PaymentAccountModel.find_by_id(rs.payment_account_id)
+                cfs_account: CfsAccountModel = CfsAccountModel.find_effective_by_account_id(payment_account.id)
+
+                CFSService.reverse_rs_receipt_in_cfs(cfs_account, rs.generate_cas_receipt_number(),
+                                                     ReverseOperation.CORRECTION.value)
+                # Update the version, which generates a new receipt number. This is to avoid duplicate receipt number.
+                rs.cas_version_suffix += 1
+                # Recreate the receipt with the modified total.
+                CFSService.create_cfs_receipt(cfs_account=cfs_account,
+                                              rcpt_number=rs.generate_cas_receipt_number(),
+                                              rcpt_date=rs.routing_slip_date.strftime('%Y-%m-%d'),
+                                              amount=rs.total,
+                                              payment_method=payment_account.payment_method,
+                                              access_token=CFSService.get_fas_token().json().get('access_token'))
+
+                cls._reset_invoices_and_references_to_created(rs)
+
+                cls._apply_routing_slips_to_pending_invoices(rs)
+
+                rs.status = RoutingSlipStatus.COMPLETE.value if rs.remaining_amount == 0 \
+                    else RoutingSlipStatus.ACTIVE.value
+
+                rs.save()
+            except Exception as e:  # NOQA # pylint: disable=broad-except
+                capture_message(
+                    f'Error on Processing CORRECTION for :={rs.number}, '
+                    f'routing slip : {rs.id}, ERROR : {str(e)}', level='error')
+                current_app.logger.error(e)
+                continue
+
     @classmethod
     def process_void(cls):
         """Process VOID routing slips.
@@ -113,12 +159,7 @@ def process_void(cls):
         3. Change the CFS Account status.
         4. Adjust the remaining amount and cas_version_suffix for VOID.
         """
-        routing_slips: List[RoutingSlipModel] = db.session.query(RoutingSlipModel) \
-            .join(PaymentAccountModel, PaymentAccountModel.id == RoutingSlipModel.payment_account_id) \
-            .join(CfsAccountModel, CfsAccountModel.account_id == PaymentAccountModel.id) \
-            .filter(RoutingSlipModel.status == RoutingSlipStatus.VOID.value) \
-            .filter(CfsAccountModel.status == CfsAccountStatus.ACTIVE.value).all()
-
+        routing_slips = cls._get_routing_slip_by_status(RoutingSlipStatus.VOID.value)
         current_app.logger.info(f'Found {len(routing_slips)} to process VOID.')
         for routing_slip in routing_slips:
             try:
@@ -134,9 +175,7 @@ def process_void(cls):
                 # Reverse all child routing slips, as all linked routing slips are also considered as VOID.
                 child_routing_slips: List[RoutingSlipModel] = RoutingSlipModel.find_children(routing_slip.number)
                 for rs in (routing_slip, *child_routing_slips):
-                    receipt_number = rs.number
-                    if rs.parent_number:
-                        receipt_number = f'{receipt_number}L'
+                    receipt_number = rs.generate_cas_receipt_number()
                     CFSService.reverse_rs_receipt_in_cfs(cfs_account, receipt_number, ReverseOperation.VOID.value)
                 # Void routing slips aren't supposed to have pending transactions, so no need to look at invoices.
                 cfs_account.status = CfsAccountStatus.INACTIVE.value
@@ -160,12 +199,7 @@ def process_nsf(cls):
         2. Reverse the receipt for the NSF routing slips.
         3. Add an invoice for NSF fees.
         """
-        routing_slips: List[RoutingSlipModel] = db.session.query(RoutingSlipModel) \
-            .join(PaymentAccountModel, PaymentAccountModel.id == RoutingSlipModel.payment_account_id) \
-            .join(CfsAccountModel, CfsAccountModel.account_id == PaymentAccountModel.id) \
-            .filter(RoutingSlipModel.status == RoutingSlipStatus.NSF.value) \
-            .filter(CfsAccountModel.status == CfsAccountStatus.ACTIVE.value).all()
-
+        routing_slips = cls._get_routing_slip_by_status(RoutingSlipStatus.NSF.value)
         current_app.logger.info(f'Found {len(routing_slips)} to process NSF.')
         for routing_slip in routing_slips:
             # 1. Reverse the routing slip receipt.
@@ -179,33 +213,16 @@ def process_nsf(cls):
                 # Find all child routing slip and reverse it, as all linked routing slips are also considered as NSF.
                 child_routing_slips: List[RoutingSlipModel] = RoutingSlipModel.find_children(routing_slip.number)
                 for rs in (routing_slip, *child_routing_slips):
-                    receipt_number = rs.number
-                    if rs.parent_number:
-                        receipt_number = f'{receipt_number}L'
+                    receipt_number = rs.generate_cas_receipt_number()
                     CFSService.reverse_rs_receipt_in_cfs(cfs_account, receipt_number, ReverseOperation.NSF.value)
 
                     for payment in db.session.query(PaymentModel) \
                             .filter(PaymentModel.receipt_number == receipt_number).all():
                         payment.payment_status_code = PaymentStatus.FAILED.value
 
-                # Update the CFS Account status to FREEZE.
                 cfs_account.status = CfsAccountStatus.FREEZE.value
 
-                # Update all invoice status to CREATED.
-                invoices: List[InvoiceModel] = db.session.query(InvoiceModel) \
-                    .filter(InvoiceModel.routing_slip == routing_slip.number) \
-                    .filter(InvoiceModel.invoice_status_code == InvoiceStatus.PAID.value) \
-                    .all()
-                for inv in invoices:
-                    # Reset the statuses
-                    inv.invoice_status_code = InvoiceStatus.CREATED.value
-                    inv_ref = InvoiceReferenceModel.find_by_invoice_id_and_status(
-                        inv.id, InvoiceReferenceStatus.COMPLETED.value
-                    )
-                    inv_ref.status_code = InvoiceReferenceStatus.ACTIVE.value
-                    # Delete receipts as receipts are reversed in CFS.
-                    for receipt in ReceiptModel.find_all_receipts_for_invoice(inv.id):
-                        db.session.delete(receipt)
+                cls._reset_invoices_and_references_to_created(routing_slip)
 
                 inv = cls._create_nsf_invoice(cfs_account, routing_slip.number, payment_account)
                 # Reduce the NSF fee from remaining amount.
@@ -244,10 +261,9 @@ def adjust_routing_slips(cls):
                 # Find all child routing slip and reverse it, as all linked routing slips are also considered as NSF.
                 child_routing_slips: List[RoutingSlipModel] = RoutingSlipModel.find_children(routing_slip.number)
                 for rs in (routing_slip, *child_routing_slips):
-                    receipt_number = rs.number
+
                     is_refund = routing_slip.status == RoutingSlipStatus.REFUND_AUTHORIZED.value
-                    if rs.parent_number:
-                        receipt_number = f'{receipt_number}L'
+                    receipt_number = rs.generate_cas_receipt_number()
                     # Adjust the receipt to zero in CFS
                     CFSService.adjust_receipt_to_zero(cfs_account, receipt_number, is_refund)
 
@@ -262,6 +278,33 @@ def adjust_routing_slips(cls):
                 current_app.logger.error(e)
                 continue
 
+    @classmethod
+    def _get_routing_slip_by_status(cls, status: RoutingSlipStatus) -> List[RoutingSlipModel]:
+        """Get routing slip by status."""
+        return db.session.query(RoutingSlipModel) \
+            .join(PaymentAccountModel, PaymentAccountModel.id == RoutingSlipModel.payment_account_id) \
+            .join(CfsAccountModel, CfsAccountModel.account_id == PaymentAccountModel.id) \
+            .filter(RoutingSlipModel.status == status) \
+            .filter(CfsAccountModel.status == CfsAccountStatus.ACTIVE.value).all()
+
+    @classmethod
+    def _reset_invoices_and_references_to_created(cls, routing_slip: RoutingSlipModel):
+        """Reset Invoices, Invoice references and Receipts for routing slip."""
+        invoices: List[InvoiceModel] = db.session.query(InvoiceModel) \
+            .filter(InvoiceModel.routing_slip == routing_slip.number) \
+            .filter(InvoiceModel.invoice_status_code == InvoiceStatus.PAID.value) \
+            .all()
+        for inv in invoices:
+            # Reset the statuses
+            inv.invoice_status_code = InvoiceStatus.CREATED.value
+            inv_ref = InvoiceReferenceModel.find_by_invoice_id_and_status(
+                inv.id, InvoiceReferenceStatus.COMPLETED.value
+            )
+            inv_ref.status_code = InvoiceReferenceStatus.ACTIVE.value
+            # Delete receipts as receipts are reversed in CFS.
+            for receipt in ReceiptModel.find_all_receipts_for_invoice(inv.id):
+                db.session.delete(receipt)
+
     @classmethod
     def _create_nsf_invoice(cls, cfs_account: CfsAccountModel, rs_number: str,
                             payment_account: PaymentAccountModel) -> InvoiceModel:
@@ -319,8 +362,8 @@ def _create_nsf_invoice(cls, cfs_account: CfsAccountModel, rs_number: str,
 
     @classmethod
     def _apply_routing_slips_to_pending_invoices(cls, routing_slip: RoutingSlipModel) -> float:
-        """Apply the routing slips again, when routing slip is linked to an NSF parent."""
-        current_app.logger.info(f'Starting NSF recovery process for {routing_slip.number}')
+        """Apply the routing slips again."""
+        current_app.logger.info(f'Applying routing slips to pending invoices for routing slip: {routing_slip.number}')
         routing_slip_payment_account: PaymentAccountModel = PaymentAccountModel.find_by_id(
             routing_slip.payment_account_id)
 
@@ -364,12 +407,9 @@ def apply_routing_slips_to_invoice(cls,  # pylint: disable = too-many-arguments,
         for routing_slip in (parent_routing_slip, *child_routing_slips):
             try:
                 # apply receipt now
-                current_app.logger.debug(f'Apply receipt {routing_slip.number} on invoice {invoice_number} '
+                receipt_number = routing_slip.generate_cas_receipt_number()
+                current_app.logger.debug(f'Apply receipt {receipt_number} on invoice {invoice_number} '
                                          f'for routing slip {routing_slip.number}')
-                receipt_number = routing_slip.number
-                # For linked routing slips, new receipt numbers ends with 'L'
-                if routing_slip.status == RoutingSlipStatus.LINKED.value:
-                    receipt_number = f'{routing_slip.number}L'
 
                 # If balance of receipt is zero, continue to next receipt.
                 receipt_balance_before_apply = float(

jobs/payment-jobs/tests/jobs/test_routing_slip_task.py

@@ -186,6 +186,46 @@ def test_process_void(session):
         mock_cfs_reverse_2.assert_not_called()
 
 
+def test_process_correction(session):
+    """Test Routing slip set to CORRECTION."""
+    number = '1111111'
+    pay_account = factory_routing_slip_account(number=number, status=CfsAccountStatus.ACTIVE.value, total=10)
+    # Create an invoice for the routing slip
+    # Create an invoice record against this routing slip.
+    invoice = factory_invoice(payment_account=pay_account, total=30,
+                              status_code=InvoiceStatus.PAID.value,
+                              payment_method_code=PaymentMethod.INTERNAL.value,
+                              routing_slip=number)
+
+    fee_schedule = FeeScheduleModel.find_by_filing_type_and_corp_type('CP', 'OTANN')
+    line = factory_payment_line_item(invoice.id, fee_schedule_id=fee_schedule.fee_schedule_id)
+    line.save()
+
+    # Create invoice reference
+    factory_invoice_reference(invoice.id, status_code=InvoiceReferenceStatus.COMPLETED.value)
+
+    # Create receipts for the invoices
+    factory_receipt(invoice.id, number)
+
+    rs = RoutingSlipModel.find_by_number(number)
+    rs.status = RoutingSlipStatus.CORRECTION.value
+    rs.total = 900
+    rs.save()
+
+    session.commit()
+
+    with patch('pay_api.services.CFSService.reverse_rs_receipt_in_cfs') as mock_reverse:
+        with patch('pay_api.services.CFSService.create_cfs_receipt') as mock_create_receipt:
+            with patch('pay_api.services.CFSService.get_invoice') as mock_get_invoice:
+                RoutingSlipTask.process_correction()
+                mock_reverse.assert_called()
+                mock_get_invoice.assert_called()
+                mock_create_receipt.assert_called()
+
+    assert rs.status == RoutingSlipStatus.COMPLETE.value
+    assert rs.cas_version_suffix == 2
+
+
 def test_link_to_nsf_rs(session):
     """Test routing slip with NSF as parent."""
     child_rs_number = '1234'