13309 - $0 Totals stuck in Approved for EJV/PAD (#977)

* Move distribution code back to being set on total = 0, because it's used for getting service fees for EJV payment processing.

* Remove line.total == 0 skip.

* Unit test fixes.

* Fix linting issues.

* Minor unit test fix.

* Remove line_item.total == 0 skip, because there can be service fees for the line item.

* Small fix for cfs_service.

* Fix service_fees line in PLI

* Partial revert "Fix service_fees line in PLI" - delete linked folders

* Fix _build_lines, so it works properly

* Missing definition

* Unit test for creating pad invoice, mixed pli values.

* Target BCGOV branch

* Update version number.

* Temp move dependencies

jobs/payment-jobs/requirements.txt

@@ -1,14 +1,14 @@
--e git+https://github.com/bcgov/sbc-common-components.git@4efa1e6664dff321030eca7448a7d5af91c2f355#egg=sbc_common_components&subdirectory=python
--e git+https://github.com/bcgov/sbc-pay.git@95cd9254ed89d01cd571918aa845bbe7b85f348b#egg=pay_api&subdirectory=pay-api
+-e git+https://github.com/bcgov/sbc-common-components.git@02f90e3a1c9b6eeff424ecec758d58a43d2cfc9b#egg=sbc_common_components&subdirectory=python
+-e git+https://github.com/bcgov/sbc-pay.git@fbc2d2bb932f04ec9e5a690d60c06f5387220de1#egg=pay_api&subdirectory=pay-api
 Flask-Caching==2.0.1
 Flask-Migrate==2.7.0
-Flask-Moment==1.0.4
+Flask-Moment==1.0.5
 Flask-OpenTracing==1.1.0
 Flask-SQLAlchemy==2.5.1
 Flask-Script==2.0.6
 Flask==1.1.2
 Jinja2==3.0.3
-Mako==1.2.2
+Mako==1.2.3
 MarkupSafe==2.1.1
 PyNaCl==1.5.0
 SQLAlchemy-Continuum==1.3.13
@@ -20,37 +20,37 @@ aniso8601==9.0.1
 asyncio-nats-client==0.11.5
 asyncio-nats-streaming==0.4.0
 attrs==22.1.0
-bcrypt==4.0.0
+bcrypt==4.0.1
 blinker==1.5
 cachelib==0.9.0
-certifi==2022.6.15.2
+certifi==2022.9.24
 cffi==1.15.1
 charset-normalizer==2.1.1
 click==8.1.3
 croniter==1.3.7
 cryptography==38.0.1
-dataclass-wizard==0.22.1
+dataclass-wizard==0.22.2
 dpath==2.0.6
 ecdsa==0.18.0
 flask-jwt-oidc==0.3.0
 flask-marshmallow==0.11.0
 flask-restx==0.5.1
 gunicorn==20.1.0
 idna==3.4
-importlib-metadata==4.12.0
-importlib-resources==5.9.0
+importlib-metadata==5.0.0
+importlib-resources==5.10.0
 itsdangerous==2.0.1
 jaeger-client==4.8.0
 jsonschema==4.16.0
 marshmallow-sqlalchemy==0.25.0
-marshmallow==3.17.1
-minio==7.1.11
+marshmallow==3.18.0
+minio==7.1.12
 opentracing==2.4.0
 packaging==21.3
 paramiko==2.11.0
 pkgutil_resolve_name==1.3.10
-protobuf==3.19.5
-psycopg2-binary==2.9.3
+protobuf==3.19.6
+psycopg2-binary==2.9.4
 pyasn1==0.4.8
 pycparser==2.21
 pyparsing==3.0.9
@@ -59,14 +59,14 @@ pysftp==0.2.9
 python-dateutil==2.8.2
 python-dotenv==0.21.0
 python-jose==3.3.0
-pytz==2022.2.1
+pytz==2022.5
 requests==2.28.1
 rsa==4.9
-sentry-sdk==1.9.8
+sentry-sdk==1.9.10
 six==1.16.0
 threadloop==1.0.2
 thrift==0.16.0
 tornado==6.2
-typing_extensions==4.3.0
+typing_extensions==4.4.0
 urllib3==1.26.12
-zipp==3.8.1
+zipp==3.9.0

jobs/payment-jobs/requirements/prod.txt

@@ -16,5 +16,5 @@ itsdangerous==2.0.1
 dataclass_wizard
 
 -e git+https://github.com/bcgov/sbc-common-components.git#egg=sbc-common-components&subdirectory=python
--e git+https://github.com/bcgov/sbc-pay.git#egg=pay_api&subdirectory=pay-api
+-e git+https://github.com/bcgov/sbc-pay.git@13309#egg=pay_api&subdirectory=pay-api
 

jobs/payment-jobs/tasks/ejv_payment_task.py

@@ -114,8 +114,6 @@ def _create_ejv_file_for_gov_account(cls, batch_type: str):  # pylint:disable=to
                 line_items = inv.payment_line_items
 
                 for line in line_items:
-                    if line.total == 0:
-                        continue
                     # Line can have 2 distribution, 1 for the total and another one for service fees.
                     line_distribution_code: DistributionCodeModel = DistributionCodeModel.find_by_id(
                         line.fee_distribution_id)

jobs/payment-jobs/tests/jobs/test_cfs_create_invoice_task.py

@@ -20,6 +20,7 @@
 from datetime import datetime, timedelta
 from unittest.mock import patch
 
+from pay_api.models import DistributionCode as DistributionCodeModel
 from pay_api.models import FeeSchedule as FeeScheduleModel
 from pay_api.models import Invoice as InvoiceModel
 from pay_api.models import InvoiceReference as InvoiceReferenceModel
@@ -65,6 +66,36 @@ def test_create_pad_invoice_single_transaction(session):
     assert updated_invoice.invoice_status_code == InvoiceStatus.APPROVED.value
 
 
+def test_create_pad_invoice_mixed_pli_values(session):
+    """Assert PAD invoices are created with total = 0, service fees > 0."""
+    # Create an account and an invoice for the account
+    account = factory_create_pad_account(auth_account_id='1', status=CfsAccountStatus.ACTIVE.value)
+    previous_day = datetime.now() - timedelta(days=1)
+    # Create an invoice for this account
+    invoice = factory_invoice(payment_account=account, created_on=previous_day, total=1.5,
+                              status_code=InvoiceStatus.APPROVED.value, payment_method_code=None)
+
+    fee_schedule = FeeScheduleModel.find_by_filing_type_and_corp_type('PPR', 'FSREG')
+    dist_code = DistributionCodeModel.find_by_active_for_fee_schedule(fee_schedule.fee_schedule_id)
+    # We need a dist code, if we're charging service fees.
+    dist_code.service_fee_distribution_code_id = 1
+    dist_code.save()
+    line = factory_payment_line_item(
+        invoice.id, fee_schedule_id=fee_schedule.fee_schedule_id, total=0, service_fees=1.5
+    )
+    line.save()
+    assert invoice.invoice_status_code == InvoiceStatus.APPROVED.value
+
+    CreateInvoiceTask.create_invoices()
+
+    updated_invoice: InvoiceModel = InvoiceModel.find_by_id(invoice.id)
+    inv_ref: InvoiceReferenceModel = InvoiceReferenceModel. \
+        find_reference_by_invoice_id_and_status(invoice.id, InvoiceReferenceStatus.ACTIVE.value)
+
+    assert inv_ref
+    assert updated_invoice.invoice_status_code == InvoiceStatus.APPROVED.value
+
+
 def test_create_rs_invoice_single_transaction(session):
     """Assert PAD invoices are created."""
     # Create an account and an invoice for the account

pay-api/src/pay_api/services/cfs_service.py

@@ -375,43 +375,42 @@ def _build_lines(cls, payment_line_items: List[PaymentLineItemModel], negate: bo
         lines_map = defaultdict(dict)  # To group all the lines with same GL code together.
         index: int = 0
         for line_item in payment_line_items:
-            if line_item.total == 0:
-                continue
             # Find the distribution from the above list
             distribution_code = [dist for dist in distribution_codes if
                                  dist.distribution_code_id == line_item.fee_distribution_id][0] \
                 if line_item.fee_distribution_id else None
 
-            # Check if a line with same GL code exists, if YES just add up the amount. if NO, create a new line.
-            line = lines_map[distribution_code.distribution_code_id]
-
-            if not line:
-                index = index + 1
-                distribution = [dict(
-                    dist_line_number=index,
-                    amount=cls._get_amount(line_item.total, negate),
-                    account=f'{distribution_code.client}.{distribution_code.responsibility_centre}.'
-                            f'{distribution_code.service_line}.{distribution_code.stob}.'
-                            f'{distribution_code.project_code}.000000.0000'
-                )] if distribution_code else None
-
-                line = {
-                    'line_number': index,
-                    'line_type': CFS_LINE_TYPE,
-                    'description': line_item.description,
-                    'unit_price': cls._get_amount(line_item.total, negate),
-                    'quantity': 1,
-                    # 'attribute1': line_item.invoice_id,
-                    # 'attribute2': line_item.id,
-                    'distribution': distribution
-                }
-            else:
-                # Add up the price and distribution
-                line['unit_price'] = line['unit_price'] + cls._get_amount(line_item.total, negate)
-                line['distribution'][0]['amount'] = line['distribution'][0]['amount'] + \
-                    cls._get_amount(line_item.total, negate)
+            if line_item.total > 0:
+                # Check if a line with same GL code exists, if YES just add up the amount. if NO, create a new line.
+                line = lines_map[distribution_code.distribution_code_id]
+
+                if not line:
+                    index = index + 1
+                    distribution = [dict(
+                        dist_line_number=index,
+                        amount=cls._get_amount(line_item.total, negate),
+                        account=f'{distribution_code.client}.{distribution_code.responsibility_centre}.'
+                                f'{distribution_code.service_line}.{distribution_code.stob}.'
+                                f'{distribution_code.project_code}.000000.0000'
+                    )] if distribution_code else None
+
+                    line = {
+                        'line_number': index,
+                        'line_type': CFS_LINE_TYPE,
+                        'description': line_item.description,
+                        'unit_price': cls._get_amount(line_item.total, negate),
+                        'quantity': 1,
+                        # 'attribute1': line_item.invoice_id,
+                        # 'attribute2': line_item.id,
+                        'distribution': distribution
+                    }
+                else:
+                    # Add up the price and distribution
+                    line['unit_price'] = line['unit_price'] + cls._get_amount(line_item.total, negate)
+                    line['distribution'][0]['amount'] = line['distribution'][0]['amount'] + \
+                        cls._get_amount(line_item.total, negate)
 
-            lines_map[distribution_code.distribution_code_id] = line
+                lines_map[distribution_code.distribution_code_id] = line
 
             if line_item.service_fees > 0:
                 service_fee_distribution: DistributionCodeModel = DistributionCodeModel.find_by_id(

pay-api/src/pay_api/services/payment_line_item.py

@@ -292,7 +292,7 @@ def create(invoice_id: int, fee: FeeSchedule, **kwargs):
 
         # Set distribution details to line item
         distribution_code = None
-        if p.total > 0:
+        if p.total > 0 or p.service_fees > 0:
             distribution_code = DistributionCodeModel.find_by_active_for_fee_schedule(fee.fee_schedule_id)
             p.fee_distribution_id = distribution_code.distribution_code_id
 

pay-api/src/pay_api/version.py

@@ -22,4 +22,4 @@
 Development release segment: .devN
 """
 
-__version__ = '1.17.1'  # pylint: disable=invalid-name
+__version__ = '1.17.2'  # pylint: disable=invalid-name

pay-api/tests/unit/api/test_account.py

@@ -25,6 +25,8 @@
 from requests.exceptions import ConnectionError
 
 from pay_api.exceptions import ServiceUnavailableException
+from pay_api.models.distribution_code import DistributionCodeLink as DistributionCodeLinkModel
+from pay_api.models.fee_schedule import FeeSchedule
 from pay_api.models.invoice import Invoice
 from pay_api.models.payment_account import PaymentAccount
 from pay_api.schemas import utils as schema_utils
@@ -77,6 +79,14 @@ def test_account_purchase_history_pagination(session, client, jwt, app):
 
 def test_account_purchase_history_with_service_account(session, client, jwt, app):
     """Assert that purchase history returns only invoices for that product."""
+    # Point CSO fee schedule to a valid distribution code.
+    fee_schedule_id = FeeSchedule.find_by_filing_type_and_corp_type('CSO', 'CSBVFEE').fee_schedule_id
+    DistributionCodeLinkModel(fee_schedule_id=fee_schedule_id, distribution_code_id=1).save()
+
+    # Point PPR fee schedule to a valid distribution code.
+    fee_schedule_id = FeeSchedule.find_by_filing_type_and_corp_type('PPR', 'FSDIS').fee_schedule_id
+    DistributionCodeLinkModel(fee_schedule_id=fee_schedule_id, distribution_code_id=1).save()
+
     # Create one invoice for CSO and one fpr BUSINESS.
     # Then query without any filter and make sure only CSO invoice is returned for service account with CSO product_code
     for corp_filing_type in (['CSO', 'CSBVFEE'], ['PPR', 'FSDIS']):
@@ -103,6 +113,10 @@ def test_account_purchase_history_with_service_account(session, client, jwt, app
 
 def test_payment_request_for_cso_with_service_account(session, client, jwt, app):
     """Assert Service charge is calculated based on quantity."""
+    # Point CSO fee schedule to a valid distribution code.
+    fee_schedule_id = FeeSchedule.find_by_filing_type_and_corp_type('CSO', 'CSBVFEE').fee_schedule_id
+    DistributionCodeLinkModel(fee_schedule_id=fee_schedule_id, distribution_code_id=1).save()
+
     quantity = 2
     token = jwt.create_jwt(get_claims(roles=[Role.SYSTEM.value], product_code='CSO'), token_header)
     headers = {'Authorization': f'Bearer {token}', 'content-type': 'application/json'}

pay-api/tests/unit/api/test_payment_request.py

@@ -26,6 +26,7 @@
 from requests.exceptions import ConnectionError
 
 from pay_api.models import DistributionCode as DistributionCodeModel
+from pay_api.models import DistributionCodeLink as DistributionCodeLinkModel
 from pay_api.models import FeeSchedule as FeeScheduleModel
 from pay_api.models import PaymentAccount as PaymentAccountModel
 from pay_api.models import RoutingSlip as RoutingSlipModel
@@ -78,6 +79,10 @@ def test_payment_creation_using_direct_pay(session, client, jwt, app):
 ])
 def test_payment_creation_with_service_account(session, client, jwt, app, payload, product_code_claim, expected_status):
     """Assert that the endpoint returns 201."""
+    # Point CSO fee schedule to a valid distribution code.
+    fee_schedule_id = FeeScheduleModel.find_by_filing_type_and_corp_type('CSO', 'CSCRMTFC').fee_schedule_id
+    DistributionCodeLinkModel(fee_schedule_id=fee_schedule_id, distribution_code_id=1).save()
+
     token = jwt.create_jwt(
         get_claims(roles=[Role.SYSTEM.value, Role.EDITOR.value], product_code=product_code_claim),
         token_header