15719 - Skip EJV Disbursement / Payment on holidays (#1155)

* Changes to check for a stat holiday, plus some LD flag cleanup.

* Move is_holiday to seperate utils file. Put in check for disbursement as well.

* Lint cleanup

* Fix import.

* Update requirements for payment reconciliation.

* Logger disabled fix.

* Fix logging for unit test.

* propogate=0

jobs/payment-jobs/logging.conf

@@ -1,5 +1,5 @@
 [loggers]
-keys=root,api
+keys=root,api,invoke_jobs
 
 [handlers]
 keys=console
@@ -17,6 +17,12 @@ handlers=console
 qualname=api
 propagate=0
 
+[logger_invoke_jobs]
+level=DEBUG
+handlers=console
+qualname=invoke_jobs
+propagate=0
+
 [handler_console]
 class=StreamHandler
 level=DEBUG

jobs/payment-jobs/requirements.txt

@@ -1,5 +1,5 @@
 -e git+https://github.com/bcgov/sbc-common-components.git@a8a2074b12dbf36cc23634206e37d57ccfa5a33d#egg=sbc_common_components&subdirectory=python
--e git+https://github.com/bcgov/sbc-pay.git@26bd80b7a07c1d5c48433a1be58f18152277af56#egg=pay_api&subdirectory=pay-api
+-e git+https://github.com/bcgov/sbc-pay.git@858402b14285103f9c2f168d9541531893184b04#egg=pay_api&subdirectory=pay-api
 Flask-Caching==2.0.2
 Flask-Migrate==2.7.0
 Flask-Moment==1.0.5
@@ -10,6 +10,7 @@ Flask==1.1.2
 Jinja2==3.0.3
 Mako==1.2.4
 MarkupSafe==2.1.2
+PyMeeus==0.5.12
 PyNaCl==1.5.0
 SQLAlchemy-Continuum==1.3.14
 SQLAlchemy-Utils==0.40.0
@@ -20,6 +21,7 @@ aniso8601==9.0.1
 asyncio-nats-client==0.11.5
 asyncio-nats-streaming==0.4.0
 attrs==22.2.0
+backports.zoneinfo==0.2.1
 bcrypt==4.0.1
 blinker==1.5
 cachelib==0.9.0
@@ -28,6 +30,7 @@ certifi==2022.12.7
 cffi==1.15.1
 charset-normalizer==3.1.0
 click==8.1.3
+convertdate==2.4.0
 croniter==1.3.8
 cryptography==40.0.1
 cx-Oracle==8.3.0
@@ -40,12 +43,15 @@ flask-jwt-oidc==0.3.0
 flask-marshmallow==0.11.0
 flask-restx==1.1.0
 gunicorn==20.1.0
+hijri-converter==2.2.4
+holidays==0.21.13
 idna==3.4
 importlib-metadata==6.1.0
 importlib-resources==5.12.0
 itsdangerous==2.0.1
 jaeger-client==4.8.0
 jsonschema==4.17.3
+korean-lunar-calendar==0.3.1
 launchdarkly-server-sdk==8.1.1
 marshmallow-sqlalchemy==0.25.0
 marshmallow==3.19.0
@@ -69,7 +75,7 @@ pytz==2023.2
 requests==2.28.2
 rsa==4.9
 semver==2.13.0
-sentry-sdk==1.17.0
+sentry-sdk==1.18.0
 six==1.16.0
 threadloop==1.0.2
 thrift==0.16.0

jobs/payment-jobs/requirements/prod.txt

@@ -18,3 +18,5 @@ dataclass_wizard
 launchdarkly-server-sdk
 cx_Oracle
 more_itertools
+holidays
+pytz

jobs/payment-jobs/tasks/ejv_partner_distribution_task.py

@@ -33,6 +33,7 @@
 from sqlalchemy import Date, cast
 
 from tasks.common.cgi_ejv import CgiEjv
+from utils.date import is_holiday
 
 
 class EjvPartnerDistributionTask(CgiEjv):
@@ -49,6 +50,10 @@ def create_ejv_file(cls):
         4. Upload to sftp for processing. First upload JV file and then a TRG file.
         5. Update the statuses and create records to for the batch.
         """
+        if is_holiday():
+            current_app.logger.info('Deferring ejv disbursement task for another day.')
+            return
+
         cls._create_ejv_file_for_partner(batch_type='GI')  # Internal ministry
         cls._create_ejv_file_for_partner(batch_type='GA')  # External ministry
 

jobs/payment-jobs/tasks/ejv_payment_task.py

@@ -29,6 +29,7 @@
 from pay_api.utils.util import generate_transaction_number
 
 from tasks.common.cgi_ejv import CgiEjv
+from utils.date import is_holiday
 
 
 class EjvPaymentTask(CgiEjv):
@@ -46,6 +47,10 @@ def create_ejv_file(cls):
         5. Upload to sftp for processing. First upload JV file and then a TRG file.
         6. Update the statuses and create records to for the batch.
         """
+        if is_holiday():
+            current_app.logger.info('Deferring ejv payment task for another day.')
+            return
+
         cls._create_ejv_file_for_gov_account(batch_type='GI')
         cls._create_ejv_file_for_gov_account(batch_type='GA')
 

jobs/payment-jobs/tests/jobs/conftest.py

@@ -25,6 +25,7 @@
 from sqlalchemy.schema import DropConstraint, MetaData
 
 from invoke_jobs import create_app
+from utils.logger import setup_logging
 
 
 @pytest.fixture(scope='session')
@@ -118,6 +119,8 @@ def db(app):  # pylint: disable=redefined-outer-name, invalid-name
         Migrate(app, _db, directory=migrations_path)
         upgrade()
 
+        # Restore the logging, alembic and sqlalchemy have their own logging from alembic.ini.
+        setup_logging(os.path.abspath('logging.conf'))
         return _db
 
 

jobs/payment-jobs/tests/jobs/test_ejv_payment_task.py

@@ -16,6 +16,8 @@
 
 Test-Suite to ensure that the CgiEjvJob is working as expected.
 """
+import unittest
+from freezegun import freeze_time
 from pay_api.models import (
     DistributionCode, EjvFile, EjvHeader, EjvInvoiceLink, FeeSchedule, Invoice, InvoiceReference, db)
 from pay_api.utils.enums import DisbursementStatus, EjvFileType, InvoiceReferenceStatus, InvoiceStatus
@@ -137,3 +139,25 @@ def test_payments_for_gov_accounts(session, monkeypatch):
         ejv_file: EjvFile = EjvFile.find_by_id(ejv_header.ejv_file_id)
         assert ejv_file
         assert ejv_file.file_type == EjvFileType.PAYMENT.value
+
+
+def test_ejv_skip_holidays(session):
+    """Assert that the EJV job skips holidays."""
+    test_case = unittest.TestCase()
+    # Christmas
+    with test_case.assertLogs('invoke_jobs') as log:
+        with freeze_time('2023-12-25 00:00:00T08:00:00'):
+            EjvPaymentTask.create_ejv_file()
+            assert [message for message in log.output if 'stat holiday' in message]
+
+    # New Years
+    with test_case.assertLogs('invoke_jobs') as log:
+        with freeze_time('2024-01-01 00:00:00T08:00:00'):
+            EjvPaymentTask.create_ejv_file()
+            assert [message for message in log.output if 'stat holiday' in message]
+
+    # Labour Day
+    with test_case.assertLogs('invoke_jobs') as log:
+        with freeze_time('2023-09-04 00:00:00T08:00:00'):
+            EjvPaymentTask.create_ejv_file()
+            assert [message for message in log.output if 'stat holiday' in message]

jobs/payment-jobs/utils/date.py

@@ -0,0 +1,13 @@
+
+from datetime import datetime
+import holidays
+import pytz
+
+from flask import current_app
+
+def is_holiday():
+    current_date_pacific = datetime.now(tz=pytz.utc).astimezone(pytz.timezone('US/Pacific')).strftime('%Y-%m-%d')
+    if holiday := holidays.CA(state='BC', observed=False).get(current_date_pacific):
+        current_app.logger.info(f'Today is a stat holiday {holiday} on {current_date_pacific}')
+        return True
+    return False

jobs/payment-jobs/utils/logger.py

@@ -26,6 +26,6 @@ def setup_logging(conf):
 
     if conf and path.isfile(conf):
         logging.config.fileConfig(conf)
-        print('Configure logging, from conf:{}'.format(conf), file=sys.stdout)
+        print(f'Configure logging, from conf:{conf}', file=sys.stdout)
     else:
-        print('Unable to configure logging, attempted conf:{}'.format(conf), file=sys.stderr)
+        print(f'Unable to configure logging, attempted conf:{conf}', file=sys.stderr)

pay-api/src/pay_api/services/fee_schedule.py

@@ -24,7 +24,6 @@
 from pay_api.models import FeeCode as FeeCodeModel
 from pay_api.models import FeeSchedule as FeeScheduleModel
 from pay_api.models import FeeScheduleSchema
-from pay_api.services.flags import flags
 from pay_api.utils.enums import Role
 from pay_api.utils.errors import Error
 from pay_api.utils.user_context import UserContext, user_context
@@ -312,11 +311,6 @@ def find_by_corp_type_and_filing_type(  # pylint: disable=too-many-arguments
         # Set transaction fees
         fee_schedule.service_fees = FeeSchedule.calculate_service_fees(fee_schedule_dao, account_fee)
 
-        # Special case for CSO partner type which is different from normal flow
-        if flags.is_on('BAD_CSO_SERVICE_FEE', default=True) and fee_schedule.corp_type_code == 'CSO' \
-           and fee_schedule.quantity:
-            fee_schedule.service_fees = fee_schedule.service_fees * fee_schedule.quantity
-
         if kwargs.get('is_priority') and fee_schedule_dao.priority_fee and apply_filing_fees:
             fee_schedule.priority_fee = fee_schedule_dao.priority_fee.amount
         if kwargs.get('is_future_effective') and fee_schedule_dao.future_effective_fee and apply_filing_fees:

queue_services/payment-reconciliations/requirements.txt

@@ -1,6 +1,6 @@
--e git+https://github.com/bcgov/lear.git@1817e8ea1d43054845384125cdcb92273f1b53c9#egg=entity_queue_common&subdirectory=queue_services/common
--e git+https://github.com/bcgov/sbc-common-components.git@a643801c373063dfa44a2f7213d9122d3ad03851#egg=sbc_common_components&subdirectory=python
--e git+https://github.com/bcgov/sbc-pay.git@e722e68450ac28e9c5f3352b10edce2d0388c327#egg=pay_api&subdirectory=pay-api
+-e git+https://github.com/bcgov/lear.git@757ebe62916fe4e9467cf8505ddd41e15ae3282e#egg=entity_queue_common&subdirectory=queue_services/common
+-e git+https://github.com/bcgov/sbc-common-components.git@a8a2074b12dbf36cc23634206e37d57ccfa5a33d#egg=sbc_common_components&subdirectory=python
+-e git+https://github.com/bcgov/sbc-pay.git@858402b14285103f9c2f168d9541531893184b04#egg=pay_api&subdirectory=pay-api
 Flask-Caching==2.0.2
 Flask-Migrate==2.7.0
 Flask-Moment==1.0.5
@@ -17,38 +17,40 @@ SQLAlchemy==1.3.24
 Werkzeug==1.0.1
 aiohttp==3.8.4
 aiosignal==1.3.1
-alembic==1.9.4
+alembic==1.10.2
 aniso8601==9.0.1
 async-timeout==4.0.2
 asyncio-nats-client==0.11.5
 asyncio-nats-streaming==0.4.0
 attrs==22.2.0
 blinker==1.5
 cachelib==0.9.0
+cattrs==22.2.0
 certifi==2022.12.7
 cffi==1.15.1
-charset-normalizer==3.0.1
+charset-normalizer==3.1.0
 click==8.1.3
 croniter==1.3.8
-cryptography==39.0.1
-dpath==2.1.4
+cryptography==40.0.1
+dpath==2.1.5
 ecdsa==0.18.0
+exceptiongroup==1.1.1
 expiringdict==1.2.2
 flask-jwt-oidc==0.3.0
 flask-marshmallow==0.11.0
-flask-restx==1.0.6
+flask-restx==1.1.0
 frozenlist==1.3.3
 gunicorn==20.1.0
 idna==3.4
-importlib-metadata==6.0.0
+importlib-metadata==6.1.0
 importlib-resources==5.12.0
 itsdangerous==2.0.1
 jaeger-client==4.8.0
 jsonschema==4.17.3
 launchdarkly-server-sdk==8.1.1
 marshmallow-sqlalchemy==0.25.0
 marshmallow==3.19.0
-minio==7.1.13
+minio==7.1.14
 multidict==6.0.4
 opentracing==2.4.0
 packaging==23.0
@@ -61,17 +63,18 @@ pycountry==22.3.5
 pycparser==2.21
 pyrsistent==0.19.3
 python-dateutil==2.8.2
-python-dotenv==0.21.1
+python-dotenv==1.0.0
 python-jose==3.3.0
-pytz==2022.7.1
+pytz==2023.2
 requests==2.28.2
 rsa==4.9
 semver==2.13.0
-sentry-sdk==1.15.0
+sentry-sdk==1.18.0
 six==1.16.0
 threadloop==1.0.2
 thrift==0.16.0
 tornado==6.2
-urllib3==1.26.14
+typing_extensions==4.5.0
+urllib3==1.26.15
 yarl==1.8.2
-zipp==3.14.0
+zipp==3.15.0