diff --git a/.github/workflows/events-listener-check.yml b/.github/workflows/events-listener-check.yml new file mode 100644 index 000000000..2a0b04bd5 --- /dev/null +++ b/.github/workflows/events-listener-check.yml @@ -0,0 +1,66 @@ +name: Events Listener Queue PR CI + +on: + pull_request: + branches: + - development + paths: + - "queue_services/events-listener/**" + +jobs: + events-listener-queue_service-check: + runs-on: ubuntu-latest + defaults: + run: + shell: bash + working-directory: ./queue_services/events-listener + + if: github.repository == 'bcgov/sbc-pay' + env: + DATABASE_TEST_URL: "postgresql://postgres:postgres@localhost:5432/postgres" + TEST_NATS_DOCKER: "YES" + STAN_CLUSTER_NAME: "test-cluster" + + strategy: + matrix: + python-version: [3.8] + services: + postgres: + image: postgres:11 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + # needed because the postgres container does not provide a healthcheck + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements/dev.txt + - name: Lint with pylint + run: | + pylint --rcfile=setup.cfg --disable=C0301,W0511 src/events_listener + # - name: Lint with flake8 + # run: | + # flake8 src/events_listener tests + - name: Test with pytest + run: | + pip install . + export PYTHONPATH=./src/ + pytest + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v1 + with: + file: ./queue_services/events-listener/coverage.xml + flags: events_listener + name: codecov-events-listener + fail_ci_if_error: true diff --git a/.github/workflows/events-listener-dev.yml b/.github/workflows/events-listener-dev.yml new file mode 100644 index 000000000..542c2f128 --- /dev/null +++ b/.github/workflows/events-listener-dev.yml @@ -0,0 +1,60 @@ +name: Events Listener Queue DEV CD + +on: + push: + branches: + - development + paths: + - "queue_services/events-listener/**" + +env: + IMAGE_NAME: events-listener + TAG_NAME: dev + +jobs: + events-listener-dev: + runs-on: ubuntu-latest + if: github.repository == 'bcgov/sbc-pay' + steps: + - uses: actions/checkout@v2 + - name: Install tools + run: | + ./scripts/install_tools.sh + oc version + op --version + - name: Login Openshift + working-directory: ./queue_services/events-listener + run: | + oc login ${{ secrets.OPENSHIFT_LOGIN_REGISTRY }} --token=${{ secrets.OPENSHIFT_SA_TOKEN }} + - name: Set Deployment Environment Variables + working-directory: . + run: | + vaults='[{ "vault": "shared", "application": ["nats"]}, {"vault": "relationship","application": ["postgres-pay", "events-listener"]}]' + ./scripts/1pass.sh ${{ secrets.op_parameters }} -m "secret" -e "${TAG_NAME}" -v "${vaults}" -a "${IMAGE_NAME}-${TAG_NAME}" -n ${{ secrets.OPENSHIFT_REPOSITORY_DEV}} + - name: Log into registry + run: echo "${{ secrets.OPENSHIFT_SA_TOKEN }}" | docker login ${{ secrets.OPENSHIFT_DOCKER_REGISTRY }} -u ${{ secrets.OPENSHIFT_SA_NAME}} --password-stdin + - name: Build image + working-directory: ./queue_services/events-listener + run: docker build . --file Dockerfile --tag image + - name: Push image + working-directory: ./queue_services/events-listener + run: | + IMAGE_ID=${{ secrets.OPENSHIFT_DOCKER_REGISTRY }}/${{ secrets.OPENSHIFT_REPOSITORY_TOOLS }}/$IMAGE_NAME + docker tag image $IMAGE_ID:latest + docker push $IMAGE_ID:latest + docker image tag $IMAGE_ID:latest $IMAGE_ID:$TAG_NAME + docker push $IMAGE_ID:$TAG_NAME + - name: Rollout new deployment + working-directory: ./queue_services/events-listener + run: | + oc rollout status dc/$IMAGE_NAME-${TAG_NAME} -n ${{ secrets.OPENSHIFT_REPOSITORY_DEV}} -w + - name: Rocket.Chat Notification + uses: RocketChat/Rocket.Chat.GitHub.Action.Notification@master + if: failure() + with: + type: ${{ job.status }} + job_name: "*Events Listener Build and Deploy to ${{env.TAG_NAME}}*" + channel: "#registries-bot" + url: ${{ secrets.ROCKETCHAT_WEBHOOK }} + commit: true + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/events-listener-test.yml b/.github/workflows/events-listener-test.yml new file mode 100644 index 000000000..49ad8fb4d --- /dev/null +++ b/.github/workflows/events-listener-test.yml @@ -0,0 +1,59 @@ +name: Events Listener Queue TEST CD + +on: + push: + branches: + - master + paths: + - "queue_services/events-listener/**" +env: + IMAGE_NAME: events-listener + TAG_NAME: test + +jobs: + events-listener-test: + runs-on: ubuntu-latest + if: github.repository == 'bcgov/sbc-pay' + steps: + - uses: actions/checkout@v2 + - name: Install tools + run: | + ./scripts/install_tools.sh + oc version + op --version + - name: Login Openshift + working-directory: ./queue_services/events-listener + run: | + oc login ${{ secrets.OPENSHIFT_LOGIN_REGISTRY }} --token=${{ secrets.OPENSHIFT_SA_TOKEN }} + - name: Set Deployment Environment Variables + working-directory: . + run: | + vaults='[{ "vault": "shared", "application": [nats"]}, {"vault": "relationship","application": ["postgres-pay", "events-listener"]}]' + ./scripts/1pass.sh ${{ secrets.op_parameters }} -m "secret" -e "${TAG_NAME}" -v "${vaults}" -a "${IMAGE_NAME}-${TAG_NAME}" -n ${{ secrets.OPENSHIFT_REPOSITORY_TEST }} + - name: Log into registry + run: echo "${{ secrets.OPENSHIFT_SA_TOKEN }}" | docker login ${{ secrets.OPENSHIFT_DOCKER_REGISTRY }} -u ${{ secrets.OPENSHIFT_SA_NAME}} --password-stdin + - name: Build image + working-directory: ./queue_services/events-listener + run: docker build . --file Dockerfile --tag image + - name: Push image + working-directory: ./queue_services/events-listener + run: | + IMAGE_ID=${{ secrets.OPENSHIFT_DOCKER_REGISTRY }}/${{ secrets.OPENSHIFT_REPOSITORY_TOOLS }}/$IMAGE_NAME + docker tag image $IMAGE_ID:latest + docker push $IMAGE_ID:latest + docker image tag $IMAGE_ID:latest $IMAGE_ID:$TAG_NAME + docker push $IMAGE_ID:$TAG_NAME + - name: Rollout new deployment + working-directory: ./queue_services/events-listener + run: | + oc rollout status dc/$IMAGE_NAME-${TAG_NAME} -n ${{ secrets.OPENSHIFT_REPOSITORY_TEST}} -w + - name: Rocket.Chat Notification + uses: RocketChat/Rocket.Chat.GitHub.Action.Notification@master + if: failure() + with: + type: ${{ job.status }} + job_name: "*Events Listener Queue Build and Deploy to ${{env.TAG_NAME}}*" + channel: "#registries-bot" + url: ${{ secrets.ROCKETCHAT_WEBHOOK }} + commit: true + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/codecov.yaml b/codecov.yaml index 85959a655..005d41aa6 100644 --- a/codecov.yaml +++ b/codecov.yaml @@ -17,6 +17,7 @@ coverage: - payapi - bcolapi - reportapi + - eventlistenerqueue ignore: - "^/tests/**/*" # ignore test harness code @@ -47,3 +48,8 @@ flags: paths: - report-api/src/api carryforward: true + + eventlistenerqueue + paths: + - queue_services/events-listener/src/events_listener + carryforward: true \ No newline at end of file diff --git a/pay-api/src/pay_api/__init__.py b/pay-api/src/pay_api/__init__.py index 85f670273..195ac6d67 100755 --- a/pay-api/src/pay_api/__init__.py +++ b/pay-api/src/pay_api/__init__.py @@ -24,8 +24,8 @@ from sbc_common_components.exception_handling.exception_handler import ExceptionHandler # noqa: I001 from sbc_common_components.utils.camel_case_response import convert_to_camel -import config -from config import _Config +import pay_api.config as config +from pay_api.config import _Config from pay_api.models import db, ma from pay_api.utils.auth import jwt from pay_api.utils.cache import cache diff --git a/pay-api/config.py b/pay-api/src/pay_api/config.py similarity index 80% rename from pay-api/config.py rename to pay-api/src/pay_api/config.py index a9e300dc0..94912fbfd 100755 --- a/pay-api/config.py +++ b/pay-api/src/pay_api/config.py @@ -11,10 +11,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""All of the configuration for the service is captured here. All items are loaded, or have Constants defined here that are loaded into the Flask configuration. All modules and lookups get their configuration from the Flask config, rather than reading environment variables directly or by accessing this configuration directly. +"""All of the configuration for the service is captured here. + +All items are loaded, +or have Constants defined here that are loaded into the Flask configuration. +All modules and lookups get their configuration from the Flask config, +rather than reading environment variables directly or by accessing this configuration directly. """ -import json import os import sys @@ -24,16 +28,16 @@ load_dotenv(find_dotenv()) CONFIGURATION = { - 'development': 'config.DevConfig', - 'testing': 'config.TestConfig', - 'production': 'config.ProdConfig', - 'default': 'config.ProdConfig', - 'migration': 'config.MigrationConfig', + 'development': 'pay_api.config.DevConfig', + 'testing': 'pay_api.config.TestConfig', + 'production': 'pay_api.config.ProdConfig', + 'default': 'pay_api.config.ProdConfig', + 'migration': 'pay_api.config.MigrationConfig', } def get_named_config(config_name: str = 'production'): - """Return the configuration object based on the name + """Return the configuration object based on the name. :raise: KeyError: if an unknown configuration is requested """ @@ -60,8 +64,8 @@ def _get_config(config_key: str, **kwargs): return value -class _Config(object): # pylint: disable=too-few-public-methods - """Base class configuration that should set reasonable defaults for all the other configurations. """ +class _Config(): # pylint: disable=too-few-public-methods + """Base class configuration that should set reasonable defaults for all the other configurations.""" PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__)) @@ -153,6 +157,8 @@ class _Config(object): # pylint: disable=too-few-public-methods class DevConfig(_Config): # pylint: disable=too-few-public-methods + """Dev config.""" + TESTING = False DEBUG = True @@ -189,33 +195,37 @@ class TestConfig(_Config): # pylint: disable=too-few-public-methods JWT_OIDC_TEST_JWKS_URI = _get_config('JWT_OIDC_TEST_JWKS_URI', default=None) JWT_OIDC_TEST_KEYS = { - "keys": [ + 'keys': [ { - "kid": "sbc-auth-web", - "kty": "RSA", - "alg": "RS256", - "use": "sig", - "n": "AN-fWcpCyE5KPzHDjigLaSUVZI0uYrcGcc40InVtl-rQRDmAh-C2W8H4_Hxhr5VLc6crsJ2LiJTV_E72S03pzpOOaaYV6-TzAjCou2GYJIXev7f6Hh512PuG5wyxda_TlBSsI-gvphRTPsKCnPutrbiukCYrnPuWxX5_cES9eStR", - "e": "AQAB" + 'kid': 'sbc-auth-web', + 'kty': 'RSA', + 'alg': 'RS256', + 'use': 'sig', + 'n': 'AN-fWcpCyE5KPzHDjigLaSUVZI0uYrcGcc40InVtl-rQRDmAh-C2W8H4_Hxhr5VLc6crsJ2LiJTV_E72S03pzpOOaaYV6-' + 'TzAjCou2GYJIXev7f6Hh512PuG5wyxda_TlBSsI-gvphRTPsKCnPutrbiukCYrnPuWxX5_cES9eStR', + 'e': 'AQAB' } ] } JWT_OIDC_TEST_PRIVATE_KEY_JWKS = { - "keys": [ + 'keys': [ { - "kid": "sbc-auth-web", - "kty": "RSA", - "alg": "RS256", - "use": "sig", - "n": "AN-fWcpCyE5KPzHDjigLaSUVZI0uYrcGcc40InVtl-rQRDmAh-C2W8H4_Hxhr5VLc6crsJ2LiJTV_E72S03pzpOOaaYV6-TzAjCou2GYJIXev7f6Hh512PuG5wyxda_TlBSsI-gvphRTPsKCnPutrbiukCYrnPuWxX5_cES9eStR", - "e": "AQAB", - "d": "C0G3QGI6OQ6tvbCNYGCqq043YI_8MiBl7C5dqbGZmx1ewdJBhMNJPStuckhskURaDwk4-8VBW9SlvcfSJJrnZhgFMjOYSSsBtPGBIMIdM5eSKbenCCjO8Tg0BUh_xa3CHST1W4RQ5rFXadZ9AeNtaGcWj2acmXNO3DVETXAX3x0", - "p": "APXcusFMQNHjh6KVD_hOUIw87lvK13WkDEeeuqAydai9Ig9JKEAAfV94W6Aftka7tGgE7ulg1vo3eJoLWJ1zvKM", - "q": "AOjX3OnPJnk0ZFUQBwhduCweRi37I6DAdLTnhDvcPTrrNWuKPg9uGwHjzFCJgKd8KBaDQ0X1rZTZLTqi3peT43s", - "dp": "AN9kBoA5o6_Rl9zeqdsIdWFmv4DB5lEqlEnC7HlAP-3oo3jWFO9KQqArQL1V8w2D4aCd0uJULiC9pCP7aTHvBhc", - "dq": "ANtbSY6njfpPploQsF9sU26U0s7MsuLljM1E8uml8bVJE1mNsiu9MgpUvg39jEu9BtM2tDD7Y51AAIEmIQex1nM", - "qi": "XLE5O360x-MhsdFXx8Vwz4304-MJg-oGSJXCK_ZWYOB_FGXFRTfebxCsSYi0YwJo-oNu96bvZCuMplzRI1liZw" + 'kid': 'sbc-auth-web', + 'kty': 'RSA', + 'alg': 'RS256', + 'use': 'sig', + 'n': 'AN-fWcpCyE5KPzHDjigLaSUVZI0uYrcGcc40InVtl-rQRDmAh-C2W8H4_Hxhr5VLc6crsJ2LiJTV_E72S03pzpOOaaYV6-' + 'TzAjCou2GYJIXev7f6Hh512PuG5wyxda_TlBSsI-gvphRTPsKCnPutrbiukCYrnPuWxX5_cES9eStR', + 'e': 'AQAB', + 'd': 'C0G3QGI6OQ6tvbCNYGCqq043YI_8MiBl7C5dqbGZmx1ewdJBhMNJPStuckhskURaDwk4-' + '8VBW9SlvcfSJJrnZhgFMjOYSSsBtPGBIMIdM5eSKbenCCjO8Tg0BUh_' + 'xa3CHST1W4RQ5rFXadZ9AeNtaGcWj2acmXNO3DVETXAX3x0', + 'p': 'APXcusFMQNHjh6KVD_hOUIw87lvK13WkDEeeuqAydai9Ig9JKEAAfV94W6Aftka7tGgE7ulg1vo3eJoLWJ1zvKM', + 'q': 'AOjX3OnPJnk0ZFUQBwhduCweRi37I6DAdLTnhDvcPTrrNWuKPg9uGwHjzFCJgKd8KBaDQ0X1rZTZLTqi3peT43s', + 'dp': 'AN9kBoA5o6_Rl9zeqdsIdWFmv4DB5lEqlEnC7HlAP-3oo3jWFO9KQqArQL1V8w2D4aCd0uJULiC9pCP7aTHvBhc', + 'dq': 'ANtbSY6njfpPploQsF9sU26U0s7MsuLljM1E8uml8bVJE1mNsiu9MgpUvg39jEu9BtM2tDD7Y51AAIEmIQex1nM', + 'qi': 'XLE5O360x-MhsdFXx8Vwz4304-MJg-oGSJXCK_ZWYOB_FGXFRTfebxCsSYi0YwJo-oNu96bvZCuMplzRI1liZw' } ] } @@ -244,9 +254,9 @@ class TestConfig(_Config): # pylint: disable=too-few-public-methods SERVER_NAME = 'auth-web.dev.com' - REPORT_API_BASE_URL = "http://localhost:8080/reports-api/api/v1/reports" + REPORT_API_BASE_URL = 'http://localhost:8080/reports-api/api/v1/reports' - AUTH_API_ENDPOINT = "http://localhost:8080/auth-api/" + AUTH_API_ENDPOINT = 'http://localhost:8080/auth-api/' NATS_SUBJECT = 'entity.filing.test' @@ -278,7 +288,8 @@ class ProdConfig(_Config): # pylint: disable=too-few-public-methods class MigrationConfig(): # pylint: disable=too-few-public-methods - """Config for db migration """ + """Config for db migration.""" + TESTING = False DEBUG = True diff --git a/pay-api/src/pay_api/models/credit_payment_account.py b/pay-api/src/pay_api/models/credit_payment_account.py index 6e047b671..73ba398d1 100644 --- a/pay-api/src/pay_api/models/credit_payment_account.py +++ b/pay-api/src/pay_api/models/credit_payment_account.py @@ -47,6 +47,11 @@ def find_by_corp_number_and_corp_type_and_auth_account_id(cls, corp_number: str, return query.join(PaymentAccount).filter(PaymentAccount.auth_account_id == str(auth_account_id)).one_or_none() + @classmethod + def find_by_corp_number(cls, corp_number: str): + """Find all payment accounts by corp number.""" + return cls.query.filter_by(corp_number=corp_number).all() + class CreditPaymentAccountSchema(ma.ModelSchema): # pylint: disable=too-many-ancestors """Main schema used to serialize the Credit Payment System Account.""" diff --git a/pay-api/src/pay_api/models/internal_payment_account.py b/pay-api/src/pay_api/models/internal_payment_account.py index e0f5b0257..306c87655 100644 --- a/pay-api/src/pay_api/models/internal_payment_account.py +++ b/pay-api/src/pay_api/models/internal_payment_account.py @@ -44,6 +44,11 @@ def find_by_corp_number_and_corp_type_and_account_id(cls, corp_number: str, corp return account + @classmethod + def find_by_corp_number(cls, corp_number: str): + """Find all payment accounts by corp number.""" + return cls.query.filter_by(corp_number=corp_number).all() + class InternalPaymentAccountSchema(ma.ModelSchema): # pylint: disable=too-many-ancestors """Main schema used to serialize the Internal Payment System Account.""" diff --git a/pay-api/src/pay_api/models/invoice.py b/pay-api/src/pay_api/models/invoice.py index 78a89aa28..bd35ee9f6 100644 --- a/pay-api/src/pay_api/models/invoice.py +++ b/pay-api/src/pay_api/models/invoice.py @@ -85,6 +85,11 @@ def update_invoices_for_revenue_updates(cls, fee_distribution_id: int): db.session.bulk_save_objects(invoices) cls.commit() + @classmethod + def find_by_business_identifier(cls, business_identifier: str): + """Find all payment accounts by business_identifier.""" + return cls.query.filter_by(business_identifier=business_identifier).all() + class InvoiceSchema(AuditSchema, BaseSchema): # pylint: disable=too-many-ancestors """Main schema used to serialize the invoice.""" diff --git a/pay-api/src/pay_api/utils/util.py b/pay-api/src/pay_api/utils/util.py index 2b5e001c2..9023077f4 100755 --- a/pay-api/src/pay_api/utils/util.py +++ b/pay-api/src/pay_api/utils/util.py @@ -28,7 +28,6 @@ def cors_preflight(methods: str = 'GET'): """Render an option method on the class.""" - def wrapper(f): def options(self, *args, **kwargs): # pylint: disable=unused-argument return {'Allow': methods}, 200, \ diff --git a/pay-api/tests/unit/conf/test_configuration.py b/pay-api/tests/unit/conf/test_configuration.py index 66d103add..fe7515de9 100755 --- a/pay-api/tests/unit/conf/test_configuration.py +++ b/pay-api/tests/unit/conf/test_configuration.py @@ -19,7 +19,7 @@ import pytest -import config +import pay_api.config as config # testdata pattern is ({str: environment}, {expected return value}) diff --git a/queue_services/events-listener/.envrc b/queue_services/events-listener/.envrc new file mode 100644 index 000000000..64fff9a69 --- /dev/null +++ b/queue_services/events-listener/.envrc @@ -0,0 +1,6 @@ +while read -r line; do + echo $line + [[ "$line" =~ ^#.*$ ]] && continue + export $line +done < .env +source venv/bin/activate diff --git a/queue_services/events-listener/Dockerfile b/queue_services/events-listener/Dockerfile new file mode 100644 index 000000000..2779ef8e4 --- /dev/null +++ b/queue_services/events-listener/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.8.5-buster +USER root + +# Create working directory +RUN mkdir /opt/app-root && chmod 755 /opt/app-root +WORKDIR /opt/app-root + +# Install the requirements +COPY ./requirements.txt . + +RUN pip install --upgrade pip +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +RUN pip install . + +USER 1001 + +# Set Python path +ENV PYTHONPATH=/opt/app-root/src + +#EXPOSE 8080 + +CMD [ "python", "/opt/app-root/app.py" ] diff --git a/queue_services/events-listener/LICENSE b/queue_services/events-listener/LICENSE new file mode 100644 index 000000000..18b5abc34 --- /dev/null +++ b/queue_services/events-listener/LICENSE @@ -0,0 +1,13 @@ +Copyright © 2018 Province of British Columbia + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/queue_services/events-listener/MANIFEST.in b/queue_services/events-listener/MANIFEST.in new file mode 100644 index 000000000..1a342bdeb --- /dev/null +++ b/queue_services/events-listener/MANIFEST.in @@ -0,0 +1,5 @@ +include requirements.txt +include config.py +include logging.conf +include LICENSE +include README.md \ No newline at end of file diff --git a/queue_services/events-listener/Makefile b/queue_services/events-listener/Makefile new file mode 100644 index 000000000..00ca679be --- /dev/null +++ b/queue_services/events-listener/Makefile @@ -0,0 +1,155 @@ +.PHONY: license +.PHONY: setup clean clean-build clean-pyc clean-test + +.PHONY: docker-setup network build start qa style safety test test-travis flake8 \ +isort isort-save stop docker-clean logs +.PHONY: mac-cov pylint flake8 + +SHELL:=/bin/bash +mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST))) +current_dir := $(notdir $(patsubst %/,%,$(dir $(mkfile_path)))) +current_abs_dir := $(patsubst %/,%,$(dir $(mkfile_path))) + +################################################################################# +# COMMANDS # +################################################################################# +clean: clean-build clean-pyc clean-test + rm -rf venv/ + +clean-build: + rm -fr build/ + rm -fr dist/ + rm -fr .eggs/ + find . -name '*.egg-info' -exec rm -fr {} + + find . -name '*.egg' -exec rm -f {} + + +clean-pyc: + find . -name '*.pyc' -exec rm -f {} + + find . -name '*.pyo' -exec rm -f {} + + find . -name '*~' -exec rm -f {} + + find . -name '__pycache__' -exec rm -fr {} + + +clean-test: + find . -name '.pytest_cache' -exec rm -fr {} + + rm -fr .tox/ + rm -f .coverage + rm -fr htmlcov/ + +setup: clean venv/bin/activate install-dev + +venv/bin/activate: requirements/prod.txt requirements/dev.txt + rm -rf venv/ + test -f venv/bin/activate || python3 -m venv $(current_abs_dir)/venv + . venv/bin/activate ;\ + pip install --upgrade pip ;\ + pip install -Ur requirements/prod.txt ;\ + pip freeze | sort > requirements.txt ;\ + cat requirements/repo-libraries.txt >> requirements.txt ;\ + pip install -Ur requirements/repo-libraries.txt ;\ + pip install -Ur requirements/dev.txt + touch venv/bin/activate # update so it's as new as requirements/prod.txt + +.PHONY: install-dev +install-dev: venv/bin/activate + . venv/bin/activate ; \ + pip install -e . + +.PHONY: activate +activate: venv/bin/activate + . venv/bin/activate + +.PHONY: local-test +local-test: venv/bin/activate + . venv/bin/activate ; \ + pytest + +.PHONY: local-coverage +local-coverage: venv/bin/activate + . venv/bin/activate ; \ + coverage run -m pytest + +.PHONY: coverage-report +coverage-report: local-coverage + . venv/bin/activate ; \ + coverage report ; \ + coverage html + +## Run the coverage report and display in a browser window +mac-cov: install-dev coverage-report + open -a "Google Chrome" htmlcov/index.html + +## run pylint on the package and tests +pylint: + pylint --rcfile=setup.cfg \ + --load-plugins=pylint_flask \ + --disable=C0301,W0511 \ + src/events_listener + +## run flake8 on the package and tests +flake8: + flake8 src/events_listener tests + +## Verify source code license headers. +license: + ./scripts/verify_license_headers.sh src/events_listener tests + +################################################################################# +# Self Documenting Commands # +################################################################################# + +.DEFAULT_GOAL := show-help + +# Inspired by +# sed script explained: +# /^##/: +# * save line in hold space +# * purge line +# * Loop: +# * append newline + line to hold space +# * go to next line +# * if line starts with doc comment, strip comment character off and loop +# * remove target prerequisites +# * append hold space (+ newline) to line +# * replace newline plus comments by `---` +# * print line +# Separate expressions are necessary because labels cannot be delimited by +# semicolon; see +.PHONY: show-help +show-help: + @echo "$$(tput bold)Available rules:$$(tput sgr0)" + @echo + @sed -n -e "/^## / { \ + h; \ + s/.*//; \ + :doc" \ + -e "H; \ + n; \ + s/^## //; \ + t doc" \ + -e "s/:.*//; \ + G; \ + s/\\n## /---/; \ + s/\\n/ /g; \ + p; \ + }" ${MAKEFILE_LIST} \ + | LC_ALL='C' sort --ignore-case \ + | awk -F '---' \ + -v ncol=$$(tput cols) \ + -v indent=19 \ + -v col_on="$$(tput setaf 6)" \ + -v col_off="$$(tput sgr0)" \ + '{ \ + printf "%s%*s%s ", col_on, -indent, $$1, col_off; \ + n = split($$2, words, " "); \ + line_length = ncol - indent; \ + for (i = 1; i <= n; i++) { \ + line_length -= length(words[i]) + 1; \ + if (line_length <= 0) { \ + line_length = ncol - indent - length(words[i]) - 1; \ + printf "\n%*s ", -indent, " "; \ + } \ + printf "%s ", words[i]; \ + } \ + printf "\n"; \ + }' \ + | more $(shell test $(shell uname) = Darwin && echo '--no-init --raw-control-chars') diff --git a/queue_services/events-listener/README.md b/queue_services/events-listener/README.md new file mode 100755 index 000000000..2379003c9 --- /dev/null +++ b/queue_services/events-listener/README.md @@ -0,0 +1,75 @@ + +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE) + + +# Application Name + +BC Registries Names Examination, research and approval system API + +## Technology Stack Used +* Python, Flask +* Postgres - SQLAlchemy, psycopg2-binary & alembic + +## Third-Party Products/Libraries used and the the License they are covert by + +## Project Status +As of 2018-02-22 in **ALPHA** + +## Documnentation + +GitHub Pages (https://guides.github.com/features/pages/) are a neat way to document you application/project. + +## Security + +Future - BCGov Keycloak + +Current - JWT hack + +## Files in this repository + +``` +docs/ - Project Documentation +└── images +└── icons + +openshift/ - OpenShift-specific files +├── scripts - helper scripts +└── templates - application templates +``` + +## Deployment (Local Development) + +* Developer Workstation Requirements/Setup +* Application Specific Setup + +## Deployment (OpenShift) + +See (openshift/Readme.md) + +## Getting Help or Reporting an Issue + +To report bugs/issues/feature requests, please file an [issue](../../issues). + +## How to Contribute + +If you would like to contribute, please see our [CONTRIBUTING](./CONTRIBUTING.md) guidelines. + +Please note that this project is released with a [Contributor Code of Conduct](./CODE_OF_CONDUCT.md). +By participating in this project you agree to abide by its terms. + +## License + + Copyright 2018 Province of British Columbia + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/queue_services/events-listener/app.py b/queue_services/events-listener/app.py new file mode 100755 index 000000000..9b53a0324 --- /dev/null +++ b/queue_services/events-listener/app.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright © 2019 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""s2i based launch script to run the service.""" +import asyncio +import os + +from events_listener.worker import APP_CONFIG, cb_subscription_handler, qsm + +if __name__ == '__main__': + + # my_config = config.get_named_config(os.getenv('DEPLOYMENT_ENV', 'production')) + + event_loop = asyncio.get_event_loop() + event_loop.run_until_complete(qsm.run(loop=event_loop, + config=APP_CONFIG, + callback=cb_subscription_handler)) + try: + event_loop.run_forever() + finally: + event_loop.close() diff --git a/queue_services/events-listener/coverage.xml b/queue_services/events-listener/coverage.xml new file mode 100644 index 000000000..1e0be32e0 --- /dev/null +++ b/queue_services/events-listener/coverage.xml @@ -0,0 +1,3 @@ + + + diff --git a/queue_services/events-listener/jenkins/dev.groovy b/queue_services/events-listener/jenkins/dev.groovy new file mode 100755 index 000000000..4e2983a30 --- /dev/null +++ b/queue_services/events-listener/jenkins/dev.groovy @@ -0,0 +1,249 @@ +#!/usr/bin/env groovy +// Copyright © 2018 Province of British Columbia +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//JENKINS DEPLOY ENVIRONMENT VARIABLES: +// - JENKINS_JAVA_OVERRIDES -Dhudson.model.DirectoryBrowserSupport.CSP= -Duser.timezone=America/Vancouver +// -> user.timezone : set the local timezone so logfiles report correxct time +// -> hudson.model.DirectoryBrowserSupport.CSP : removes restrictions on CSS file load, thus html pages of test reports are displayed pretty +// See: https://docs.openshift.com/container-platform/3.9/using_images/other_images/jenkins.html for a complete list of JENKINS env vars + +import groovy.json.* + +// define constants - values sent in as env vars from whatever calls this pipeline +def APP_NAME = 'events-listener' +def DESTINATION_TAG = 'dev' +def E2E_TAG = 'e2e' +def TOOLS_TAG = 'tools' +def NAMESPACE_APP = 'l4ygcl' +def NAMESPACE_SHARED = 'd7eovc' +def NAMESPACE_BUILD = "${NAMESPACE_APP}" + '-' + "${TOOLS_TAG}" +def NAMESPACE_DEPLOY = "${NAMESPACE_APP}" + '-' + "${DESTINATION_TAG}" +def NAMESPACE_UNITTEST = "${NAMESPACE_SHARED}" + '-'+ "${TOOLS_TAG}" + +def ROCKETCHAT_DEVELOPER_CHANNEL='#relationship-developers' + +// post a notify to rocketchat +def rocketChatNotificaiton(token, channel, comments) { + def payload = JsonOutput.toJson([text: comments, channel: channel]) + def rocketChatUrl = "https://chat.pathfinder.gov.bc.ca/hooks/" + "${token}" + + sh(returnStdout: true, + script: "curl -X POST -H 'Content-Type: application/json' --data \'${payload}\' ${rocketChatUrl}") +} + +@NonCPS +boolean triggerBuild(String contextDirectory) { + // Determine if code has changed within the source context directory. + def changeLogSets = currentBuild.changeSets + def filesChangeCnt = 0 + for (int i = 0; i < changeLogSets.size(); i++) { + def entries = changeLogSets[i].items + for (int j = 0; j < entries.length; j++) { + def entry = entries[j] + //echo "${entry.commitId} by ${entry.author} on ${new Date(entry.timestamp)}: ${entry.msg}" + def files = new ArrayList(entry.affectedFiles) + for (int k = 0; k < files.size(); k++) { + def file = files[k] + def filePath = file.path + //echo ">> ${file.path}" + if (filePath.contains(contextDirectory)) { + filesChangeCnt = 1 + k = files.size() + j = entries.length + } + } + } + } + + if ( filesChangeCnt < 1 ) { + echo('The changes do not require a build.') + return false + } else { + echo('The changes require a build.') + return true + } +} + +// Get an image's hash tag +String getImageTagHash(String imageName, String tag = "") { + + if(!tag?.trim()) { + tag = "latest" + } + + def istag = openshift.raw("get istag ${imageName}:${tag} -o template --template='{{.image.dockerImageReference}}'") + return istag.out.tokenize('@')[1].trim() +} + +// define job properties - keep 10 builds only +properties([[$class: 'BuildDiscarderProperty', strategy: [$class: 'LogRotator', artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10']]]) + +def run_pipeline = true + +// build wasn't triggered by changes so check with user +if( !triggerBuild(APP_NAME) ) { + stage('No changes. Run pipeline?') { + try { + timeout(time: 1, unit: 'DAYS') { + input message: "Run pipeline?", id: "1234"//, submitter: 'admin' + } + } catch (Exception e) { + run_pipeline = false; + } + } +} + +if( run_pipeline ) { + node { + def build_ok = true + def old_version + + stage("Build ${APP_NAME}") { + script { + openshift.withCluster() { + openshift.withProject("${NAMESPACE_BUILD}") { + try { + echo "Building ${APP_NAME} ..." + def build = openshift.selector("bc", "${APP_NAME}").startBuild() + build.untilEach { + return it.object().status.phase == "Running" + } + build.logs('-f') + } catch (Exception e) { + echo e.getMessage() + build_ok = false + } + } + } + } + } + + if (build_ok) { + stage("Tag ${APP_NAME}:${DESTINATION_TAG}") { + script { + openshift.withCluster() { + openshift.withProject("${NAMESPACE_DEPLOY}") { + old_version = openshift.selector('dc', "${APP_NAME}-${DESTINATION_TAG}").object().status.latestVersion + } + } + openshift.withCluster() { + openshift.withProject("${NAMESPACE_BUILD}") { + try { + echo "Tagging ${APP_NAME} for deployment to ${DESTINATION_TAG} ..." + + // Don't tag with BUILD_ID so the pruner can do it's job; it won't delete tagged images. + // Tag the images for deployment based on the image's hash + def IMAGE_HASH = getImageTagHash("${APP_NAME}") + echo "IMAGE_HASH: ${IMAGE_HASH}" + openshift.tag("${APP_NAME}@${IMAGE_HASH}", "${APP_NAME}:${DESTINATION_TAG}") + } catch (Exception e) { + echo e.getMessage() + build_ok = false + } + } + } + } + } + + stage("Tag ${APP_NAME}:${E2E_TAG}") { + script { + openshift.withCluster() { + openshift.withProject("${NAMESPACE_BUILD}") { + try { + echo "Tagging ${APP_NAME} for deployment to ${E2E_TAG} ..." + + // Don't tag with BUILD_ID so the pruner can do it's job; it won't delete tagged images. + // Tag the images for deployment based on the image's hash + def IMAGE_HASH = getImageTagHash("${APP_NAME}") + echo "IMAGE_HASH: ${IMAGE_HASH}" + openshift.tag("${APP_NAME}@${IMAGE_HASH}", "${APP_NAME}:${E2E_TAG}") + } catch (Exception e) { + echo e.getMessage() + build_ok = false + } + } + } + } + } + } + + if (build_ok) { + stage("Deploy ${APP_NAME}-${DESTINATION_TAG}") { + sleep 10 + script { + openshift.withCluster() { + openshift.withProject("${NAMESPACE_DEPLOY}") { + try { + def new_version = openshift.selector('dc', "${APP_NAME}-${DESTINATION_TAG}").object().status.latestVersion + if (new_version == old_version) { + echo "New deployment was not triggered." + } + + def pod_selector = openshift.selector('pod', [ app:"${APP_NAME}-${DESTINATION_TAG}" ]) + pod_selector.untilEach { + deployment = it.objects()[0].metadata.labels.deployment + echo deployment + if (deployment == "${APP_NAME}-${DESTINATION_TAG}-${new_version}" && it.objects()[0].status.phase == 'Running' && it.objects()[0].status.containerStatuses[0].ready) { + return true + } else { + echo "Pod for new deployment not ready" + sleep 5 + return false + } + } + } catch (Exception e) { + echo e.getMessage() + build_ok = false + } + } + } + } + } + } + + if (build_ok) { + try { + stage("Run tests on ${APP_NAME}:${DESTINATION_TAG}") { + + } + } catch (Exception e) { + echo e.getMessage() + echo "Not all tests passed." + build_ok = false + } + } + + if (build_ok) { + stage("Run E2E API tests") { + + } + } + + stage("Notify on RocketChat") { + if(build_ok) { + currentBuild.result = "SUCCESS" + } else { + currentBuild.result = "FAILURE" + } + + ROCKETCHAT_TOKEN = sh ( + script: """oc get secret/apitest-secrets -n ${NAMESPACE_BUILD} -o template --template="{{.data.ROCKETCHAT_TOKEN}}" | base64 --decode""", + returnStdout: true).trim() + + rocketChatNotificaiton("${ROCKETCHAT_TOKEN}", "${ROCKETCHAT_DEVELOPER_CHANNEL}", "${APP_NAME} build and deploy to ${DESTINATION_TAG} ${currentBuild.result}!") + } + } +} + diff --git a/queue_services/events-listener/jenkins/prod.groovy b/queue_services/events-listener/jenkins/prod.groovy new file mode 100755 index 000000000..a645e3695 --- /dev/null +++ b/queue_services/events-listener/jenkins/prod.groovy @@ -0,0 +1,137 @@ +#!/usr/bin/env groovy +// Copyright © 2018 Province of British Columbia +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//JENKINS DEPLOY ENVIRONMENT VARIABLES: +// - JENKINS_JAVA_OVERRIDES -Dhudson.model.DirectoryBrowserSupport.CSP= -Duser.timezone=America/Vancouver +// -> user.timezone : set the local timezone so logfiles report correxct time +// -> hudson.model.DirectoryBrowserSupport.CSP : removes restrictions on CSS file load, thus html pages of test reports are displayed pretty +// See: https://docs.openshift.com/container-platform/3.9/using_images/other_images/jenkins.html for a complete list of JENKINS env vars + +import groovy.json.* + +// define constants - values sent in as env vars from whatever calls this pipeline +def APP_NAME = 'events-listener' +def SOURCE_TAG = 'test' +def DESTINATION_TAG = 'prod' +def TOOLS_TAG = 'tools' + +def NAMESPACE_APP = 'l4ygcl' +def NAMESPACE_BUILD = "${NAMESPACE_APP}" + '-' + "${TOOLS_TAG}" +def NAMESPACE_DEPLOY = "${NAMESPACE_APP}" + '-' + "${DESTINATION_TAG}" + +def ROCKETCHAT_DEVELOPER_CHANNEL='#relationship-developers' + +// Get an image's hash tag +String getImageTagHash(String imageName, String tag = "") { + + if(!tag?.trim()) { + tag = "latest" + } + + def istag = openshift.raw("get istag ${imageName}:${tag} -o template --template='{{.image.dockerImageReference}}'") + return istag.out.tokenize('@')[1].trim() +} + +// post a notify to rocketchat +def rocketChatNotificaiton(token, channel, comments) { + def payload = JsonOutput.toJson([text: comments, channel: channel]) + def rocketChatUrl = "https://chat.pathfinder.gov.bc.ca/hooks/" + "${token}" + + sh(returnStdout: true, + script: "curl -X POST -H 'Content-Type: application/json' --data \'${payload}\' ${rocketChatUrl}") +} + +node { + properties([[$class: 'BuildDiscarderProperty', strategy: [$class: 'LogRotator', artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10']]]) + + def build_ok = true + def old_version + + try { + stage("Tag ${APP_NAME}:${DESTINATION_TAG}") { + script { + openshift.withCluster() { + openshift.withProject("${NAMESPACE_DEPLOY}") { + old_version = openshift.selector('dc', "${APP_NAME}-${DESTINATION_TAG}").object().status.latestVersion + } + } + openshift.withCluster() { + openshift.withProject("${NAMESPACE_BUILD}") { + echo "Tagging ${APP_NAME}:${DESTINATION_TAG}-prev ..." + def IMAGE_HASH = getImageTagHash("${APP_NAME}", "${DESTINATION_TAG}") + echo "IMAGE_HASH: ${IMAGE_HASH}" + openshift.tag("${APP_NAME}@${IMAGE_HASH}", "${APP_NAME}:${DESTINATION_TAG}-prev") + + echo "Tagging ${APP_NAME} for deployment to ${DESTINATION_TAG} ..." + openshift.tag("${APP_NAME}:${SOURCE_TAG}", "${APP_NAME}:${DESTINATION_TAG}") + } + } + } + } + } catch (Exception e) { + echo e.getMessage() + build_ok = false + } + + + if (build_ok) { + try { + stage("Deploy ${APP_NAME}-${DESTINATION_TAG}") { + sleep 10 + script { + openshift.withCluster() { + openshift.withProject("${NAMESPACE_DEPLOY}") { + + def new_version = openshift.selector('dc', "${APP_NAME}-${DESTINATION_TAG}").object().status.latestVersion + if (new_version == old_version) { + echo "New deployment was not triggered." + } + + def pod_selector = openshift.selector('pod', [ app:"${APP_NAME}-${DESTINATION_TAG}" ]) + pod_selector.untilEach { + deployment = it.objects()[0].metadata.labels.deployment + echo deployment + if (deployment == "${APP_NAME}-${DESTINATION_TAG}-${new_version}" && it.objects()[0].status.phase == 'Running' && it.objects()[0].status.containerStatuses[0].ready) { + return true + } else { + echo "Pod for new deployment not ready" + sleep 5 + return false + } + } + } + } + } + } + } catch (Exception e) { + echo e.getMessage() + build_ok = false + } + } + + stage("Notify on RocketChat") { + if(build_ok) { + currentBuild.result = "SUCCESS" + } else { + currentBuild.result = "FAILURE" + } + + ROCKETCHAT_TOKEN = sh ( + script: """oc get secret/apitest-secrets -n ${NAMESPACE_BUILD} -o template --template="{{.data.ROCKETCHAT_TOKEN}}" | base64 --decode""", + returnStdout: true).trim() + + rocketChatNotificaiton("${ROCKETCHAT_TOKEN}", "${ROCKETCHAT_DEVELOPER_CHANNEL}", "${APP_NAME} build and deploy to ${DESTINATION_TAG} ${currentBuild.result}!") + } +} \ No newline at end of file diff --git a/queue_services/events-listener/jenkins/test.groovy b/queue_services/events-listener/jenkins/test.groovy new file mode 100755 index 000000000..6b0c7ea23 --- /dev/null +++ b/queue_services/events-listener/jenkins/test.groovy @@ -0,0 +1,118 @@ +#!/usr/bin/env groovy +// Copyright © 2018 Province of British Columbia +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//JENKINS DEPLOY ENVIRONMENT VARIABLES: +// - JENKINS_JAVA_OVERRIDES -Dhudson.model.DirectoryBrowserSupport.CSP= -Duser.timezone=America/Vancouver +// -> user.timezone : set the local timezone so logfiles report correxct time +// -> hudson.model.DirectoryBrowserSupport.CSP : removes restrictions on CSS file load, thus html pages of test reports are displayed pretty +// See: https://docs.openshift.com/container-platform/3.9/using_images/other_images/jenkins.html for a complete list of JENKINS env vars + +import groovy.json.* + +// define constants - values sent in as env vars from whatever calls this pipeline +def APP_NAME = 'events-listener' +def DESTINATION_TAG = 'test' +def TOOLS_TAG = 'tools' + +def NAMESPACE_APP = 'l4ygcl' +def NAMESPACE_BUILD = "${NAMESPACE_APP}" + '-' + "${TOOLS_TAG}" +def NAMESPACE_DEPLOY = "${NAMESPACE_APP}" + '-' + "${DESTINATION_TAG}" + +def ROCKETCHAT_DEVELOPER_CHANNEL='#relationship-developers' + +// post a notify to rocketchat +def rocketChatNotificaiton(token, channel, comments) { + def payload = JsonOutput.toJson([text: comments, channel: channel]) + def rocketChatUrl = "https://chat.pathfinder.gov.bc.ca/hooks/" + "${token}" + + sh(returnStdout: true, + script: "curl -X POST -H 'Content-Type: application/json' --data \'${payload}\' ${rocketChatUrl}") +} + +node { + properties([[$class: 'BuildDiscarderProperty', strategy: [$class: 'LogRotator', artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10']]]) + + def build_ok = true + def old_version + + try { + stage("Build ${APP_NAME}-${DESTINATION_TAG}") { + script { + openshift.withCluster() { + openshift.withProject("${NAMESPACE_BUILD}") { + echo "Building ${APP_NAME}-${DESTINATION_TAG} ..." + def build = openshift.selector("bc", "${APP_NAME}-${DESTINATION_TAG}").startBuild() + build.untilEach { + return it.object().status.phase == "Running" + } + build.logs('-f') + } + } + } + } + } catch (Exception e) { + echo e.getMessage() + build_ok = false + } + + if (build_ok) { + try { + stage("Deploy ${APP_NAME}-${DESTINATION_TAG}") { + sleep 10 + script { + openshift.withCluster() { + openshift.withProject("${NAMESPACE_DEPLOY}") { + + def new_version = openshift.selector('dc', "${APP_NAME}-${DESTINATION_TAG}").object().status.latestVersion + if (new_version == old_version) { + echo "New deployment was not triggered." + } + + def pod_selector = openshift.selector('pod', [ app:"${APP_NAME}-${DESTINATION_TAG}" ]) + pod_selector.untilEach { + deployment = it.objects()[0].metadata.labels.deployment + echo deployment + if (deployment == "${APP_NAME}-${DESTINATION_TAG}-${new_version}" && it.objects()[0].status.phase == 'Running' && it.objects()[0].status.containerStatuses[0].ready) { + return true + } else { + echo "Pod for new deployment not ready" + sleep 5 + return false + } + } + } + } + } + } + } catch (Exception e) { + echo e.getMessage() + build_ok = false + } + } + + stage("Notify on RocketChat") { + if(build_ok) { + currentBuild.result = "SUCCESS" + } else { + currentBuild.result = "FAILURE" + } + + ROCKETCHAT_TOKEN = sh ( + script: """oc get secret/apitest-secrets -n ${NAMESPACE_BUILD} -o template --template="{{.data.ROCKETCHAT_TOKEN}}" | base64 --decode""", + returnStdout: true).trim() + + rocketChatNotificaiton("${ROCKETCHAT_TOKEN}", "${ROCKETCHAT_DEVELOPER_CHANNEL}", "${APP_NAME} build and deploy to ${DESTINATION_TAG} ${currentBuild.result}!") + } +} diff --git a/queue_services/events-listener/logging.conf b/queue_services/events-listener/logging.conf new file mode 100644 index 000000000..ffc1a01e3 --- /dev/null +++ b/queue_services/events-listener/logging.conf @@ -0,0 +1,28 @@ +[loggers] +keys=root,api + +[handlers] +keys=console + +[formatters] +keys=simple + +[logger_root] +level=DEBUG +handlers=console + +[logger_api] +level=DEBUG +handlers=console +qualname=api +propagate=0 + +[handler_console] +class=StreamHandler +level=DEBUG +formatter=simple +args=(sys.stdout,) + +[formatter_simple] +format=%(asctime)s - %(name)s - %(levelname)s in %(module)s:%(filename)s:%(lineno)d - %(funcName)s: %(message)s +datefmt= \ No newline at end of file diff --git a/queue_services/events-listener/openshift/templates/events-listener-build.json b/queue_services/events-listener/openshift/templates/events-listener-build.json new file mode 100755 index 000000000..ce0bb88e3 --- /dev/null +++ b/queue_services/events-listener/openshift/templates/events-listener-build.json @@ -0,0 +1,111 @@ +{ + "kind": "Template", + "apiVersion": "v1", + "metadata": { + "annotations": { + "description": "Build template for a events listener.", + "tags": "flask", + "iconClass": "icon-python" + }, + "name": "${NAME}-build" + }, + "objects": [ + { + "kind": "ImageStream", + "apiVersion": "v1", + "metadata": { + "name": "${NAME}" + } + }, + { + "kind": "BuildConfig", + "apiVersion": "v1", + "metadata": { + "name": "${NAME}", + "labels": { + "app": "${NAME}", + "app-group": "${APP_GROUP}", + "template": "${NAME}-build" + } + }, + "spec": { + "source": { + "type": "Git", + "git": { + "uri": "${GIT_REPO_URL}", + "ref": "${GIT_REF}" + }, + "contextDir": "${SOURCE_CONTEXT_DIR}" + }, + "strategy": { + "type": "Docker", + "dockerStrategy": { + "dockerfilePath": "${DOCKER_FILE_PATH}" + } + }, + "output": { + "to": { + "kind": "ImageStreamTag", + "name": "${NAME}:${OUTPUT_IMAGE_TAG}" + } + }, + "triggers": [ + { + "type": "ConfigChange" + } + ] + } + } + ], + "parameters": [ + { + "name": "NAME", + "displayName": "Name", + "description": "The name assigned to all of the objects defined in this template. You should keep this as default unless your know what your doing.", + "required": true, + "value": "events-listener" + }, + { + "name": "APP_GROUP", + "displayName": "App Group", + "description": "The name assigned to all of the deployments in this project.", + "required": true, + "value": "sbc-pay" + }, + { + "name": "GIT_REPO_URL", + "displayName": "Git Repo URL", + "description": "The URL to your GIT repo, don't use the this default unless your just experimenting.", + "required": true, + "value": "https://github.com/bcgov/sbc-pay.git" + }, + { + "name": "GIT_REF", + "displayName": "Git Reference", + "description": "The git reference or branch.", + "required": true, + "value": "development" + }, + { + "name": "SOURCE_CONTEXT_DIR", + "displayName": "Source Context Directory", + "description": "The source context directory.", + "required": true, + "value": "queue_services/events-listener" + }, + { + "name": "OUTPUT_IMAGE_TAG", + "displayName": "Output Image Tag", + "description": "The tag given to the built image.", + "required": true, + "value": "latest" + }, + { + "name": "DOCKER_FILE_PATH", + "displayName": "Docker File Path", + "description": "The path to the docker file defining the build.", + "required": false, + "value": "Dockerfile" + } + ] +} \ No newline at end of file diff --git a/queue_services/events-listener/openshift/templates/events-listener-deploy.json b/queue_services/events-listener/openshift/templates/events-listener-deploy.json new file mode 100755 index 000000000..b30d289f5 --- /dev/null +++ b/queue_services/events-listener/openshift/templates/events-listener-deploy.json @@ -0,0 +1,200 @@ +{ + "kind": "Template", + "apiVersion": "v1", + "metadata": { + "annotations": { + "description": "Deployment template for events listener service.", + "tags": "${NAME}-${TAG_NAME}" + }, + "name": "${NAME}-${TAG_NAME}-deploy" + }, + "objects": [ + { + "kind": "DeploymentConfig", + "apiVersion": "v1", + "metadata": { + "name": "${NAME}-${TAG_NAME}", + "labels": { + "app": "${NAME}-${TAG_NAME}", + "app-group": "${APP_GROUP}", + "template": "${NAME}-deploy" + } + }, + "spec": { + "strategy": { + "type": "Rolling", + "rollingParams": { + "updatePeriodSeconds": 1, + "intervalSeconds": 1, + "timeoutSeconds": 600, + "maxUnavailable": "25%", + "maxSurge": "25%" + } + }, + "triggers": [ + { + "type": "ImageChange", + "imageChangeParams": { + "automatic": true, + "containerNames": [ + "${NAME}-${TAG_NAME}" + ], + "from": { + "kind": "ImageStreamTag", + "namespace": "${IMAGE_NAMESPACE}", + "name": "${NAME}:${TAG_NAME}" + } + } + }, + { + "type": "ConfigChange" + } + ], + "replicas": "${REPLICAS}", + "test": false, + "selector": { + "app": "${NAME}-${TAG_NAME}", + "deploymentconfig": "${NAME}-${TAG_NAME}" + }, + "template": { + "metadata": { + "labels": { + "app": "${NAME}-${TAG_NAME}", + "app-group": "${APP_GROUP}", + "deploymentconfig": "${NAME}-${TAG_NAME}", + "template": "${NAME}-deploy" + } + }, + "spec": { + "containers": [ + { + "name": "${NAME}-${TAG_NAME}", + "image": "docker-registry.default.svc:5000/${IMAGE_NAMESPACE}/${NAME}:${TAG_NAME}", + "ports": [ + { + "containerPort": 8080, + "protocol": "TCP" + } + ], + "env": [ + ], + "resources": { + "requests": { + "cpu": "${CPU_REQUEST}", + "memory": "${MEMORY_REQUEST}" + }, + "limits": { + "cpu": "${CPU_LIMIT}", + "memory": "${MEMORY_LIMIT}" + } + }, + "livenessProbe": { + "httpGet": { + "path": "/healthz", + "port": 7070, + "scheme": "HTTP" + }, + "timeoutSeconds": 1, + "periodSeconds": 10, + "successThreshold": 1, + "failureThreshold": 3 + }, + "readinessProbe": { + "httpGet": { + "path": "/readyz", + "port": 7070, + "scheme": "HTTP" + }, + "timeoutSeconds": 1, + "periodSeconds": 10, + "successThreshold": 1, + "failureThreshold": 3 + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "imagePullPolicy": "Always" + } + ], + "restartPolicy": "Always", + "terminationGracePeriodSeconds": 30, + "dnsPolicy": "ClusterFirst", + "securityContext": {}, + "schedulerName": "default-scheduler" + } + } + } + } + ], + "parameters": [ + { + "name": "NAME", + "displayName": "Name", + "description": "The name assigned to all of the OpenShift resources associated to the server instance.", + "required": true, + "value": "events-listener" + }, + { + "name": "APP_GROUP", + "displayName": "App Group", + "description": "The name assigned to all of the deployments in this project.", + "required": true, + "value": "sbc-pay" + }, + { + "name": "IMAGE_NAMESPACE", + "displayName": "Image Namespace", + "required": true, + "description": "The namespace of the OpenShift project containing the imagestream for the application.", + "value": "l4ygcl-tools" + }, + { + "name": "TAG_NAME", + "displayName": "Environment TAG name", + "description": "The TAG name for this environment, e.g., dev, test, prod", + "required": true, + "value": "dev" + }, + { + "name": "DATABASE_NAME", + "displayName": "Database App Name", + "description": "A valid database app name used by the service.", + "required": true, + "value": "postgresql" + }, + { + "name": "CPU_REQUEST", + "displayName": "Resources CPU Request", + "description": "The resources CPU request (in cores) for this build.", + "required": true, + "value": "100m" + }, + { + "name": "CPU_LIMIT", + "displayName": "Resources CPU Limit", + "description": "The resources CPU limit (in cores) for this build.", + "required": true, + "value": "750m" + }, + { + "name": "MEMORY_REQUEST", + "displayName": "Resources Memory Request", + "description": "The resources Memory request (in Mi, Gi, etc) for this build.", + "required": true, + "value": "100Mi" + }, + { + "name": "MEMORY_LIMIT", + "displayName": "Resources Memory Limit", + "description": "The resources Memory limit (in Mi, Gi, etc) for this build.", + "required": true, + "value": "2Gi" + }, + { + "name": "REPLICAS", + "displayName": "The number of replicas to run", + "description": "The number of replicas to run in this environment.", + "required": true, + "value": "1" + } + ] +} diff --git a/queue_services/events-listener/openshift/templates/events-listener-pipeline.json b/queue_services/events-listener/openshift/templates/events-listener-pipeline.json new file mode 100755 index 000000000..2cc54cf7b --- /dev/null +++ b/queue_services/events-listener/openshift/templates/events-listener-pipeline.json @@ -0,0 +1,113 @@ +{ + "kind": "Template", + "apiVersion": "v1", + "metadata": { + "name": "${NAME}-pipeline" + }, + "objects": [ + { + "kind": "BuildConfig", + "apiVersion": "v1", + "metadata": { + "name": "${NAME}-${TAG_NAME}-pipeline", + "creationTimestamp": null, + "labels": { + "app": "${NAME}-${TAG_NAME}", + "app-group": "${APP_GROUP}", + "template": "${NAME}-pipeline" + } + }, + "spec": { + "triggers": [ + { + "type": "GitHub", + "github": { + "secretReference": { + "name": "${WEBHOOK}" + } + } + } + ], + "runPolicy": "Serial", + "source": { + "type": "Git", + "git": { + "uri": "${GIT_REPO_URL}", + "ref": "${GIT_REF}" + }, + "contextDir": "${SOURCE_CONTEXT_DIR}" + }, + "strategy": { + "type": "JenkinsPipeline", + "jenkinsPipelineStrategy": { + "jenkinsfilePath": "${JENKINS_FILE}" + } + }, + "output": {}, + "resources": {}, + "postCommit": {}, + "nodeSelector": {}, + "successfulBuildsHistoryLimit": 5, + "failedBuildsHistoryLimit": 5 + } + } + ], + "parameters": [ + { + "name": "NAME", + "displayName": "Name", + "description": "The name assigned to all of the resources defined in this template.", + "required": true, + "value": "events-listener" + }, + { + "name": "APP_GROUP", + "displayName": "App Group", + "description": "The name assigned to all of the deployments in this project.", + "required": true, + "value": "sbc-pay" + }, + { + "name": "TAG_NAME", + "displayName": "Pipeline purpose", + "description": "The activity that this pipeline will manage. eg. build, test, promote, etc.", + "required": true, + "value": "dev" + }, + { + "name": "GIT_REPO_URL", + "displayName": "Git Repo URL", + "description": "The URL to your GIT repo.", + "required": true, + "value": "https://github.com/bcgov/sbc-pay.git" + }, + { + "name": "GIT_REF", + "displayName": "Git Reference", + "description": "The git reference or branch.", + "required": true, + "value": "development" + }, + { + "name": "WEBHOOK", + "displayName": "Secret name for the webhook.", + "description": "The name of the Secret that holds the webhook.", + "required": true, + "value": "github-events-listener-dev" + }, + { + "name": "SOURCE_CONTEXT_DIR", + "displayName": "Source Context Directory", + "description": "The source context directory.", + "required": false, + "value": "queue_services/events-listener" + }, + { + "name": "JENKINS_FILE", + "displayName": "The Jenksinfile this pipeline should use.", + "description": "The Jenkinsfile this pipeline should use.", + "required": false, + "value": "./jenkins/dev.groovy" + } + ] +} \ No newline at end of file diff --git a/queue_services/events-listener/q_cli.py b/queue_services/events-listener/q_cli.py new file mode 100755 index 000000000..6b6ac40d0 --- /dev/null +++ b/queue_services/events-listener/q_cli.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright © 2019 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Service for listening and handling Queue Messages. + +This service registers interest in listening to a Queue and processing received messages. +""" +import asyncio +import functools +import getopt +import json +import os +import random +import signal +import sys + +from nats.aio.client import Client as NATS # noqa N814; by convention the name is NATS +from stan.aio.client import Client as STAN # noqa N814; by convention the name is STAN + +from entity_queue_common.service_utils import error_cb, logger, signal_handler + + +async def run(loop, old_identifier, new_identifier): # pylint: disable=too-many-locals + """Run the main application loop for the service. + + This runs the main top level service functions for working with the Queue. + """ + # NATS client connections + nc = NATS() + sc = STAN() + + async def close(): + """Close the stream and nats connections.""" + await sc.close() + await nc.close() + + # Connection and Queue configuration. + def nats_connection_options(): + return { + 'servers': os.getenv('NATS_SERVERS', 'nats://127.0.0.1:4222').split(','), + 'io_loop': loop, + 'error_cb': error_cb, + 'name': os.getenv('NATS_CLIENT_NAME', 'entity.filing.tester') + } + + def stan_connection_options(): + return { + 'cluster_id': os.getenv('NATS_CLUSTER_ID', 'test-cluster'), + 'client_id': str(random.SystemRandom().getrandbits(0x58)), + 'nats': nc + } + + def subscription_options(): + return { + 'subject': os.getenv('NATS_SUBJECT', 'entity.filings'), + 'queue': os.getenv('NATS_QUEUE', 'filing-worker'), + 'durable_name': os.getenv('NATS_QUEUE', 'filing-worker') + '_durable' + } + + try: + # Connect to the NATS server, and then use that for the streaming connection. + await nc.connect(**nats_connection_options()) + await sc.connect(**stan_connection_options()) + + # register the signal handler + for sig in ('SIGINT', 'SIGTERM'): + loop.add_signal_handler(getattr(signal, sig), + functools.partial(signal_handler, sig_loop=loop, sig_nc=nc, task=close) + ) + + payload = {'oldBusinessIdentifier': old_identifier, 'newBusinessIdentifier': new_identifier} + await sc.publish(subject=subscription_options().get('subject'), + payload=json.dumps(payload).encode('utf-8')) + + except Exception as e: # pylint: disable=broad-except + # TODO tighten this error and decide when to bail on the infinite reconnect + logger.error(e) + + +if __name__ == '__main__': + try: + opts, args = getopt.getopt(sys.argv[1:], "ho:n:", ["oldid=", "newid="]) + except getopt.GetoptError: + print('q_cli.py -o -n ') + sys.exit(2) + + for opt, arg in opts: + if opt == '-h': + print('q_cli.py -o -n ') + sys.exit() + elif opt in ("-o", "--oldid"): + old_id = arg + elif opt in ("-n", "--newid"): + new_id = arg + + print('publish:', old_id, new_id) + event_loop = asyncio.get_event_loop() + event_loop.run_until_complete(run(event_loop, old_id, new_id)) diff --git a/queue_services/events-listener/requirements.txt b/queue_services/events-listener/requirements.txt new file mode 100644 index 000000000..a592fb5e4 --- /dev/null +++ b/queue_services/events-listener/requirements.txt @@ -0,0 +1,24 @@ +Flask==1.1.2 +Jinja2==2.11.2 +MarkupSafe==1.1.1 +Werkzeug==0.16.1 +asyncio-nats-client==0.10.0 +asyncio-nats-streaming==0.4.0 +attrs==20.2.0 +blinker==1.4 +certifi==2020.6.20 +click==7.1.2 +itsdangerous==1.1.0 +jsonschema==3.2.0 +protobuf==3.13.0 +pycountry==20.7.3 +pyrsistent==0.16.0 +python-dotenv==0.14.0 +sentry-sdk==0.17.3 +six==1.15.0 +urllib3==1.25.10 +-e git+https://github.com/bcgov/lear.git#egg=entity_queue_common&subdirectory=queue_services/common +-e git+https://github.com/pwei1018/jaeger-client-python.git@186f14e14758273ed108508c0d388a4f4de5c75b#egg=jaeger-client +-e git+https://github.com/bcgov/sbc-common-components.git#egg=sbc-common-components-1.0.0&subdirectory=python +# -e git+https://github.com/bcgov/sbc-pay.git#egg=pay-api&subdirectory=pay-api +-e git+https://github.com/pksumesh/sbc-pay.git@event_listener#egg=pay-api&subdirectory=pay-api \ No newline at end of file diff --git a/queue_services/events-listener/requirements/dev.txt b/queue_services/events-listener/requirements/dev.txt new file mode 100755 index 000000000..02e01335d --- /dev/null +++ b/queue_services/events-listener/requirements/dev.txt @@ -0,0 +1,31 @@ +# Everything the developer needs in addition to the production requirements +-r prod.txt + +# Testing +pytest +pytest-mock +requests +pyhamcrest +pytest-cov +FreezeGun + +# Lint and code style +flake8 +flake8-blind-except +flake8-debugger +flake8-docstrings +flake8-isort +flake8-quotes +pep8-naming +autopep8 +coverage +pylint<2.5 +pylint-flask +pydocstyle +isort<5 + + +# docker +lovely-pytest-docker +pytest-asyncio + diff --git a/queue_services/events-listener/requirements/prod.txt b/queue_services/events-listener/requirements/prod.txt new file mode 100755 index 000000000..68c31b9b2 --- /dev/null +++ b/queue_services/events-listener/requirements/prod.txt @@ -0,0 +1,8 @@ +Flask +jsonschema +python-dotenv +sentry-sdk[flask] +asyncio-nats-client +asyncio-nats-streaming +pycountry +Werkzeug<1 diff --git a/queue_services/events-listener/requirements/repo-libraries.txt b/queue_services/events-listener/requirements/repo-libraries.txt new file mode 100644 index 000000000..0dc96142d --- /dev/null +++ b/queue_services/events-listener/requirements/repo-libraries.txt @@ -0,0 +1,5 @@ +-e git+https://github.com/bcgov/lear.git#egg=entity_queue_common&subdirectory=queue_services/common +-e git+https://github.com/pwei1018/jaeger-client-python.git@186f14e14758273ed108508c0d388a4f4de5c75b#egg=jaeger-client +-e git+https://github.com/bcgov/sbc-common-components.git#egg=sbc-common-components-1.0.0&subdirectory=python +# -e git+https://github.com/bcgov/sbc-pay.git#egg=pay-api&subdirectory=pay-api +-e git+https://github.com/pksumesh/sbc-pay.git@event_listener#egg=pay-api&subdirectory=pay-api \ No newline at end of file diff --git a/queue_services/events-listener/scripts/verify_license_headers.sh b/queue_services/events-listener/scripts/verify_license_headers.sh new file mode 100755 index 000000000..028b95c63 --- /dev/null +++ b/queue_services/events-listener/scripts/verify_license_headers.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +# Copyright © 2019 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +COPYRIGHT="Copyright © 2019 Province of British Columbia" +RET=0 + +for file in $(find $@ -not \( -path */venv -prune \) -not \( -path */migrations -prune \) -not \( -path */tests -prune \) -not \( -path */.egg* -prune \) -name \*.py) +do + grep "${COPYRIGHT}" ${file} >/dev/null + if [[ $? != 0 ]] + then + echo "${file} missing copyright header" + RET=1 + fi +done +exit ${RET} diff --git a/queue_services/events-listener/setup.cfg b/queue_services/events-listener/setup.cfg new file mode 100644 index 000000000..966220229 --- /dev/null +++ b/queue_services/events-listener/setup.cfg @@ -0,0 +1,120 @@ +[metadata] +name = events_listener +url = https://github.com/bcgov/sbc-pay/queue_services/events-listener +author = SBC Relationships team +author_email = +classifiers = + Development Status :: Beta + Intended Audience :: Developers / QA + Topic :: Payments + License :: OSI Approved :: Apache Software License + Natural Language :: English + Programming Language :: Python :: 3.8 +license = Apache Software License Version 2.0 +description = A short description of the project +long_description = file: README.md +keywords = + +[options] +zip_safe = True +python_requires = >=3.6 +include_package_data = True +packages = find: + +[options.package_data] +events_listener = + +[wheel] +universal = 1 + +[bdist_wheel] +universal = 1 + +[aliases] +test = pytest + +[flake8] +exclude = .git,*migrations* +max-line-length = 120 +docstring-min-length=10 +per-file-ignores = + */__init__.py:F401 + +[pycodestyle] +max_line_length = 120 +ignore = E501 +docstring-min-length=10 +notes=FIXME,XXX # TODO is ignored +match_dir = src/events_listener +ignored-modules=flask_sqlalchemy + sqlalchemy +per-file-ignores = + */__init__.py:F401 +good-names= + b, + d, + i, + e, + f, + k, + u, + v, + ar, + cb, #common shorthand for callback + nc, + rv, + sc, + event_loop, + logger, + loop, + +[pylint] +ignore=migrations,test +max_line_length=120 +notes=FIXME,XXX,TODO +ignored-modules=flask_sqlalchemy,sqlalchemy,SQLAlchemy,alembic,scoped_session +ignored-classes=scoped_session +disable=C0301,W0511,R0801,R0902 + +[isort] +line_length = 120 +indent = 4 +multi_line_output = 4 +lines_after_imports = 2 + +[tool:pytest] +minversion = 2.0 +testpaths = tests +addopts = --verbose + --strict + -p no:warnings +python_files = tests/*/test*.py +norecursedirs = .git .tox venv* requirements* build +log_cli = true +log_cli_level = 1 +filterwarnings = + ignore::UserWarning +markers = + slow + serial + +[coverage:run] +branch = True +source = + src/events_listener +omit = + src/events_listener/wsgi.py + src/events_listener/gunicorn_config.py + +[report:run] +exclude_lines = + pragma: no cover + from + import + def __repr__ + if self.debug: + if settings.DEBUG + raise AssertionError + raise NotImplementedError + if 0: + if __name__ == .__main__.: diff --git a/queue_services/events-listener/setup.py b/queue_services/events-listener/setup.py new file mode 100644 index 000000000..9aa441a30 --- /dev/null +++ b/queue_services/events-listener/setup.py @@ -0,0 +1,69 @@ +# Copyright © 2019 Province of British Columbia. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Installer and setup for this module +""" +import ast +from glob import glob +from os.path import basename, splitext +import re + +from setuptools import setup, find_packages + +_version_re = re.compile(r'__version__\s+=\s+(.*)') # pylint: disable=invalid-name + +with open('src/events_listener/version.py', 'rb') as f: + version = str(ast.literal_eval(_version_re.search( # pylint: disable=invalid-name + f.read().decode('utf-8')).group(1))) + + +def read_requirements(filename): + """ + Get application requirements from + the requirements.txt file. + :return: Python requirements + """ + with open(filename, 'r') as req: + requirements = req.readlines() + install_requires = [r.strip() for r in requirements if (r.find('git+') != 0 and r.find('-e git+') != 0)] + return install_requires + + +def read(filepath): + """ + Read the contents from a file. + :param str filepath: path to the file to be read + :return: file contents + """ + with open(filepath, 'r') as file_handle: + content = file_handle.read() + return content + + +REQUIREMENTS = read_requirements('requirements.txt') + +setup( + name="events_listener", + version=version, + author_email='', + packages=find_packages('src'), + package_dir={'': 'src'}, + py_modules=[splitext(basename(path))[0] for path in glob('src/*.py')], + include_package_data=True, + license=read('LICENSE'), + long_description=read('README.md'), + zip_safe=False, + install_requires=REQUIREMENTS, + setup_requires=["pytest-runner", ], + tests_require=["pytest", ], +) diff --git a/queue_services/events-listener/src/events_listener/__init__.py b/queue_services/events-listener/src/events_listener/__init__.py new file mode 100644 index 000000000..7fcd624dc --- /dev/null +++ b/queue_services/events-listener/src/events_listener/__init__.py @@ -0,0 +1,17 @@ +# Copyright © 2019 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The Events Listener service. + +This module is the service worker for applying filings to the Business Database structure. +""" diff --git a/queue_services/events-listener/src/events_listener/config.py b/queue_services/events-listener/src/events_listener/config.py new file mode 100644 index 000000000..15a38288f --- /dev/null +++ b/queue_services/events-listener/src/events_listener/config.py @@ -0,0 +1,135 @@ +# Copyright © 2019 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""All of the configuration for the service is captured here. + +All items are loaded, or have Constants defined here that +are loaded into the Flask configuration. +All modules and lookups get their configuration from the +Flask config, rather than reading environment variables directly +or by accessing this configuration directly. +""" +import os +import random + +from dotenv import find_dotenv, load_dotenv + + +# this will load all the envars from a .env file located in the project root (api) +load_dotenv(find_dotenv()) + +CONFIGURATION = { + 'development': 'events_listener.config.DevConfig', + 'testing': 'events_listener.config.TestConfig', + 'production': 'events_listener.config.ProdConfig', + 'default': 'events_listener.config.ProdConfig' +} + + +def get_named_config(config_name: str = 'production'): + """Return the configuration object based on the name. + + :raise: KeyError: if an unknown configuration is requested + """ + if config_name in ['production', 'staging', 'default']: + app_config = ProdConfig() + elif config_name == 'testing': + app_config = TestConfig() + elif config_name == 'development': + app_config = DevConfig() + else: + raise KeyError(f'Unknown configuration: {config_name}') + return app_config + + +class _Config(): # pylint: disable=too-few-public-methods + """Base class configuration that should set reasonable defaults. + + Used as the base for all the other configurations. + """ + + PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__)) + + SENTRY_DSN = os.getenv('SENTRY_DSN', None) + + SQLALCHEMY_TRACK_MODIFICATIONS = False + + # POSTGRESQL + DB_USER = os.getenv('DATABASE_USERNAME', '') + DB_PASSWORD = os.getenv('DATABASE_PASSWORD', '') + DB_NAME = os.getenv('DATABASE_NAME', '') + DB_HOST = os.getenv('DATABASE_HOST', '') + DB_PORT = os.getenv('DATABASE_PORT', '5432') + SQLALCHEMY_DATABASE_URI = 'postgresql://{user}:{password}@{host}:{port}/{name}'.format( + user=DB_USER, + password=DB_PASSWORD, + host=DB_HOST, + port=int(DB_PORT), + name=DB_NAME, + ) + + NATS_CONNECTION_OPTIONS = { + 'servers': os.getenv('NATS_SERVERS', 'nats://127.0.0.1:4222').split(','), + 'name': os.getenv('NATS_CLIENT_NAME', 'entity.events.worker') + + } + STAN_CONNECTION_OPTIONS = { + 'cluster_id': os.getenv('NATS_CLUSTER_ID', 'test-cluster'), + 'client_id': str(random.SystemRandom().getrandbits(0x58)), + 'ping_interval': 1, + 'ping_max_out': 5, + } + + SUBSCRIPTION_OPTIONS = { + 'subject': os.getenv('NATS_SUBJECT', 'entity.events'), + 'queue': os.getenv('NATS_QUEUE', 'filing-worker'), + 'durable_name': os.getenv('NATS_QUEUE', 'filing-worker') + '_durable', + } + + +class DevConfig(_Config): # pylint: disable=too-few-public-methods + """Creates the Development Config object.""" + + TESTING = False + DEBUG = True + + +class TestConfig(_Config): # pylint: disable=too-few-public-methods + """In support of testing only. + + Used by the py.test suite + """ + + DEBUG = True + TESTING = True + # POSTGRESQL + DB_USER = os.getenv('DATABASE_TEST_USERNAME', '') + DB_PASSWORD = os.getenv('DATABASE_TEST_PASSWORD', '') + DB_NAME = os.getenv('DATABASE_TEST_NAME', '') + DB_HOST = os.getenv('DATABASE_TEST_HOST', '') + DB_PORT = os.getenv('DATABASE_TEST_PORT', '5432') + SQLALCHEMY_DATABASE_URI = os.getenv( + 'DATABASE_TEST_URL', + default='postgresql://{user}:{password}@{host}:{port}/{name}'.format( + user=DB_USER, password=DB_PASSWORD, host=DB_HOST, port=int(DB_PORT), name=DB_NAME + ), + ) + + STAN_CLUSTER_NAME = 'test-cluster' + + +class ProdConfig(_Config): # pylint: disable=too-few-public-methods + """Production environment configuration.""" + + TESTING = False + DEBUG = False diff --git a/queue_services/events-listener/src/events_listener/utils.py b/queue_services/events-listener/src/events_listener/utils.py new file mode 100644 index 000000000..a5c34c3fb --- /dev/null +++ b/queue_services/events-listener/src/events_listener/utils.py @@ -0,0 +1,32 @@ +# Copyright © 2019 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Supply version and commit hash info. + +When deployed in OKD, it adds the last commit hash onto the version info. +""" +import os + +from events_listener.version import __version__ + + +def _get_build_openshift_commit_hash(): + return os.getenv('OPENSHIFT_BUILD_COMMIT', None) + + +def get_run_version(): + """Return a formatted version string for this service.""" + commit_hash = _get_build_openshift_commit_hash() + if commit_hash: + return f'{__version__}-{commit_hash}' + return __version__ diff --git a/queue_services/events-listener/src/events_listener/version.py b/queue_services/events-listener/src/events_listener/version.py new file mode 100644 index 000000000..405b38b1c --- /dev/null +++ b/queue_services/events-listener/src/events_listener/version.py @@ -0,0 +1,25 @@ +# Copyright © 2019 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Version of this service in PEP440. + +[N!]N(.N)*[{a|b|rc}N][.postN][.devN] +Epoch segment: N! +Release segment: N(.N)* +Pre-release segment: {a|b|rc}N +Post-release segment: .postN +Development release segment: .devN +""" + +__version__ = '2.15.1' # pylint: disable=invalid-name diff --git a/queue_services/events-listener/src/events_listener/worker.py b/queue_services/events-listener/src/events_listener/worker.py new file mode 100644 index 000000000..fbb7b09e1 --- /dev/null +++ b/queue_services/events-listener/src/events_listener/worker.py @@ -0,0 +1,87 @@ +# Copyright © 2019 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The unique worker functionality for this service is contained here. + +The entry-point is the **cb_subscription_handler** + +The design and flow leverage a few constraints that are placed upon it +by NATS Streaming and using AWAIT on the default loop. +- NATS streaming queues require one message to be processed at a time. +- AWAIT on the default loop effectively runs synchronously + +If these constraints change, the use of Flask-SQLAlchemy would need to change. +Flask-SQLAlchemy currently allows the base model to be changed, or reworking +the model to a standalone SQLAlchemy usage with an async engine would need +to be pursued. +""" +import json +import os + +import nats +from entity_queue_common.service import QueueServiceManager +from entity_queue_common.service_utils import QueueException, logger +from flask import Flask # pylint: disable=wrong-import-order +from pay_api.models import CreditPaymentAccount, InternalPaymentAccount, Invoice, db + +from events_listener import config + + +qsm = QueueServiceManager() # pylint: disable=invalid-name +APP_CONFIG = config.get_named_config(os.getenv('DEPLOYMENT_ENV', 'production')) +FLASK_APP = Flask(__name__) +FLASK_APP.config.from_object(APP_CONFIG) +db.init_app(FLASK_APP) + + +async def process_event(event_message, flask_app): + """Render the payment status.""" + if not flask_app: + raise QueueException('Flask App not available.') + + with flask_app.app_context(): + old_identifier = event_message.get('oldBusinessIdentifier') + new_identifier = event_message.get('newBusinessIdentifier') + logger.debug('Received message to update %s to %s', old_identifier, new_identifier) + + # Find all credit card payment accounts which have the old corp number + accounts = CreditPaymentAccount.find_by_corp_number(old_identifier) + for account in accounts: + account.corp_number = new_identifier + account.flush() + + # Find all internal payment accounts which have the old corp number + accounts = InternalPaymentAccount.find_by_corp_number(old_identifier) + for account in accounts: + account.corp_number = new_identifier + account.flush() + + # Find all invoice records which have the old corp number + invoices = Invoice.find_by_business_identifier(old_identifier) + for inv in invoices: + inv.business_identifier = new_identifier + inv.flush() + + db.session.commit() + + +async def cb_subscription_handler(msg: nats.aio.client.Msg): + """Use Callback to process Queue Msg objects.""" + try: + logger.info('Received raw message seq:%s, data= %s', msg.sequence, msg.data.decode()) + event_message = json.loads(msg.data.decode('utf-8')) + logger.debug('Event Message Received: %s', event_message) + await process_event(event_message, FLASK_APP) + except Exception: # pylint: disable=broad-except + # Catch Exception so that any error is still caught and the message is removed from the queue + logger.error('Queue Error: %s', json.dumps(event_message), exc_info=True) diff --git a/queue_services/events-listener/tests/__init__.py b/queue_services/events-listener/tests/__init__.py new file mode 100644 index 000000000..3e44a42f5 --- /dev/null +++ b/queue_services/events-listener/tests/__init__.py @@ -0,0 +1,32 @@ +# Copyright © 2019 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The Test Suites to ensure that the service is built and operating correctly.""" +import datetime + + +EPOCH_DATETIME = datetime.datetime.utcfromtimestamp(0) +FROZEN_DATETIME = datetime.datetime(2001, 8, 5, 7, 7, 58, 272362) + + +def add_years(d, years): + """Return a date that's `years` years after the date (or datetime). + + Return the same calendar date (month and day) in the destination year, + if it exists, otherwise use the following day + (thus changing February 29 to February 28). + """ + try: + return d.replace(year=d.year + years) + except ValueError: + return d + (datetime.date(d.year + years, 3, 1) - datetime.date(d.year, 3, 1)) diff --git a/queue_services/events-listener/tests/conftest.py b/queue_services/events-listener/tests/conftest.py new file mode 100644 index 000000000..f70892042 --- /dev/null +++ b/queue_services/events-listener/tests/conftest.py @@ -0,0 +1,244 @@ +# Copyright © 2019 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Common setup and fixtures for the pytest suite used by this service.""" +import asyncio +import os +import random +import time +from contextlib import contextmanager + +import pytest +from flask import Flask +from flask_migrate import Migrate, upgrade +from nats.aio.client import Client as Nats +from pay_api import db as _db +from sqlalchemy import event, text +from sqlalchemy.schema import DropConstraint, MetaData +from stan.aio.client import Client as Stan + +from events_listener.config import get_named_config + + +@contextmanager +def not_raises(exception): + """Corallary to the pytest raises builtin. + + Assures that an exception is NOT thrown. + """ + try: + yield + except exception: + raise pytest.fail(f'DID RAISE {exception}') + + +@pytest.fixture(scope='session') +def app(): + """Return a session-wide application configured in TEST mode.""" + # _app = create_app('testing') + _app = Flask(__name__) + _app.config.from_object(get_named_config('testing')) + _db.init_app(_app) + + return _app + + +@pytest.fixture(scope='session') +def db(app): # pylint: disable=redefined-outer-name, invalid-name + """Return a session-wide initialised database. + + Drops all existing tables - Meta follows Postgres FKs + """ + with app.app_context(): + # Clear out any existing tables + metadata = MetaData(_db.engine) + metadata.reflect() + for table in metadata.tables.values(): + for fk in table.foreign_keys: # pylint: disable=invalid-name + _db.engine.execute(DropConstraint(fk.constraint)) + metadata.drop_all() + _db.drop_all() + + sequence_sql = """SELECT sequence_name FROM information_schema.sequences + WHERE sequence_schema='public' + """ + + sess = _db.session() + for seq in [name for (name,) in sess.execute(text(sequence_sql))]: + try: + sess.execute(text('DROP SEQUENCE public.%s ;' % seq)) + print('DROP SEQUENCE public.%s ' % seq) + except Exception as err: # pylint: disable=broad-except + print(f'Error: {err}') + sess.commit() + + # ############################################ + # There are 2 approaches, an empty database, or the same one that the app will use + # create the tables + # _db.create_all() + # or + # Use Alembic to load all of the DB revisions including supporting lookup data + # This is the path we'll use in legal_api!! + + # even though this isn't referenced directly, it sets up the internal configs that upgrade + import sys + pay_api_folder = [folder for folder in sys.path if 'pay-api' in folder][0] + migration_path = pay_api_folder.replace('/pay-api/src', '/pay-api/migrations') + + Migrate(app, _db, directory=migration_path) + upgrade() + + return _db + + +@pytest.fixture +def config(app): + """Return the application config.""" + return app.config + + +@pytest.fixture(scope='session') +def client(app): # pylint: disable=redefined-outer-name + """Return a session-wide Flask test client.""" + return app.test_client() + + +@pytest.fixture(scope='session') +def client_ctx(app): # pylint: disable=redefined-outer-name + """Return session-wide Flask test client.""" + with app.test_client() as _client: + yield _client + + +@pytest.fixture('function') +def client_id(): + """Return a unique client_id that can be used in tests.""" + _id = random.SystemRandom().getrandbits(0x58) + # _id = (base64.urlsafe_b64encode(uuid.uuid4().bytes)).replace('=', '') + + return f'client-{_id}' + + +@pytest.fixture(scope='function') +def session(app, db): # pylint: disable=redefined-outer-name, invalid-name + """Return a function-scoped session.""" + with app.app_context(): + conn = db.engine.connect() + txn = conn.begin() + + options = dict(bind=conn, binds={}) + sess = db.create_scoped_session(options=options) + + # establish a SAVEPOINT just before beginning the test + # (http://docs.sqlalchemy.org/en/latest/orm/session_transaction.html#using-savepoint) + sess.begin_nested() + + @event.listens_for(sess(), 'after_transaction_end') + def restart_savepoint(sess2, trans): # pylint: disable=unused-variable + # Detecting whether this is indeed the nested transaction of the test + if trans.nested and not trans._parent.nested: # pylint: disable=protected-access + # Handle where test DOESN'T session.commit(), + sess2.expire_all() + sess.begin_nested() + + db.session = sess + + sql = text('select 1') + sess.execute(sql) + + yield sess + + # Cleanup + sess.remove() + # This instruction rollsback any commit that were executed in the tests. + txn.rollback() + conn.close() + + +@pytest.fixture(scope='session') +def stan_server(docker_services): + """Create the nats / stan services that the integration tests will use.""" + if os.getenv('TEST_NATS_DOCKER'): + docker_services.start('nats') + time.sleep(2) + # TODO get the wait part working, as opposed to sleeping for 2s + # public_port = docker_services.wait_for_service("nats", 4222) + # dsn = "{docker_services.docker_ip}:{public_port}".format(**locals()) + # return dsn + + +@pytest.fixture(scope='function') +@pytest.mark.asyncio +async def stan(event_loop, client_id): + """Create a stan connection for each function, to be used in the tests.""" + nc = Nats() + sc = Stan() + cluster_name = 'test-cluster' + + await nc.connect(io_loop=event_loop, name='entity.filing.tester') + + await sc.connect(cluster_name, client_id, nats=nc) + + yield sc + + await sc.close() + await nc.close() + + +@pytest.fixture(scope='function') +@pytest.mark.asyncio +async def events_stan(app, event_loop, client_id): + """Create a stan connection for each function. + + Uses environment variables for the cluster name. + """ + nc = Nats() + sc = Stan() + + await nc.connect(io_loop=event_loop) + + cluster_name = os.getenv('STAN_CLUSTER_NAME') + + if not cluster_name: + raise ValueError('Missing env variable: STAN_CLUSTER_NAME') + + await sc.connect(cluster_name, client_id, nats=nc) + + yield sc + + await sc.close() + await nc.close() + + +@pytest.fixture(scope='function') +def future(event_loop): + """Return a future that is used for managing function tests.""" + _future = asyncio.Future(loop=event_loop) + return _future + + +@pytest.fixture +def create_mock_coro(mocker, monkeypatch): + """Return a mocked coroutine, and optionally patch-it in.""" + + def _create_mock_patch_coro(to_patch=None): + mock = mocker.Mock() + + async def _coro(*args, **kwargs): + return mock(*args, **kwargs) + + if to_patch: # <-- may not need/want to patch anything + monkeypatch.setattr(to_patch, _coro) + return mock, _coro + + return _create_mock_patch_coro diff --git a/queue_services/events-listener/tests/docker-compose.yml b/queue_services/events-listener/tests/docker-compose.yml new file mode 100644 index 000000000..db7bd6eaa --- /dev/null +++ b/queue_services/events-listener/tests/docker-compose.yml @@ -0,0 +1,15 @@ +version: '2.1' +services: + nats: + image: nats-streaming + restart: always + mem_limit: 512m + expose: + - 4222 + - 8222 + labels: + - entity.services=nats + ports: + - 4222:4222 + - 8222:8222 + tty: true \ No newline at end of file diff --git a/queue_services/events-listener/tests/integration/__init__.py b/queue_services/events-listener/tests/integration/__init__.py new file mode 100644 index 000000000..95f29abce --- /dev/null +++ b/queue_services/events-listener/tests/integration/__init__.py @@ -0,0 +1,116 @@ +# Copyright © 2019 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test suite for the integrations to NATS Queue.""" + +from datetime import datetime + +from pay_api.models import ( + BcolPaymentAccount, CreditPaymentAccount, InternalPaymentAccount, Invoice, Payment, PaymentAccount) +from pay_api.utils.enums import InvoiceStatus, PaymentMethod, PaymentStatus, PaymentSystem + + +def factory_payment_account(corp_number: str = 'CP0001234', corp_type_code: str = 'CP', + payment_system_code: str = 'PAYBC', payment_method_code: str = 'CC', account_number='4101', + bcol_user_id='test', + auth_account_id: str = '1234'): + """Return Factory.""" + # Create a payment account + account = PaymentAccount(auth_account_id=auth_account_id).save() + + if payment_system_code == PaymentSystem.BCOL.value: + return BcolPaymentAccount( + bcol_user_id=bcol_user_id, + bcol_account_id='TEST', + account_id=account.id, + + ) + elif payment_system_code == PaymentSystem.PAYBC.value: + if payment_method_code == PaymentMethod.CC.value: + return CreditPaymentAccount( + corp_number=corp_number, + corp_type_code=corp_type_code, + paybc_party='11111', + paybc_account=account_number, + paybc_site='29921', + account_id=account.id + ) + elif payment_method_code == PaymentMethod.DIRECT_PAY.value: + return CreditPaymentAccount( + corp_number=corp_number, + corp_type_code=corp_type_code, + account_id=account.id + ) + elif payment_system_code == PaymentSystem.INTERNAL.value: + return InternalPaymentAccount( + corp_number=corp_number, + corp_type_code=corp_type_code, + account_id=account.id + ) + + +def factory_premium_payment_account(bcol_user_id='PB25020', bcol_account_id='1234567890', auth_account_id='1234'): + """Return Factory.""" + account = PaymentAccount(auth_account_id=auth_account_id).save() + + return BcolPaymentAccount( + bcol_user_id=bcol_user_id, + bcol_account_id=bcol_account_id, + account_id=account.id + ) + + +def factory_payment( + payment_system_code: str = 'PAYBC', payment_method_code: str = 'CC', + payment_status_code: str = PaymentStatus.CREATED.value, + created_on: datetime = datetime.now() +): + """Return Factory.""" + return Payment( + payment_system_code=payment_system_code, + payment_method_code=payment_method_code, + payment_status_code=payment_status_code, + created_by='test', + created_on=created_on, + ) + + +def factory_invoice(payment: Payment, payment_account, status_code: str = InvoiceStatus.CREATED.value, + corp_type_code='CP', + business_identifier: str = 'CP0001234', + service_fees: float = 0.0, total=0): + """Return Factory.""" + bcol_account_id = None + credit_account_id = None + internal_account_id = None + if isinstance(payment_account, BcolPaymentAccount): + bcol_account_id = payment_account.id + elif isinstance(payment_account, InternalPaymentAccount): + internal_account_id = payment_account.id + if isinstance(payment_account, CreditPaymentAccount): + credit_account_id = payment_account.id + + return Invoice( + payment_id=payment.id, + invoice_status_code=status_code, + bcol_account_id=bcol_account_id, + credit_account_id=credit_account_id, + internal_account_id=internal_account_id, + total=total, + created_by='test', + created_on=datetime.now(), + business_identifier=business_identifier, + corp_type_code=corp_type_code, + folio_number='1234567890', + service_fees=service_fees + ) diff --git a/queue_services/events-listener/tests/integration/test_worker_queue.py b/queue_services/events-listener/tests/integration/test_worker_queue.py new file mode 100644 index 000000000..8f47942a0 --- /dev/null +++ b/queue_services/events-listener/tests/integration/test_worker_queue.py @@ -0,0 +1,143 @@ +# Copyright © 2019 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test Suite to ensure the worker routines are working as expected.""" + +import pytest +from entity_queue_common.service_utils import subscribe_to_queue +from pay_api.models import CreditPaymentAccount, InternalPaymentAccount, Invoice, Payment +from pay_api.utils.enums import PaymentMethod, PaymentSystem + +from tests.integration import factory_invoice, factory_payment, factory_payment_account + +from .utils import helper_add_event_to_queue + + +@pytest.mark.asyncio +async def test_events_listener_queue(app, session, stan_server, event_loop, client_id, events_stan, future): + """Assert that events can be retrieved and decoded from the Queue.""" + # Call back for the subscription + from events_listener.worker import cb_subscription_handler + + # vars + old_identifier = 'T000000000' + new_identifier = 'BC12345678' + + events_subject = 'test_subject' + events_queue = 'test_queue' + events_durable_name = 'test_durable' + + # Create a Credit Card Payment + + # register the handler to test it + await subscribe_to_queue(events_stan, + events_subject, + events_queue, + events_durable_name, + cb_subscription_handler) + + # add an event to queue + await helper_add_event_to_queue(events_stan, events_subject, old_identifier=old_identifier, + new_identifier=new_identifier) + + assert True + + +@pytest.mark.asyncio +async def test_update_internal_payment(app, session, stan_server, event_loop, client_id, events_stan, future): + """Assert that the update internal payment records works.""" + # Call back for the subscription + from events_listener.worker import cb_subscription_handler + + # vars + old_identifier = 'T000000000' + new_identifier = 'BC12345678' + + events_subject = 'test_subject' + events_queue = 'test_queue' + events_durable_name = 'test_durable' + + # Create an Internal Payment + internal_account: InternalPaymentAccount = factory_payment_account( + corp_number=old_identifier, payment_system_code=PaymentSystem.INTERNAL.value).save() + + account_id = internal_account.id + + payment: Payment = factory_payment(payment_method_code=PaymentMethod.INTERNAL.value).save() + invoice: Invoice = factory_invoice(payment=payment, payment_account=internal_account, + business_identifier=old_identifier).save() + + invoice_id = invoice.id + + # register the handler to test it + await subscribe_to_queue(events_stan, + events_subject, + events_queue, + events_durable_name, + cb_subscription_handler) + + # add an event to queue + await helper_add_event_to_queue(events_stan, events_subject, old_identifier=old_identifier, + new_identifier=new_identifier) + + # Get the internal account and invoice and assert that the identifier is new identifier + updated_internal_account = InternalPaymentAccount.find_by_id(account_id) + invoice = Invoice.find_by_id(invoice_id) + + assert updated_internal_account.corp_number == new_identifier + assert invoice.business_identifier == new_identifier + + +@pytest.mark.asyncio +async def test_update_credit_payment(app, session, stan_server, event_loop, client_id, events_stan, future): + """Assert that the update credit payment records works.""" + # Call back for the subscription + from events_listener.worker import cb_subscription_handler + + # vars + old_identifier = 'T000000000' + new_identifier = 'BC12345678' + + events_subject = 'test_subject' + events_queue = 'test_queue' + events_durable_name = 'test_durable' + + # Create an Internal Payment + credit_account: CreditPaymentAccount = factory_payment_account( + corp_number=old_identifier, payment_system_code=PaymentSystem.PAYBC.value).save() + + account_id = credit_account.id + + payment: Payment = factory_payment(payment_method_code=PaymentMethod.CC.value).save() + invoice: Invoice = factory_invoice(payment=payment, payment_account=credit_account, + business_identifier=old_identifier).save() + + invoice_id = invoice.id + + # register the handler to test it + await subscribe_to_queue(events_stan, + events_subject, + events_queue, + events_durable_name, + cb_subscription_handler) + + # add an event to queue + await helper_add_event_to_queue(events_stan, events_subject, old_identifier=old_identifier, + new_identifier=new_identifier) + + # Get the internal account and invoice and assert that the identifier is new identifier + updated_internal_account = CreditPaymentAccount.find_by_id(account_id) + invoice = Invoice.find_by_id(invoice_id) + + assert updated_internal_account.corp_number == new_identifier + assert invoice.business_identifier == new_identifier diff --git a/queue_services/events-listener/tests/integration/utils.py b/queue_services/events-listener/tests/integration/utils.py new file mode 100644 index 000000000..aa19abab5 --- /dev/null +++ b/queue_services/events-listener/tests/integration/utils.py @@ -0,0 +1,27 @@ +# Copyright © 2019 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Utilities used by the integration tests.""" +import json + +import stan + + +async def helper_add_event_to_queue(stan_client: stan.aio.client.Client, + subject: str, + old_identifier: str = 'T1234567890', + new_identifier: str = 'BC1234567890'): + """Add event to the Queue.""" + payload = {'oldBusinessIdentifier': old_identifier, 'newBusinessIdentifier': new_identifier} + await stan_client.publish(subject=subject, + payload=json.dumps(payload).encode('utf-8'))