From 78d398b9e8dc61ad94360ba6951d80936daacbcb Mon Sep 17 00:00:00 2001 From: linuxbandit Date: Thu, 13 Apr 2023 22:38:15 +0200 Subject: [PATCH 01/32] feat: add new notification dispatcher mechanism --- dispatcher/README.md | 43 +++++ dispatcher/dispatcher/__init__.py | 0 dispatcher/dispatcher/main.py | 51 ++++++ dispatcher/docker-compose.yaml | 25 +++ dispatcher/helpers/send.py | 38 ++++ dispatcher/poetry.lock | 143 +++++++++++++++ dispatcher/pyproject.toml | 19 ++ dispatcher/templates/member_expired.jinja2 | 73 ++++++++ dispatcher/templates/snippets/css.jinja2 | 202 +++++++++++++++++++++ dispatcher/templates/snippets/head.jinja2 | 11 ++ dispatcher/tests/__init__.py | 0 11 files changed, 605 insertions(+) create mode 100644 dispatcher/README.md create mode 100644 dispatcher/dispatcher/__init__.py create mode 100755 dispatcher/dispatcher/main.py create mode 100644 dispatcher/docker-compose.yaml create mode 100644 dispatcher/helpers/send.py create mode 100644 dispatcher/poetry.lock create mode 100644 dispatcher/pyproject.toml create mode 100644 dispatcher/templates/member_expired.jinja2 create mode 100644 dispatcher/templates/snippets/css.jinja2 create mode 100644 dispatcher/templates/snippets/head.jinja2 create mode 100644 dispatcher/tests/__init__.py diff --git a/dispatcher/README.md b/dispatcher/README.md new file mode 100644 index 00000000..a8b4b958 --- /dev/null +++ b/dispatcher/README.md @@ -0,0 +1,43 @@ +# Dispatcher + +Polls rabbitMQ and takes action (sends mail). + +## How to run it +### Pre-requisites +In the Vagrant's appserver VM, the packages are installed globally: +1. faker +1. jinja2 +1. pika + +Otherwise use poetry to install dependencies and launch the virtual environment. + +### Launching it and testing it +In a console, run `python3 dispatcher/main.py`. +In another console, run `python3 helpers/send.py`. +Control on `appserver.test:8025` the emails sent. It is also possible to control rabbit's stats on `appserver.test:8080` + +This method is not dockerised. Currently unsure about adding the docker way. + +### Rationale +We do not need a web service for this, only a worker. Doing it this way only means it cannot be scaled (unless precautions are taken for the ack of the message, but pika should already give this out of the box). +In order to add templates, one can work on the filesystem: as the template file is read from memory at the time a message is received, there is basically a mechanism of hot-reload ready to be used. + +We do not need a web service, because we do not need to pilot anything. + #FIXME validate this. Run app and add a new template and push to the queue a new template + +## Queues + +Current queues: +1. email + +Queues envisioned: +1. email +1. telegram +1. slack (If EBs have enabled it) +1. webgui (handled by vue, NOT by this program) + +## TODOs and next steps +rather in order: +1. (not on this project): run core with the email as 'inserting in the queue' instead of 'API request to mailer' +1. include traefik configuration to have the mailhog and rabbit on a subdomain instead of `domain:port` +1. add the telegram queue diff --git a/dispatcher/dispatcher/__init__.py b/dispatcher/dispatcher/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dispatcher/dispatcher/main.py b/dispatcher/dispatcher/main.py new file mode 100755 index 00000000..079a51c8 --- /dev/null +++ b/dispatcher/dispatcher/main.py @@ -0,0 +1,51 @@ +import pika +import json +import smtplib +from email.message import EmailMessage +from jinja2 import Environment, FileSystemLoader + +""" +continuously polls the email queue and renders+sends the template on every acked message +""" + +environment = Environment(loader=FileSystemLoader("../templates/")) + +EMAIL_HOST='172.18.0.13' #FIXME +smtpObj = smtplib.SMTP( EMAIL_HOST, 1025 ) + +RABBIT_HOST='172.18.0.11' #FIXME +connection = pika.BlockingConnection(pika.ConnectionParameters(RABBIT_HOST)) +channel = connection.channel() + +channel.queue_declare(queue='email') + +def send_email(ch, method, properties, body): + msg = json.loads(body) + email = EmailMessage() + email['From'] = msg['from'] + email['Reply-To'] = msg['reply_to'] + email['To'] = msg['to'] + email['Subject'] = msg['subject'] + template = environment.get_template(f"{msg['template']}.jinja2") + rendered = template.render(msg['parameters'], altro=msg['subject']) + email.set_content(rendered, subtype='html') + smtpObj.send_message(email) + +channel.basic_consume(queue='email', + auto_ack=True, + on_message_callback=send_email) + +print(' [*] Waiting for messages. To exit press CTRL+C') +channel.start_consuming() + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + print('Interrupted') + smtpObj.quit() + try: + sys.exit(0) + except SystemExit: + os._exit(0) + diff --git a/dispatcher/docker-compose.yaml b/dispatcher/docker-compose.yaml new file mode 100644 index 00000000..3fce07de --- /dev/null +++ b/dispatcher/docker-compose.yaml @@ -0,0 +1,25 @@ +version: "3.4" + +services: + + rabbit: + image: rabbitmq:3.11-management + restart: always + expose: + - 5672 + ports: + - 8080:15672 + + mailhog: + image: mailhog/mailhog:v1.0.1 + restart: always + ports: + - 8025:8025 + - 1025:1025 + + + +networks: + default: + external: + name: OMS diff --git a/dispatcher/helpers/send.py b/dispatcher/helpers/send.py new file mode 100644 index 00000000..bf69a503 --- /dev/null +++ b/dispatcher/helpers/send.py @@ -0,0 +1,38 @@ +import pika +import json +import random + +""" +creates between 1 and 8 fake emails and puts in the queue +""" + +from faker import Faker +faker = Faker() + +RABBIT_HOST='172.18.0.11' +connection = pika.BlockingConnection(pika.ConnectionParameters(RABBIT_HOST)) +channel = connection.channel() + + +def generate_fake_payload(): + email = { + "from": "mailer@aegee.eu", + "to": [faker.email() for _ in range(random.randrange(1,3))], + "reply_to": "noreply@aegee.eu", + "subject": faker.sentence(), + "template": "member_expired", + "parameters": {"first_name": faker.name(), "body": f"AEGEE-{faker.city()}", "last_payment": faker.date()} + } + return email + +for _ in range(random.randrange(1,8)): + email = generate_fake_payload() + channel.queue_declare(queue='email') + channel.basic_publish(exchange='', + routing_key='email', + body=json.dumps(email)) + print(f" [x] Sent {email['subject']} (to {email['to']})") + + + +connection.close() diff --git a/dispatcher/poetry.lock b/dispatcher/poetry.lock new file mode 100644 index 00000000..255b0182 --- /dev/null +++ b/dispatcher/poetry.lock @@ -0,0 +1,143 @@ +# This file is automatically @generated by Poetry 1.4.0 and should not be changed by hand. + +[[package]] +name = "faker" +version = "18.4.0" +description = "Faker is a Python package that generates fake data for you." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "Faker-18.4.0-py3-none-any.whl", hash = "sha256:170ead9d0d140916168b142df69c44722b8f622ced2070802d0af9c476f0cb84"}, + {file = "Faker-18.4.0.tar.gz", hash = "sha256:977ad0b7aa7a61ed57287d6a0723a827e9d3dd1f8cc82aaf08707f281b33bacc"}, +] + +[package.dependencies] +python-dateutil = ">=2.4" + +[[package]] +name = "jinja2" +version = "3.1.2" +description = "A very fast and expressive template engine." +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, + {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "markupsafe" +version = "2.1.2" +description = "Safely add untrusted strings to HTML/XML markup." +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:340bea174e9761308703ae988e982005aedf427de816d1afe98147668cc03036"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22152d00bf4a9c7c83960521fc558f55a1adbc0631fbb00a9471e097b19d72e1"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28057e985dace2f478e042eaa15606c7efccb700797660629da387eb289b9323"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca244fa73f50a800cf8c3ebf7fd93149ec37f5cb9596aa8873ae2c1d23498601"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d9d971ec1e79906046aa3ca266de79eac42f1dbf3612a05dc9368125952bd1a1"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7e007132af78ea9df29495dbf7b5824cb71648d7133cf7848a2a5dd00d36f9ff"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7313ce6a199651c4ed9d7e4cfb4aa56fe923b1adf9af3b420ee14e6d9a73df65"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-win32.whl", hash = "sha256:c4a549890a45f57f1ebf99c067a4ad0cb423a05544accaf2b065246827ed9603"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:835fb5e38fd89328e9c81067fd642b3593c33e1e17e2fdbf77f5676abb14a156"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2ec4f2d48ae59bbb9d1f9d7efb9236ab81429a764dedca114f5fdabbc3788013"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:608e7073dfa9e38a85d38474c082d4281f4ce276ac0010224eaba11e929dd53a"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65608c35bfb8a76763f37036547f7adfd09270fbdbf96608be2bead319728fcd"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2bfb563d0211ce16b63c7cb9395d2c682a23187f54c3d79bfec33e6705473c6"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:da25303d91526aac3672ee6d49a2f3db2d9502a4a60b55519feb1a4c7714e07d"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9cad97ab29dfc3f0249b483412c85c8ef4766d96cdf9dcf5a1e3caa3f3661cf1"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:085fd3201e7b12809f9e6e9bc1e5c96a368c8523fad5afb02afe3c051ae4afcc"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1bea30e9bf331f3fef67e0a3877b2288593c98a21ccb2cf29b74c581a4eb3af0"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-win32.whl", hash = "sha256:7df70907e00c970c60b9ef2938d894a9381f38e6b9db73c5be35e59d92e06625"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:e55e40ff0cc8cc5c07996915ad367fa47da6b3fc091fdadca7f5403239c5fec3"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a6e40afa7f45939ca356f348c8e23048e02cb109ced1eb8420961b2f40fb373a"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf877ab4ed6e302ec1d04952ca358b381a882fbd9d1b07cccbfd61783561f98a"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63ba06c9941e46fa389d389644e2d8225e0e3e5ebcc4ff1ea8506dce646f8c8a"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f1cd098434e83e656abf198f103a8207a8187c0fc110306691a2e94a78d0abb2"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:55f44b440d491028addb3b88f72207d71eeebfb7b5dbf0643f7c023ae1fba619"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a6f2fcca746e8d5910e18782f976489939d54a91f9411c32051b4aab2bd7c513"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0b462104ba25f1ac006fdab8b6a01ebbfbce9ed37fd37fd4acd70c67c973e460"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-win32.whl", hash = "sha256:7668b52e102d0ed87cb082380a7e2e1e78737ddecdde129acadb0eccc5423859"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6d6607f98fcf17e534162f0709aaad3ab7a96032723d8ac8750ffe17ae5a0666"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a806db027852538d2ad7555b203300173dd1b77ba116de92da9afbc3a3be3eed"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a4abaec6ca3ad8660690236d11bfe28dfd707778e2442b45addd2f086d6ef094"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f03a532d7dee1bed20bc4884194a16160a2de9ffc6354b3878ec9682bb623c54"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4cf06cdc1dda95223e9d2d3c58d3b178aa5dacb35ee7e3bbac10e4e1faacb419"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:22731d79ed2eb25059ae3df1dfc9cb1546691cc41f4e3130fe6bfbc3ecbbecfa"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f8ffb705ffcf5ddd0e80b65ddf7bed7ee4f5a441ea7d3419e861a12eaf41af58"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8db032bf0ce9022a8e41a22598eefc802314e81b879ae093f36ce9ddf39ab1ba"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2298c859cfc5463f1b64bd55cb3e602528db6fa0f3cfd568d3605c50678f8f03"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-win32.whl", hash = "sha256:50c42830a633fa0cf9e7d27664637532791bfc31c731a87b202d2d8ac40c3ea2"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:bb06feb762bade6bf3c8b844462274db0c76acc95c52abe8dbed28ae3d44a147"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:99625a92da8229df6d44335e6fcc558a5037dd0a760e11d84be2260e6f37002f"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8bca7e26c1dd751236cfb0c6c72d4ad61d986e9a41bbf76cb445f69488b2a2bd"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40627dcf047dadb22cd25ea7ecfe9cbf3bbbad0482ee5920b582f3809c97654f"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40dfd3fefbef579ee058f139733ac336312663c6706d1163b82b3003fb1925c4"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:090376d812fb6ac5f171e5938e82e7f2d7adc2b629101cec0db8b267815c85e2"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2e7821bffe00aa6bd07a23913b7f4e01328c3d5cc0b40b36c0bd81d362faeb65"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c0a33bc9f02c2b17c3ea382f91b4db0e6cde90b63b296422a939886a7a80de1c"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b8526c6d437855442cdd3d87eede9c425c4445ea011ca38d937db299382e6fa3"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-win32.whl", hash = "sha256:137678c63c977754abe9086a3ec011e8fd985ab90631145dfb9294ad09c102a7"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:0576fe974b40a400449768941d5d0858cc624e3249dfd1e0c33674e5c7ca7aed"}, + {file = "MarkupSafe-2.1.2.tar.gz", hash = "sha256:abcabc8c2b26036d62d4c746381a6f7cf60aafcc653198ad678306986b09450d"}, +] + +[[package]] +name = "pika" +version = "1.3.1" +description = "Pika Python AMQP Client Library" +category = "main" +optional = false +python-versions = ">=3.4" +files = [ + {file = "pika-1.3.1-py3-none-any.whl", hash = "sha256:89f5e606646caebe3c00cbdbc4c2c609834adde45d7507311807b5775edac8e0"}, + {file = "pika-1.3.1.tar.gz", hash = "sha256:beb19ff6dd1547f99a29acc2c6987ebb2ba7c44bf44a3f8e305877c5ef7d2fdc"}, +] + +[package.extras] +gevent = ["gevent"] +tornado = ["tornado"] +twisted = ["twisted"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.10" +content-hash = "c8c63462889297aa1d01f2c1b74a2d56f3fa034994b301d3302c455ba7257887" diff --git a/dispatcher/pyproject.toml b/dispatcher/pyproject.toml new file mode 100644 index 00000000..cb28b021 --- /dev/null +++ b/dispatcher/pyproject.toml @@ -0,0 +1,19 @@ +[tool.poetry] +name = "dispatcher" +version = "0.1.0" +description = "Polls rabbitMQ and takes action" +authors = ["linuxbandit "] +readme = "README.md" + +[tool.poetry.dependencies] +python = "^3.10" +pika = "^1.3.1" +jinja2 = "^3.1.2" + + +[tool.poetry.group.dev.dependencies] +faker = "^18.4.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/dispatcher/templates/member_expired.jinja2 b/dispatcher/templates/member_expired.jinja2 new file mode 100644 index 00000000..fd5d80dd --- /dev/null +++ b/dispatcher/templates/member_expired.jinja2 @@ -0,0 +1,73 @@ + + + + +{% with title="MyAEGEE: Membership expired" %} + {% include "snippets/head.jinja2" %} +{% endwith %} + + + + + + + + + + +
  + + + + + + +
+ + + + + + +
+
 
+
+ + + + + + + + + + + + +
+ AEGEE-Europe +
+
Membership expired
+
Your membership in {{ body }} has expired! Contact your local board in case you believe this is a mistake!
+
+
Your last fee payment was on {{ last_payment }}. You will not be able to apply to statutory events until your membership is reactivated.
+
+ + + + + + +
+
This email was autogenerated by MyAEGEE mailer.
+
+
+
+ + + diff --git a/dispatcher/templates/snippets/css.jinja2 b/dispatcher/templates/snippets/css.jinja2 new file mode 100644 index 00000000..f4a3ce3e --- /dev/null +++ b/dispatcher/templates/snippets/css.jinja2 @@ -0,0 +1,202 @@ + diff --git a/dispatcher/templates/snippets/head.jinja2 b/dispatcher/templates/snippets/head.jinja2 new file mode 100644 index 00000000..d987ad09 --- /dev/null +++ b/dispatcher/templates/snippets/head.jinja2 @@ -0,0 +1,11 @@ + + + + + + {{ title }} + + + + {% include "snippets/css.jinja2" %} + diff --git a/dispatcher/tests/__init__.py b/dispatcher/tests/__init__.py new file mode 100644 index 00000000..e69de29b From 67693ffbdd9bde02aff84d5abffdd18b9ec7a51f Mon Sep 17 00:00:00 2001 From: linuxbandit Date: Fri, 14 Apr 2023 09:57:41 +0200 Subject: [PATCH 02/32] feat: add retry if template not found removing the auto unack, the program will not remove the message from the queue if it can't process it --- dispatcher/README.md | 2 +- dispatcher/dispatcher/main.py | 17 +++-- dispatcher/helpers/send.py | 4 +- dispatcher/templates/new_member.jinja2 | 94 ++++++++++++++++++++++++++ 4 files changed, 109 insertions(+), 8 deletions(-) create mode 100644 dispatcher/templates/new_member.jinja2 diff --git a/dispatcher/README.md b/dispatcher/README.md index a8b4b958..fc6f8a4c 100644 --- a/dispatcher/README.md +++ b/dispatcher/README.md @@ -23,7 +23,6 @@ We do not need a web service for this, only a worker. Doing it this way only mea In order to add templates, one can work on the filesystem: as the template file is read from memory at the time a message is received, there is basically a mechanism of hot-reload ready to be used. We do not need a web service, because we do not need to pilot anything. - #FIXME validate this. Run app and add a new template and push to the queue a new template ## Queues @@ -40,4 +39,5 @@ Queues envisioned: rather in order: 1. (not on this project): run core with the email as 'inserting in the queue' instead of 'API request to mailer' 1. include traefik configuration to have the mailhog and rabbit on a subdomain instead of `domain:port` +1. When RabbitMQ quits or crashes it will forget the queues and messages unless you tell it not tot: we need to mark both the queue and messages as durable 1. add the telegram queue diff --git a/dispatcher/dispatcher/main.py b/dispatcher/dispatcher/main.py index 079a51c8..829a55bc 100755 --- a/dispatcher/dispatcher/main.py +++ b/dispatcher/dispatcher/main.py @@ -2,7 +2,7 @@ import json import smtplib from email.message import EmailMessage -from jinja2 import Environment, FileSystemLoader +from jinja2 import Environment, FileSystemLoader, exceptions """ continuously polls the email queue and renders+sends the template on every acked message @@ -21,18 +21,25 @@ def send_email(ch, method, properties, body): msg = json.loads(body) + try: + template = environment.get_template(f"{msg['template']}.jinja2") + except exceptions.TemplateNotFound: + return + # TODO: check if there is an auto-delete after 30 minutes for stuck un-acked messages + # TODO: add auto-retry. rabbit is smart and doesn't let me process a message again unless i force it + rendered = template.render(msg['parameters'], altro=msg['subject']) + email = EmailMessage() + email.set_content(rendered, subtype='html') email['From'] = msg['from'] email['Reply-To'] = msg['reply_to'] email['To'] = msg['to'] email['Subject'] = msg['subject'] - template = environment.get_template(f"{msg['template']}.jinja2") - rendered = template.render(msg['parameters'], altro=msg['subject']) - email.set_content(rendered, subtype='html') smtpObj.send_message(email) + ch.basic_ack(delivery_tag = method.delivery_tag) channel.basic_consume(queue='email', - auto_ack=True, + auto_ack=False, on_message_callback=send_email) print(' [*] Waiting for messages. To exit press CTRL+C') diff --git a/dispatcher/helpers/send.py b/dispatcher/helpers/send.py index bf69a503..8edd5216 100644 --- a/dispatcher/helpers/send.py +++ b/dispatcher/helpers/send.py @@ -20,8 +20,8 @@ def generate_fake_payload(): "to": [faker.email() for _ in range(random.randrange(1,3))], "reply_to": "noreply@aegee.eu", "subject": faker.sentence(), - "template": "member_expired", - "parameters": {"first_name": faker.name(), "body": f"AEGEE-{faker.city()}", "last_payment": faker.date()} + "template": "new_member", + "parameters": {"member_firstname": faker.first_name(), "body": f"AEGEE-{faker.city()}", "last_payment": faker.date(), "body_name": faker.language_name()} } return email diff --git a/dispatcher/templates/new_member.jinja2 b/dispatcher/templates/new_member.jinja2 new file mode 100644 index 00000000..c6e14564 --- /dev/null +++ b/dispatcher/templates/new_member.jinja2 @@ -0,0 +1,94 @@ + + + + +{% with title="MyAEGEE: Welcome to AEGEE" %} + {% include "snippets/head.jinja2" %} +{% endwith %} + + + + + + + + + +
  + + + + + + +
+ + + + + + +
+
 
+
+ + + + + + + + + + + + + + + + + + + + + +
+ AEGEE-Europe +
+
Hello, {{member_firstname}}
+
Welcome to AEGEE! You have taken the first steps to discover the AEGEE Universe. In this mail you can find some further information to help you further along in your journey.
+
+
For quick information and communication the following links will help you out:
+ AEGEE Wiki (the knowledge backbone of AEGEE)
+ Mailing lists (official communication channels of AEGEE-Europe which keep you updated)
+ AEGEEans (Facebook group with the latest news in AEGEE)
+ Telegram channel (updates from AEGEE-Europe)
+
+
MyAEGEE offers most information. Ranging from Events listing to all the locals and internal bodies of AEGEE-Europe. In the left menu under ‘Resources’ one can find quick access to the most important information.
+
+
In order to find out what’s happening in your local, get in touch with members and the board of {{ body_name }}.
+
+
Enjoy your time in AEGEE!
+
+ + + + + + +
+
This email was autogenerated by MyAEGEE mailer.
+
+
+
+ + + + + + From 5f9f4376f26cff024fa9966921da4729ba142ab6 Mon Sep 17 00:00:00 2001 From: linuxbandit Date: Mon, 17 Apr 2023 17:03:49 +0200 Subject: [PATCH 03/32] chore: add docker setup --- .env.example | 2 ++ dispatcher/README.md | 11 ++++++---- dispatcher/dispatcher/main.py | 8 +++---- dispatcher/docker-compose.yaml | 25 ---------------------- dispatcher/docker/Dockerfile | 17 +++++++++++++++ dispatcher/docker/docker-compose.dev.yml | 21 ++++++++++++++++++ dispatcher/docker/docker-compose.yml | 27 ++++++++++++++++++++++++ dispatcher/helpers/send.py | 12 ++++++++--- dispatcher/requirements.txt | 5 +++++ 9 files changed, 92 insertions(+), 36 deletions(-) delete mode 100644 dispatcher/docker-compose.yaml create mode 100644 dispatcher/docker/Dockerfile create mode 100644 dispatcher/docker/docker-compose.dev.yml create mode 100644 dispatcher/docker/docker-compose.yml create mode 100644 dispatcher/requirements.txt diff --git a/.env.example b/.env.example index 257630fa..71972c59 100644 --- a/.env.example +++ b/.env.example @@ -78,6 +78,7 @@ SUBDOMAIN_SURVEY=survey. SUBDOMAIN_UPMONITOR=ciao. SUBDOMAIN_STATPING=ping. SUBDOMAIN_APIDOCS=apidocs. +SUBDOMAIN_RABBITMQ=rabbit. SUBDOMAIN_REDISADMIN=redis. # Paths in the filesystem @@ -94,6 +95,7 @@ PATH_KNOWLEDGE=knowledge/docker/ PATH_PASS_MANAGER=pass-manager/docker/ PATH_MONITOR=monitor/docker/ PATH_GSUITE_WRAPPER=gsuite-wrapper/docker/ +PATH_DISPATCHER=dispatcher/docker/ PATH_OMS_ELASTIC=oms-elastic/docker/ # Other variables diff --git a/dispatcher/README.md b/dispatcher/README.md index fc6f8a4c..1c7c10ac 100644 --- a/dispatcher/README.md +++ b/dispatcher/README.md @@ -37,7 +37,10 @@ Queues envisioned: ## TODOs and next steps rather in order: -1. (not on this project): run core with the email as 'inserting in the queue' instead of 'API request to mailer' -1. include traefik configuration to have the mailhog and rabbit on a subdomain instead of `domain:port` -1. When RabbitMQ quits or crashes it will forget the queues and messages unless you tell it not tot: we need to mark both the queue and messages as durable -1. add the telegram queue + +1. [x] (not on this project): run core with the email as 'inserting in the queue' instead of 'API request to mailer' +1. [?] (not on this project): run core with the email as 'exchange' instead of 'inserting in the queue' +1. [x] include traefik configuration to have the mailhog and rabbit on a subdomain instead of `domain:port` +1. [ ] When RabbitMQ quits or crashes it will forget the queues and messages unless you tell it not to: we need to mark both the queue and messages as durable +1. [ ] Add auto-retry (DLQ). rabbit is smart and doesn't let me process a message again unless i force it.. https://devcorner.digitalpress.blog/rabbitmq-retries-the-new-full-story/ +1. [ ] add the telegram queue diff --git a/dispatcher/dispatcher/main.py b/dispatcher/dispatcher/main.py index 829a55bc..f6f66aae 100755 --- a/dispatcher/dispatcher/main.py +++ b/dispatcher/dispatcher/main.py @@ -10,10 +10,10 @@ environment = Environment(loader=FileSystemLoader("../templates/")) -EMAIL_HOST='172.18.0.13' #FIXME +EMAIL_HOST='mailhog' #FIXME differentiate between dev and prod smtpObj = smtplib.SMTP( EMAIL_HOST, 1025 ) -RABBIT_HOST='172.18.0.11' #FIXME +RABBIT_HOST='rabbit' connection = pika.BlockingConnection(pika.ConnectionParameters(RABBIT_HOST)) channel = connection.channel() @@ -24,9 +24,9 @@ def send_email(ch, method, properties, body): try: template = environment.get_template(f"{msg['template']}.jinja2") except exceptions.TemplateNotFound: + # TODO: send a notification to someone about adding a template return # TODO: check if there is an auto-delete after 30 minutes for stuck un-acked messages - # TODO: add auto-retry. rabbit is smart and doesn't let me process a message again unless i force it rendered = template.render(msg['parameters'], altro=msg['subject']) email = EmailMessage() @@ -35,7 +35,7 @@ def send_email(ch, method, properties, body): email['Reply-To'] = msg['reply_to'] email['To'] = msg['to'] email['Subject'] = msg['subject'] - smtpObj.send_message(email) + smtpObj.send_message(email) #TODO handle case in which smtp not ready ch.basic_ack(delivery_tag = method.delivery_tag) channel.basic_consume(queue='email', diff --git a/dispatcher/docker-compose.yaml b/dispatcher/docker-compose.yaml deleted file mode 100644 index 3fce07de..00000000 --- a/dispatcher/docker-compose.yaml +++ /dev/null @@ -1,25 +0,0 @@ -version: "3.4" - -services: - - rabbit: - image: rabbitmq:3.11-management - restart: always - expose: - - 5672 - ports: - - 8080:15672 - - mailhog: - image: mailhog/mailhog:v1.0.1 - restart: always - ports: - - 8025:8025 - - 1025:1025 - - - -networks: - default: - external: - name: OMS diff --git a/dispatcher/docker/Dockerfile b/dispatcher/docker/Dockerfile new file mode 100644 index 00000000..cf176cc8 --- /dev/null +++ b/dispatcher/docker/Dockerfile @@ -0,0 +1,17 @@ +FROM python:3.10-slim + +WORKDIR /usr/app/src + +COPY ./dispatcher/main.py /usr/app/src/main.py +COPY ./templates /usr/app/templates +COPY ./requirements.txt /usr/app/src/requirements.txt + +#USER python + +#ENV NPM_CONFIG_PREFIX=/home/node/.npm-global +#ENV PATH="/home/node/.npm-global/bin:${PATH}" + +RUN pip install --no-cache-dir -r requirements.txt + +ENTRYPOINT [ "python" ] +CMD ["main.py"] diff --git a/dispatcher/docker/docker-compose.dev.yml b/dispatcher/docker/docker-compose.dev.yml new file mode 100644 index 00000000..b0cd655b --- /dev/null +++ b/dispatcher/docker/docker-compose.dev.yml @@ -0,0 +1,21 @@ +version: "3.4" + +services: + + dispatcher: + build: + context: ./${PATH_DISPATCHER}/.. + dockerfile: ./docker/Dockerfile + image: aegee/dispatcher:dev + + mailhog: + image: mailhog/mailhog:v1.0.1 + restart: always + ports: + - 8025:8025 + - 1025:1025 + labels: + - "traefik.frontend.rule=Host:${SUBDOMAIN_MAILHOG}${BASE_URL};" + - "traefik.backend=mailhog" + - "traefik.port=8025" + - "traefik.enable=true" diff --git a/dispatcher/docker/docker-compose.yml b/dispatcher/docker/docker-compose.yml new file mode 100644 index 00000000..8669fa7e --- /dev/null +++ b/dispatcher/docker/docker-compose.yml @@ -0,0 +1,27 @@ +version: "3.4" + +services: + + rabbit: + image: rabbitmq:3.11-management + restart: always + expose: + - 5672 + - 15672 + labels: + - "traefik.frontend.rule=Host:${SUBDOMAIN_RABBITMQ}${BASE_URL};PathPrefix:/" + - "traefik.backend=rabbit" + - "traefik.port=15672" + - "traefik.enable=true" + + # Not a webservice, but a worker service + dispatcher: + image: aegee/dispatcher + restart: always + + + +networks: + default: + external: + name: OMS diff --git a/dispatcher/helpers/send.py b/dispatcher/helpers/send.py index 8edd5216..39c1f00c 100644 --- a/dispatcher/helpers/send.py +++ b/dispatcher/helpers/send.py @@ -9,7 +9,7 @@ from faker import Faker faker = Faker() -RABBIT_HOST='172.18.0.11' +RABBIT_HOST='172.18.0.8' #FIXME connection = pika.BlockingConnection(pika.ConnectionParameters(RABBIT_HOST)) channel = connection.channel() @@ -20,8 +20,14 @@ def generate_fake_payload(): "to": [faker.email() for _ in range(random.randrange(1,3))], "reply_to": "noreply@aegee.eu", "subject": faker.sentence(), - "template": "new_member", - "parameters": {"member_firstname": faker.first_name(), "body": f"AEGEE-{faker.city()}", "last_payment": faker.date(), "body_name": faker.language_name()} + "template": "new_member_dynamic", + "parameters": { # Not all will be used at the same time but this is not important, it's a test + "member_firstname": faker.first_name(), + "body": f"AEGEE-{faker.city()}", + "last_payment": faker.date(), + "body_name": faker.language_name(), + "place": faker.ssn(), + } } return email diff --git a/dispatcher/requirements.txt b/dispatcher/requirements.txt new file mode 100644 index 00000000..147529a3 --- /dev/null +++ b/dispatcher/requirements.txt @@ -0,0 +1,5 @@ +urllib3==1.22 +requests==2.18.4 +pika==1.3.1 +Jinja2==3.0.3 +Faker==14.2.1 From ba33f790b075868bb56fb28ecf60fec5c3fc04f8 Mon Sep 17 00:00:00 2001 From: linuxbandit Date: Wed, 19 Apr 2023 08:38:50 -0400 Subject: [PATCH 04/32] chore: declare queue only once --- dispatcher/helpers/send.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dispatcher/helpers/send.py b/dispatcher/helpers/send.py index 39c1f00c..939cc63f 100644 --- a/dispatcher/helpers/send.py +++ b/dispatcher/helpers/send.py @@ -12,6 +12,7 @@ RABBIT_HOST='172.18.0.8' #FIXME connection = pika.BlockingConnection(pika.ConnectionParameters(RABBIT_HOST)) channel = connection.channel() +channel.queue_declare(queue='email') def generate_fake_payload(): @@ -33,7 +34,6 @@ def generate_fake_payload(): for _ in range(random.randrange(1,8)): email = generate_fake_payload() - channel.queue_declare(queue='email') channel.basic_publish(exchange='', routing_key='email', body=json.dumps(email)) From b0cffbeb2559825d5f8d622216779f99e263ecbc Mon Sep 17 00:00:00 2001 From: linuxbandit Date: Wed, 19 Apr 2023 08:42:11 -0400 Subject: [PATCH 05/32] feat: make persistent queue and message --- dispatcher/README.md | 2 +- dispatcher/dispatcher/main.py | 2 +- dispatcher/helpers/send.py | 8 +++++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/dispatcher/README.md b/dispatcher/README.md index 1c7c10ac..cc94331a 100644 --- a/dispatcher/README.md +++ b/dispatcher/README.md @@ -41,6 +41,6 @@ rather in order: 1. [x] (not on this project): run core with the email as 'inserting in the queue' instead of 'API request to mailer' 1. [?] (not on this project): run core with the email as 'exchange' instead of 'inserting in the queue' 1. [x] include traefik configuration to have the mailhog and rabbit on a subdomain instead of `domain:port` -1. [ ] When RabbitMQ quits or crashes it will forget the queues and messages unless you tell it not to: we need to mark both the queue and messages as durable +1. [x] When RabbitMQ quits or crashes it will forget the queues and messages unless you tell it not to: we need to mark both the queue and messages as durable 1. [ ] Add auto-retry (DLQ). rabbit is smart and doesn't let me process a message again unless i force it.. https://devcorner.digitalpress.blog/rabbitmq-retries-the-new-full-story/ 1. [ ] add the telegram queue diff --git a/dispatcher/dispatcher/main.py b/dispatcher/dispatcher/main.py index f6f66aae..d34c6827 100755 --- a/dispatcher/dispatcher/main.py +++ b/dispatcher/dispatcher/main.py @@ -17,7 +17,7 @@ connection = pika.BlockingConnection(pika.ConnectionParameters(RABBIT_HOST)) channel = connection.channel() -channel.queue_declare(queue='email') +channel.queue_declare(queue='email', durable=True) def send_email(ch, method, properties, body): msg = json.loads(body) diff --git a/dispatcher/helpers/send.py b/dispatcher/helpers/send.py index 939cc63f..16eaa04d 100644 --- a/dispatcher/helpers/send.py +++ b/dispatcher/helpers/send.py @@ -12,8 +12,7 @@ RABBIT_HOST='172.18.0.8' #FIXME connection = pika.BlockingConnection(pika.ConnectionParameters(RABBIT_HOST)) channel = connection.channel() -channel.queue_declare(queue='email') - +channel.queue_declare(queue='email', durable=True) def generate_fake_payload(): email = { @@ -36,7 +35,10 @@ def generate_fake_payload(): email = generate_fake_payload() channel.basic_publish(exchange='', routing_key='email', - body=json.dumps(email)) + body=json.dumps(email), + properties=pika.BasicProperties( + delivery_mode = pika.spec.PERSISTENT_DELIVERY_MODE + )) print(f" [x] Sent {email['subject']} (to {email['to']})") From d9c689d051c1b45db0cb6d3c9d7f918692fbb224 Mon Sep 17 00:00:00 2001 From: linuxbandit Date: Wed, 19 Apr 2023 08:46:13 -0400 Subject: [PATCH 06/32] chore: add explicit exchange as per pattern --- dispatcher/dispatcher/main.py | 5 +++++ dispatcher/helpers/send.py | 9 +++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/dispatcher/dispatcher/main.py b/dispatcher/dispatcher/main.py index d34c6827..33932e38 100755 --- a/dispatcher/dispatcher/main.py +++ b/dispatcher/dispatcher/main.py @@ -17,7 +17,12 @@ connection = pika.BlockingConnection(pika.ConnectionParameters(RABBIT_HOST)) channel = connection.channel() +channel.exchange_declare(exchange='eml', + exchange_type='direct') channel.queue_declare(queue='email', durable=True) +channel.queue_bind(exchange='eml', + queue='email') +channel.basic_qos(prefetch_count=1) def send_email(ch, method, properties, body): msg = json.loads(body) diff --git a/dispatcher/helpers/send.py b/dispatcher/helpers/send.py index 16eaa04d..45e04f00 100644 --- a/dispatcher/helpers/send.py +++ b/dispatcher/helpers/send.py @@ -12,7 +12,12 @@ RABBIT_HOST='172.18.0.8' #FIXME connection = pika.BlockingConnection(pika.ConnectionParameters(RABBIT_HOST)) channel = connection.channel() + +channel.exchange_declare(exchange='eml', + exchange_type='direct') channel.queue_declare(queue='email', durable=True) +channel.queue_bind(exchange='eml', + queue='email') def generate_fake_payload(): email = { @@ -33,8 +38,8 @@ def generate_fake_payload(): for _ in range(random.randrange(1,8)): email = generate_fake_payload() - channel.basic_publish(exchange='', - routing_key='email', + channel.basic_publish(exchange='eml', + routing_key='mail', body=json.dumps(email), properties=pika.BasicProperties( delivery_mode = pika.spec.PERSISTENT_DELIVERY_MODE From ce89fb25ba5ef56e965c67c4278fd81d04610154 Mon Sep 17 00:00:00 2001 From: linuxbandit Date: Wed, 19 Apr 2023 10:14:53 -0400 Subject: [PATCH 07/32] chore: make exchange persistent and add route key --- dispatcher/dispatcher/main.py | 9 ++++++--- dispatcher/helpers/send.py | 6 ++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/dispatcher/dispatcher/main.py b/dispatcher/dispatcher/main.py index 33932e38..0b7b0e10 100755 --- a/dispatcher/dispatcher/main.py +++ b/dispatcher/dispatcher/main.py @@ -5,7 +5,8 @@ from jinja2 import Environment, FileSystemLoader, exceptions """ -continuously polls the email queue and renders+sends the template on every acked message +continuously polls(*) the email queue and renders+sends the template on every acked message +(*) = waits for the queue to push a message onto the app """ environment = Environment(loader=FileSystemLoader("../templates/")) @@ -18,10 +19,12 @@ channel = connection.channel() channel.exchange_declare(exchange='eml', - exchange_type='direct') + exchange_type='direct', + durable=True) channel.queue_declare(queue='email', durable=True) channel.queue_bind(exchange='eml', - queue='email') + queue='email', + routing_key='mail') channel.basic_qos(prefetch_count=1) def send_email(ch, method, properties, body): diff --git a/dispatcher/helpers/send.py b/dispatcher/helpers/send.py index 45e04f00..26afed8d 100644 --- a/dispatcher/helpers/send.py +++ b/dispatcher/helpers/send.py @@ -14,10 +14,12 @@ channel = connection.channel() channel.exchange_declare(exchange='eml', - exchange_type='direct') + exchange_type='direct', + durable=True) channel.queue_declare(queue='email', durable=True) channel.queue_bind(exchange='eml', - queue='email') + queue='email', + routing_key='mail') def generate_fake_payload(): email = { From 5019cb260feb6680c4756cf5cd55ef84481b7a87 Mon Sep 17 00:00:00 2001 From: linuxbandit Date: Wed, 19 Apr 2023 12:27:13 -0400 Subject: [PATCH 08/32] feat: add full templates tester, add templates --- dispatcher/dispatcher/main.py | 3 +- dispatcher/helpers/send.py | 69 ++++++++++++++---- dispatcher/templates/confirm_email.jinja2 | 84 +++++++++++++++++++++ dispatcher/templates/mail_change.jinja2 | 89 +++++++++++++++++++++++ dispatcher/templates/member_joined.jinja2 | 77 ++++++++++++++++++++ 5 files changed, 307 insertions(+), 15 deletions(-) create mode 100644 dispatcher/templates/confirm_email.jinja2 create mode 100644 dispatcher/templates/mail_change.jinja2 create mode 100644 dispatcher/templates/member_joined.jinja2 diff --git a/dispatcher/dispatcher/main.py b/dispatcher/dispatcher/main.py index 0b7b0e10..ba44d811 100755 --- a/dispatcher/dispatcher/main.py +++ b/dispatcher/dispatcher/main.py @@ -25,7 +25,7 @@ channel.queue_bind(exchange='eml', queue='email', routing_key='mail') -channel.basic_qos(prefetch_count=1) +#channel.basic_qos(prefetch_count=1) #TODO: notice that an error processing a message will BLOCK the others from being processed def send_email(ch, method, properties, body): msg = json.loads(body) @@ -33,6 +33,7 @@ def send_email(ch, method, properties, body): template = environment.get_template(f"{msg['template']}.jinja2") except exceptions.TemplateNotFound: # TODO: send a notification to someone about adding a template + print(f"template {msg['template']}.jinja2 not found") return # TODO: check if there is an auto-delete after 30 minutes for stuck un-acked messages rendered = template.render(msg['parameters'], altro=msg['subject']) diff --git a/dispatcher/helpers/send.py b/dispatcher/helpers/send.py index 26afed8d..042574ba 100644 --- a/dispatcher/helpers/send.py +++ b/dispatcher/helpers/send.py @@ -9,6 +9,34 @@ from faker import Faker faker = Faker() +RANDOM_AMOUNT_TEST=False +ALL_TEMPLATES_TEST=True + +BODIES_LIST = [ "ITC", "HRC", "CC", "SomeCommission", "JC", "DPC", "MedCom" ] +# from constants.js +MAIL_SUBJECTS = { + "CORE": { + "MAIL_CONFIRMATION": 'MyAEGEE: Please confirm your account', + "MAIL_CHANGE": 'MyAEGEE: Email change', + "PASSWORD_RESET": 'MyAEGEE: Password reset request', + "NEW_JOIN_REQUEST": 'MyAEGEE: New join request for your body', + "NEW_MEMBER": 'MyAEGEE: Welcome to AEGEE' + }, + #"EVENTS": {}, +} +# should exist in constants.js but it does not yet. +# anyway here one could #TODO a smarter way: look into the filesystem +MAIL_TEMPLATES = { + "CORE": { + "MAIL_CONFIRMATION": 'confirm_email', + "MAIL_CHANGE": 'mail_change', + "PASSWORD_RESET": 'password_reset', + "NEW_JOIN_REQUEST": 'member_joined', + "NEW_MEMBER": 'new_member' + }, + #"EVENTS": {}, +} + RABBIT_HOST='172.18.0.8' #FIXME connection = pika.BlockingConnection(pika.ConnectionParameters(RABBIT_HOST)) channel = connection.channel() @@ -21,33 +49,46 @@ queue='email', routing_key='mail') -def generate_fake_payload(): +def generate_fake_payload(subj="", template=""): email = { "from": "mailer@aegee.eu", "to": [faker.email() for _ in range(random.randrange(1,3))], "reply_to": "noreply@aegee.eu", - "subject": faker.sentence(), - "template": "new_member_dynamic", + "subject": subj or faker.sentence(), + "template": template or "new_member_dynamic", #TODO remove the or/make it fail "parameters": { # Not all will be used at the same time but this is not important, it's a test "member_firstname": faker.first_name(), + "member_lastname": faker.last_name(), "body": f"AEGEE-{faker.city()}", "last_payment": faker.date(), - "body_name": faker.language_name(), - "place": faker.ssn(), + "body_name": random.choice(BODIES_LIST), + "place": faker.city(), } } return email -for _ in range(random.randrange(1,8)): - email = generate_fake_payload() - channel.basic_publish(exchange='eml', - routing_key='mail', - body=json.dumps(email), - properties=pika.BasicProperties( - delivery_mode = pika.spec.PERSISTENT_DELIVERY_MODE - )) - print(f" [x] Sent {email['subject']} (to {email['to']})") +if(RANDOM_AMOUNT_TEST): + for _ in range(random.randrange(1,8)): + email = generate_fake_payload() + channel.basic_publish(exchange='eml', + routing_key='mail', + body=json.dumps(email), + properties=pika.BasicProperties( + delivery_mode = pika.spec.PERSISTENT_DELIVERY_MODE + )) + print(f" [x] Sent {email['subject']} (to {email['to']})") +if(ALL_TEMPLATES_TEST): + for ms in MAIL_TEMPLATES: + for case in MAIL_TEMPLATES[ms]: + email = generate_fake_payload(MAIL_SUBJECTS[ms][case], MAIL_TEMPLATES[ms][case]) + channel.basic_publish(exchange='eml', + routing_key='mail', + body=json.dumps(email), + properties=pika.BasicProperties( + delivery_mode = pika.spec.PERSISTENT_DELIVERY_MODE + )) + print(f" [x] Sent {email['subject']}") connection.close() diff --git a/dispatcher/templates/confirm_email.jinja2 b/dispatcher/templates/confirm_email.jinja2 new file mode 100644 index 00000000..980a5548 --- /dev/null +++ b/dispatcher/templates/confirm_email.jinja2 @@ -0,0 +1,84 @@ + + + + +{% import "snippets/macros.jinja2" as macros %} +{{ macros.head("MyAEGEE: Confirm your registation") }} + + + + + + + + + + +
  + + + + + + +
+ + + + + + +
+
 
+
+ + + + + + + + + + + + + + + +
+ AEGEE-Europe +
+
Hello {{member_firstname}} {{member_surname}}
+
You've recently registered on AEGEE Intranet. To confirm your account, follow this link
+
+ +
+
If the link does not work for you, go to this page: https://my.aegee.eu/confirm_signup and paste this token into it:
+
<%= Map.fetch!(@parameters, "token") %>
+
+ + + + + + +
+
This email was autogenerated by MyAEGEE mailer.
+
+
+
+ + + + + + diff --git a/dispatcher/templates/mail_change.jinja2 b/dispatcher/templates/mail_change.jinja2 new file mode 100644 index 00000000..761e9b33 --- /dev/null +++ b/dispatcher/templates/mail_change.jinja2 @@ -0,0 +1,89 @@ + + + + +{% import "snippets/macros.jinja2" as macros %} +{{ macros.head("MyAEGEE: Mail change") }} + + + + + + + + + +
  + + + + + + +
+ + + + + + +
+
 
+
+ + + + + + + + + + + + + + + + + + +
+ +
+
Hello {{member_firstname}} {{member_surname}}
+
You've asked to change your email. To do so, follow this link:
+
+ +
+
If the link does not work for you, go to this page: https://my.aegee.eu/mail-change and paste this token into it:
+
<%= Map.fetch!(@parameters, "token") %>
+
+
+ If you didn't request a mail change, just ignore this email. +
+
+ + + + + + +
+
This email was autogenerated by MyAEGEE mailer.
+
+
+
+ + + + + + diff --git a/dispatcher/templates/member_joined.jinja2 b/dispatcher/templates/member_joined.jinja2 new file mode 100644 index 00000000..188350d3 --- /dev/null +++ b/dispatcher/templates/member_joined.jinja2 @@ -0,0 +1,77 @@ + + + + +{% import "snippets/macros.jinja2" as macros %} +{{ macros.head("MyAEGEE: New Join Request") }} + + + + + + + + + +
  + + + + + + +
+ + + + + + +
+
 
+
+ + + + + + + + + + + + +
+ Logo +
+
You might have a new member soon!
+
Recently member {{member_firstname}} {{member_surname}} requested to join your body {{body_name}}
+
+ +
+ + + + + + +
+
This email was autogenerated by MyAEGEE mailer.
+
+
+
+ + + + + + From ff69624cedd410e178f8cbc1f5f2e1d0a0c0185d Mon Sep 17 00:00:00 2001 From: linuxbandit Date: Wed, 19 Apr 2023 12:31:47 -0400 Subject: [PATCH 09/32] chore: optimise docker image building; add volumes --- dispatcher/docker/Dockerfile | 12 ++++++------ dispatcher/docker/docker-compose.dev.yml | 3 +++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/dispatcher/docker/Dockerfile b/dispatcher/docker/Dockerfile index cf176cc8..f8418f9b 100644 --- a/dispatcher/docker/Dockerfile +++ b/dispatcher/docker/Dockerfile @@ -2,16 +2,16 @@ FROM python:3.10-slim WORKDIR /usr/app/src -COPY ./dispatcher/main.py /usr/app/src/main.py -COPY ./templates /usr/app/templates COPY ./requirements.txt /usr/app/src/requirements.txt +RUN pip install --no-cache-dir -r requirements.txt + +COPY ./templates /usr/app/templates +COPY ./dispatcher/main.py /usr/app/src/main.py #USER python -#ENV NPM_CONFIG_PREFIX=/home/node/.npm-global +#ENV PORCO=dio #ENV PATH="/home/node/.npm-global/bin:${PATH}" -RUN pip install --no-cache-dir -r requirements.txt - -ENTRYPOINT [ "python" ] +ENTRYPOINT [ "python", "-u" ] CMD ["main.py"] diff --git a/dispatcher/docker/docker-compose.dev.yml b/dispatcher/docker/docker-compose.dev.yml index b0cd655b..6ad4d22d 100644 --- a/dispatcher/docker/docker-compose.dev.yml +++ b/dispatcher/docker/docker-compose.dev.yml @@ -7,6 +7,9 @@ services: context: ./${PATH_DISPATCHER}/.. dockerfile: ./docker/Dockerfile image: aegee/dispatcher:dev + volumes: + - ./${PATH_DISPATCHER}/../templates:/usr/app/templates:ro + - ./${PATH_DISPATCHER}/../dispatcher:/usr/app/src:ro mailhog: image: mailhog/mailhog:v1.0.1 From ae97e74936c58babe97f3c37740940a8f53d0ff1 Mon Sep 17 00:00:00 2001 From: linuxbandit Date: Wed, 19 Apr 2023 13:31:50 -0400 Subject: [PATCH 10/32] chore: correct core templates --- dispatcher/helpers/send.py | 2 + dispatcher/templates/confirm_email.jinja2 | 5 +- dispatcher/templates/mail_change.jinja2 | 4 +- dispatcher/templates/member_joined.jinja2 | 2 +- dispatcher/templates/password_reset.jinja2 | 84 ++++++++++++++++++++++ 5 files changed, 91 insertions(+), 6 deletions(-) create mode 100644 dispatcher/templates/password_reset.jinja2 diff --git a/dispatcher/helpers/send.py b/dispatcher/helpers/send.py index 042574ba..5865d3c8 100644 --- a/dispatcher/helpers/send.py +++ b/dispatcher/helpers/send.py @@ -62,7 +62,9 @@ def generate_fake_payload(subj="", template=""): "body": f"AEGEE-{faker.city()}", "last_payment": faker.date(), "body_name": random.choice(BODIES_LIST), + "body_id": random.choice(range(random.randrange(10,70))), "place": faker.city(), + "token": faker.md5(), } } return email diff --git a/dispatcher/templates/confirm_email.jinja2 b/dispatcher/templates/confirm_email.jinja2 index 980a5548..d27154f9 100644 --- a/dispatcher/templates/confirm_email.jinja2 +++ b/dispatcher/templates/confirm_email.jinja2 @@ -5,7 +5,6 @@ {% import "snippets/macros.jinja2" as macros %} {{ macros.head("MyAEGEE: Confirm your registation") }} - @@ -46,7 +45,7 @@ @@ -54,7 +53,7 @@ diff --git a/dispatcher/templates/mail_change.jinja2 b/dispatcher/templates/mail_change.jinja2 index 761e9b33..070fdf4b 100644 --- a/dispatcher/templates/mail_change.jinja2 +++ b/dispatcher/templates/mail_change.jinja2 @@ -44,7 +44,7 @@ @@ -52,7 +52,7 @@ diff --git a/dispatcher/templates/member_joined.jinja2 b/dispatcher/templates/member_joined.jinja2 index 188350d3..7db54f6f 100644 --- a/dispatcher/templates/member_joined.jinja2 +++ b/dispatcher/templates/member_joined.jinja2 @@ -45,7 +45,7 @@ diff --git a/dispatcher/templates/password_reset.jinja2 b/dispatcher/templates/password_reset.jinja2 new file mode 100644 index 00000000..593630e6 --- /dev/null +++ b/dispatcher/templates/password_reset.jinja2 @@ -0,0 +1,84 @@ + + + + +{% import "snippets/macros.jinja2" as macros %} +{{ macros.head("MyAEGEE: Password reset") }} + + +
If the link does not work for you, go to this page: https://my.aegee.eu/confirm_signup and paste this token into it:
-
<%= Map.fetch!(@parameters, "token") %>
+
{{token}}
If the link does not work for you, go to this page: https://my.aegee.eu/mail-change and paste this token into it:
-
<%= Map.fetch!(@parameters, "token") %>
+
{{token}}
+ + + + + + +
  + + + + + + +
+ + + + + + +
+
 
+
+ + + + + + + + + + + + + + + +
+ AEGEE Logo +
+
Hello
+
You requested a password reset. To finish the password reset, follow this link:
+
+ +
+
If the link does not work for you, go to this page: https://my.aegee.eu/password_confirm and paste this token into it:
+
{{token}}
+
Also notice that password resets expire after 30 minutes
+
+ + + + + + +
+
This email was autogenerated by MyAEGEE mailer.
+
+
+
+ + + + + + From 4a0fddf2434d1651e748806b504cb80418d48025 Mon Sep 17 00:00:00 2001 From: linuxbandit Date: Wed, 19 Apr 2023 15:31:27 -0400 Subject: [PATCH 11/32] feat: add SU templates. Also add macros.jinja2 not sure how it was not present at all --- dispatcher/helpers/send.py | 26 +++++- dispatcher/templates/snippets/head.jinja2 | 2 - dispatcher/templates/snippets/macros.jinja2 | 10 +++ ...summeruniversity_application_edited.jinja2 | 76 +++++++++++++++++ .../templates/summeruniversity_applied.jinja2 | 77 +++++++++++++++++ .../summeruniversity_board_applied.jinja2 | 84 ++++++++++++++++++ .../summeruniversity_board_edited.jinja2 | 85 +++++++++++++++++++ .../summeruniversity_organizer_applied.jinja2 | 85 +++++++++++++++++++ .../summeruniversity_organizer_edited.jinja2 | 85 +++++++++++++++++++ 9 files changed, 527 insertions(+), 3 deletions(-) create mode 100644 dispatcher/templates/snippets/macros.jinja2 create mode 100644 dispatcher/templates/summeruniversity_application_edited.jinja2 create mode 100644 dispatcher/templates/summeruniversity_applied.jinja2 create mode 100644 dispatcher/templates/summeruniversity_board_applied.jinja2 create mode 100644 dispatcher/templates/summeruniversity_board_edited.jinja2 create mode 100644 dispatcher/templates/summeruniversity_organizer_applied.jinja2 create mode 100644 dispatcher/templates/summeruniversity_organizer_edited.jinja2 diff --git a/dispatcher/helpers/send.py b/dispatcher/helpers/send.py index 5865d3c8..a9b8d8d2 100644 --- a/dispatcher/helpers/send.py +++ b/dispatcher/helpers/send.py @@ -4,6 +4,8 @@ """ creates between 1 and 8 fake emails and puts in the queue +OR +tests all templates it finds in the folder """ from faker import Faker @@ -13,6 +15,9 @@ ALL_TEMPLATES_TEST=True BODIES_LIST = [ "ITC", "HRC", "CC", "SomeCommission", "JC", "DPC", "MedCom" ] +su_words_list = ['summer', 'awesome', 'your', 'now', 'sustainability', 'culture', 'europe', 'balkan', 'russia', 'adventure', 'ukraine', 'capital', 'montenegro', 'ireland', 'serbia', 'crimea', 'amazing', 'slavaukraini', 'heroiamslava'] +def su_sentence(): + return faker.sentence(nb_words=5, ext_word_list=su_words_list) # from constants.js MAIL_SUBJECTS = { "CORE": { @@ -23,9 +28,18 @@ "NEW_MEMBER": 'MyAEGEE: Welcome to AEGEE' }, #"EVENTS": {}, + "SUMMERUNIVERSITIES": { + "MAIL_APPLIED_MEMBER": f"You've successfully applied for {su_sentence()}", + "MAIL_APPLIED_ORGANISERS": f"Somebody has applied for {su_sentence()}", + "MAIL_APPLIED_BOARD": f"One of your body members has applied to {su_sentence()}", + "MAIL_UPDATED_MEMBER": f"Your application for {su_sentence()} was updated", + "MAIL_UPDATED_ORGANISERS": f"Somebody has updated their application for {su_sentence()}", + "MAIL_UPDATED_BOARD": f"One of your body members has updated their application to {su_sentence()}", + }, } # should exist in constants.js but it does not yet. # anyway here one could #TODO a smarter way: look into the filesystem +# but then you miss the correspondence between the subject and the template MAIL_TEMPLATES = { "CORE": { "MAIL_CONFIRMATION": 'confirm_email', @@ -35,6 +49,14 @@ "NEW_MEMBER": 'new_member' }, #"EVENTS": {}, + "SUMMERUNIVERSITIES": { + "MAIL_APPLIED_MEMBER": "summeruniversity_applied", + "MAIL_APPLIED_ORGANISERS": "summeruniversity_organizer_applied", + "MAIL_APPLIED_BOARD": "summeruniversity_board_applied", + "MAIL_UPDATED_MEMBER": "summeruniversity_application_edited", + "MAIL_UPDATED_ORGANISERS": "summeruniversity_organizer_edited", + "MAIL_UPDATED_BOARD": "summeruniversity_board_edited", + }, } RABBIT_HOST='172.18.0.8' #FIXME @@ -61,10 +83,12 @@ def generate_fake_payload(subj="", template=""): "member_lastname": faker.last_name(), "body": f"AEGEE-{faker.city()}", "last_payment": faker.date(), - "body_name": random.choice(BODIES_LIST), + "body_name": random.choice(BODIES_LIST), #note: discrepancy in the microservices on the use of body vs body_name "body_id": random.choice(range(random.randrange(10,70))), "place": faker.city(), "token": faker.md5(), + "event": { "name": su_sentence(), "location": faker.city(), "url": "example.org"}, + "application": { "first_name": faker.first_name(), "last_name": faker.last_name(), "body_name": random.choice(BODIES_LIST)}, } } return email diff --git a/dispatcher/templates/snippets/head.jinja2 b/dispatcher/templates/snippets/head.jinja2 index d987ad09..b8367653 100644 --- a/dispatcher/templates/snippets/head.jinja2 +++ b/dispatcher/templates/snippets/head.jinja2 @@ -1,6 +1,4 @@ - - {{ title }} diff --git a/dispatcher/templates/snippets/macros.jinja2 b/dispatcher/templates/snippets/macros.jinja2 new file mode 100644 index 00000000..4a07a4e9 --- /dev/null +++ b/dispatcher/templates/snippets/macros.jinja2 @@ -0,0 +1,10 @@ +{% macro head(title) %} + + + {{ title }} + + + + {% include "snippets/css.jinja2" %} + +{% endmacro %} diff --git a/dispatcher/templates/summeruniversity_application_edited.jinja2 b/dispatcher/templates/summeruniversity_application_edited.jinja2 new file mode 100644 index 00000000..62de0d15 --- /dev/null +++ b/dispatcher/templates/summeruniversity_application_edited.jinja2 @@ -0,0 +1,76 @@ + + + + +{% import "snippets/macros.jinja2" as macros %} +{{ macros.head("MyAEGEE: Your application for VAR"|replace("VAR", event["name"])) }} + + + + + + + + + +
  + + + + + + +
+ + + + + + +
+
 
+
+ + + + + + + + + + + + +
+ AEGEE-Europe +
+
Your application for {{event["name"]}} was successfully updated.
+
+
+ Here are your application details: + <%= render(OmsmailerWeb.SnippetsView, "summeruniversity_application_info.html", parameters: @parameters ) %> +
+
+ + + + + + +
+
This email was autogenerated by MyAEGEE mailer.
+
+
+
+ + + + + + diff --git a/dispatcher/templates/summeruniversity_applied.jinja2 b/dispatcher/templates/summeruniversity_applied.jinja2 new file mode 100644 index 00000000..b403366f --- /dev/null +++ b/dispatcher/templates/summeruniversity_applied.jinja2 @@ -0,0 +1,77 @@ + + + + +{% import "snippets/macros.jinja2" as macros %} +{{ macros.head("MyAEGEE: Your application for VAR"|replace("VAR", event["name"])) }} + + + + + + + + + + +
  + + + + + + +
+ + + + + + +
+
 
+
+ + + + + + + + + + + + +
+ AEGEE-Europe +
+
You've successfully applied to {{event["name"]}}
+
+
+ Here are your application details: + <%= render(OmsmailerWeb.SnippetsView, "summeruniversity_application_info.html", parameters: @parameters ) %> +
+
+ + + + + + +
+
This email was autogenerated by MyAEGEE mailer.
+
+
+
+ + + + + + diff --git a/dispatcher/templates/summeruniversity_board_applied.jinja2 b/dispatcher/templates/summeruniversity_board_applied.jinja2 new file mode 100644 index 00000000..e6ddeb88 --- /dev/null +++ b/dispatcher/templates/summeruniversity_board_applied.jinja2 @@ -0,0 +1,84 @@ + + + + +{% import "snippets/macros.jinja2" as macros %} +{{ macros.head("MyAEGEE: A member of your body applied for VAR"|replace("VAR", event["name"])) }} + + + + + + + + + +
  + + + + + + +
+ + + + + + +
+
 
+
+ + + + + + + + + + + + + + + +
+ +
+
A member of your body ({{application["body_name"]}}) applied for {{event["name"]}}
+
+
+ Here are the application details: + <%= render(OmsmailerWeb.SnippetsView, "summeruniversity_application_info.html", parameters: @parameters ) %> +
+
+ +
+ + + + + + +
+
This email was autogenerated by MyAEGEE mailer.
+
+
+
+ + + + + + diff --git a/dispatcher/templates/summeruniversity_board_edited.jinja2 b/dispatcher/templates/summeruniversity_board_edited.jinja2 new file mode 100644 index 00000000..21ddea90 --- /dev/null +++ b/dispatcher/templates/summeruniversity_board_edited.jinja2 @@ -0,0 +1,85 @@ + + + + +{% import "snippets/macros.jinja2" as macros %} +{{ macros.head("MyAEGEE: A member of your body edited the application for VAR"|replace("VAR", event["name"])) }} + + + + + + + + + + +
  + + + + + + +
+ + + + + + +
+
 
+
+ + + + + + + + + + + + + + + +
+ +
+
A member of your body ({{application["body_name"]}}) edited the application for {{event["name"]}}
+
+
+ Here are the application details: + <%= render(OmsmailerWeb.SnippetsView, "summeruniversity_application_info.html", parameters: @parameters ) %> +
+
+ +
+ + + + + + +
+
This email was autogenerated by MyAEGEE mailer.
+
+
+
+ + + + + + diff --git a/dispatcher/templates/summeruniversity_organizer_applied.jinja2 b/dispatcher/templates/summeruniversity_organizer_applied.jinja2 new file mode 100644 index 00000000..c6f9ceed --- /dev/null +++ b/dispatcher/templates/summeruniversity_organizer_applied.jinja2 @@ -0,0 +1,85 @@ + + + + +{% import "snippets/macros.jinja2" as macros %} +{{ macros.head("MyAEGEE: Somebody has applied for VAR"|replace("VAR", event["name"])) }} + + + + + + + + + + +
  + + + + + + +
+ + + + + + +
+
 
+
+ + + + + + + + + + + + + + + +
+ +
+
{{application["first_name"]}} {{application["last_name"]}} applied for {{event["name"]}}
+
+
+ Here are the application details: + <%= render(OmsmailerWeb.SnippetsView, "summeruniversity_application_info.html", parameters: @parameters ) %> +
+
+ +
+ + + + + + +
+
This email was autogenerated by MyAEGEE mailer.
+
+
+
+ + + + + + diff --git a/dispatcher/templates/summeruniversity_organizer_edited.jinja2 b/dispatcher/templates/summeruniversity_organizer_edited.jinja2 new file mode 100644 index 00000000..ca683208 --- /dev/null +++ b/dispatcher/templates/summeruniversity_organizer_edited.jinja2 @@ -0,0 +1,85 @@ + + + + +{% import "snippets/macros.jinja2" as macros %} +{{ macros.head("MyAEGEE: Somebody has updated their application for VAR"|replace("VAR", event["name"])) }} + + + + + + + + + + +
  + + + + + + +
+ + + + + + +
+
 
+
+ + + + + + + + + + + + + + + +
+ +
+
{{application["first_name"]}} {{application["last_name"]}} edited their application for {{event["name"]}}
+
+
+ Here are the application details: + <%= render(OmsmailerWeb.SnippetsView, "summeruniversity_application_info.html", parameters: @parameters ) %> +
+
+ +
+ + + + + + +
+
This email was autogenerated by MyAEGEE mailer.
+
+
+
+ + + + + + From cb9c695ae429275c3867f3e6f2d3dfaf082480a9 Mon Sep 17 00:00:00 2001 From: linuxbandit Date: Wed, 19 Apr 2023 15:32:03 -0400 Subject: [PATCH 12/32] chore: remove comment --- dispatcher/docker/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dispatcher/docker/Dockerfile b/dispatcher/docker/Dockerfile index f8418f9b..0c94e39b 100644 --- a/dispatcher/docker/Dockerfile +++ b/dispatcher/docker/Dockerfile @@ -10,8 +10,7 @@ COPY ./dispatcher/main.py /usr/app/src/main.py #USER python -#ENV PORCO=dio -#ENV PATH="/home/node/.npm-global/bin:${PATH}" +#ENV FOO=BAR ENTRYPOINT [ "python", "-u" ] CMD ["main.py"] From bae8aa9163dce6a92ad87603235647d4b0aefb69 Mon Sep 17 00:00:00 2001 From: linuxbandit Date: Fri, 21 Apr 2023 09:54:47 -0400 Subject: [PATCH 13/32] chore: add all templates for SU, working --- .env.example | 1 + dispatcher/docker/docker-compose.yml | 1 - dispatcher/helpers/send.py | 44 +++++++++- .../summeruniversity_application_info.jinja2 | 15 ++++ ...summeruniversity_application_edited.jinja2 | 2 +- ...iversity_application_status_updated.jinja2 | 78 +++++++++++++++++ .../templates/summeruniversity_applied.jinja2 | 2 +- .../summeruniversity_board_applied.jinja2 | 2 +- .../summeruniversity_board_edited.jinja2 | 2 +- .../summeruniversity_event_created.jinja2 | 82 ++++++++++++++++++ .../summeruniversity_event_updated.jinja2 | 82 ++++++++++++++++++ .../summeruniversity_organizer_applied.jinja2 | 2 +- .../summeruniversity_organizer_edited.jinja2 | 2 +- .../summeruniversity_status_changed.jinja2 | 83 +++++++++++++++++++ .../summeruniversity_submitted.jinja2 | 82 ++++++++++++++++++ 15 files changed, 470 insertions(+), 10 deletions(-) create mode 100644 dispatcher/templates/snippets/summeruniversity_application_info.jinja2 create mode 100644 dispatcher/templates/summeruniversity_application_status_updated.jinja2 create mode 100644 dispatcher/templates/summeruniversity_event_created.jinja2 create mode 100644 dispatcher/templates/summeruniversity_event_updated.jinja2 create mode 100644 dispatcher/templates/summeruniversity_status_changed.jinja2 create mode 100644 dispatcher/templates/summeruniversity_submitted.jinja2 diff --git a/.env.example b/.env.example index 71972c59..9b990d00 100644 --- a/.env.example +++ b/.env.example @@ -79,6 +79,7 @@ SUBDOMAIN_UPMONITOR=ciao. SUBDOMAIN_STATPING=ping. SUBDOMAIN_APIDOCS=apidocs. SUBDOMAIN_RABBITMQ=rabbit. +SUBDOMAIN_MAILHOG=mailhog. SUBDOMAIN_REDISADMIN=redis. # Paths in the filesystem diff --git a/dispatcher/docker/docker-compose.yml b/dispatcher/docker/docker-compose.yml index 8669fa7e..1acc1067 100644 --- a/dispatcher/docker/docker-compose.yml +++ b/dispatcher/docker/docker-compose.yml @@ -20,7 +20,6 @@ services: restart: always - networks: default: external: diff --git a/dispatcher/helpers/send.py b/dispatcher/helpers/send.py index a9b8d8d2..86745ef2 100644 --- a/dispatcher/helpers/send.py +++ b/dispatcher/helpers/send.py @@ -12,6 +12,8 @@ faker = Faker() RANDOM_AMOUNT_TEST=False +MIN_MSG=1 +MAX_MSG=8 ALL_TEMPLATES_TEST=True BODIES_LIST = [ "ITC", "HRC", "CC", "SomeCommission", "JC", "DPC", "MedCom" ] @@ -35,6 +37,11 @@ def su_sentence(): "MAIL_UPDATED_MEMBER": f"Your application for {su_sentence()} was updated", "MAIL_UPDATED_ORGANISERS": f"Somebody has updated their application for {su_sentence()}", "MAIL_UPDATED_BOARD": f"One of your body members has updated their application to {su_sentence()}", + "MAIL_APPL_STATUS_CHANGED": f"Your application status for {su_sentence()} was updated", + "MAIL_SU_CREATED": "The event was created", + "MAIL_SU_UPDATED": "The event was updated", + "MAIL_SU_STATUS_CHANGED": "Your event's status was changed", + "MAIL_SU_SUBMITTED": "An event was submitted", }, } # should exist in constants.js but it does not yet. @@ -56,6 +63,11 @@ def su_sentence(): "MAIL_UPDATED_MEMBER": "summeruniversity_application_edited", "MAIL_UPDATED_ORGANISERS": "summeruniversity_organizer_edited", "MAIL_UPDATED_BOARD": "summeruniversity_board_edited", + "MAIL_APPL_STATUS_CHANGED": "summeruniversity_application_status_updated", + "MAIL_SU_CREATED": "summeruniversity_event_created", + "MAIL_SU_UPDATED": "summeruniversity_event_updated", + "MAIL_SU_STATUS_CHANGED": "summeruniversity_status_changed", + "MAIL_SU_SUBMITTED": "summeruniversity_submitted", }, } @@ -87,15 +99,40 @@ def generate_fake_payload(subj="", template=""): "body_id": random.choice(range(random.randrange(10,70))), "place": faker.city(), "token": faker.md5(), - "event": { "name": su_sentence(), "location": faker.city(), "url": "example.org"}, - "application": { "first_name": faker.first_name(), "last_name": faker.last_name(), "body_name": random.choice(BODIES_LIST)}, + "old_status": "snafu", + "event": { + "name": su_sentence(), + "location": faker.city(), + "url": "example.org", + "status": "damned", + "questions": [ + { "description": "Who are you?" }, + { "description": "What is the answer to life ...etc?" }, + ], + }, + "application": { + "first_name": faker.first_name(), + "last_name": faker.last_name(), + "body_name": random.choice(BODIES_LIST), + "created_at": "yesterday or whatever", + "updated_at": "now more or less", + "aegee_experience": "I suck", + "ideal_su": "I get to suck", + "motivation": "I wanna suck", + "status": "totally snafu", + "answers": [ + "ho-hoo, ho-hoo", + "42", + ], + }, } } return email if(RANDOM_AMOUNT_TEST): - for _ in range(random.randrange(1,8)): + amount = random.randrange(MIN_MSG,MAX_MSG) + for _ in range(amount): email = generate_fake_payload() channel.basic_publish(exchange='eml', routing_key='mail', @@ -104,6 +141,7 @@ def generate_fake_payload(subj="", template=""): delivery_mode = pika.spec.PERSISTENT_DELIVERY_MODE )) print(f" [x] Sent {email['subject']} (to {email['to']})") + print(f" Gee, I sent all {amount} ") if(ALL_TEMPLATES_TEST): for ms in MAIL_TEMPLATES: diff --git a/dispatcher/templates/snippets/summeruniversity_application_info.jinja2 b/dispatcher/templates/snippets/summeruniversity_application_info.jinja2 new file mode 100644 index 00000000..7919428e --- /dev/null +++ b/dispatcher/templates/snippets/summeruniversity_application_info.jinja2 @@ -0,0 +1,15 @@ +
    +
  • Body name: {{ application["body_name"] }}
    • +
  • Applied on: {{ application["created_at"] }}
    • +
  • Updated on: {{ application["updated_at"] }}
    • +
  • First name: {{ application["first_name"] }}
    • +
  • Last name: {{ application["last_name"] }}
    • +
  • AEGEE experience: {{ application["aegee_experience"] }}
    • +
  • Ideal SU: {{ application["ideal_su"] }}
    • +
  • Motivation: {{ application["motivation"] }}
    • + + {% for question in event["questions"] %} +
  • {{ question["description"] }} {{ application["answers"][loop.index0] }}
    • + {% endfor %} + +
diff --git a/dispatcher/templates/summeruniversity_application_edited.jinja2 b/dispatcher/templates/summeruniversity_application_edited.jinja2 index 62de0d15..370987ac 100644 --- a/dispatcher/templates/summeruniversity_application_edited.jinja2 +++ b/dispatcher/templates/summeruniversity_application_edited.jinja2 @@ -45,7 +45,7 @@
Here are your application details: - <%= render(OmsmailerWeb.SnippetsView, "summeruniversity_application_info.html", parameters: @parameters ) %> + {% include "snippets/summeruniversity_application_info.jinja2" %}
diff --git a/dispatcher/templates/summeruniversity_application_status_updated.jinja2 b/dispatcher/templates/summeruniversity_application_status_updated.jinja2 new file mode 100644 index 00000000..ab054e60 --- /dev/null +++ b/dispatcher/templates/summeruniversity_application_status_updated.jinja2 @@ -0,0 +1,78 @@ + + + + +{% import "snippets/macros.jinja2" as macros %} +{{ macros.head("MyAEGEE: Updated status of your application for VAR"|replace("VAR", event["name"])) }} + + + + + + + + + + +
  + + + + + + +
+ + + + + + +
+
 
+
+ + + + + + + + + + + + +
+ AEGEE-Europe +
+
The status of your application for {{event["name"]}} was changed.
+
+
+ The status of your application for {{event["name"]}} is {{application["status"]}}. + If you have been accepted, the organizers of the Summer University will be in touch with you soon and provide you with more information. + If you have been rejected, there might be other Summer Universities with open calls. If there are any at this moment you can find them on MyAEGEE. +
+
+ + + + + + +
+
This email was autogenerated by MyAEGEE mailer.
+
+
+
+ + + + + + diff --git a/dispatcher/templates/summeruniversity_applied.jinja2 b/dispatcher/templates/summeruniversity_applied.jinja2 index b403366f..20443d0b 100644 --- a/dispatcher/templates/summeruniversity_applied.jinja2 +++ b/dispatcher/templates/summeruniversity_applied.jinja2 @@ -46,7 +46,7 @@
Here are your application details: - <%= render(OmsmailerWeb.SnippetsView, "summeruniversity_application_info.html", parameters: @parameters ) %> + {% include "snippets/summeruniversity_application_info.jinja2" %}
diff --git a/dispatcher/templates/summeruniversity_board_applied.jinja2 b/dispatcher/templates/summeruniversity_board_applied.jinja2 index e6ddeb88..91ff6ffd 100644 --- a/dispatcher/templates/summeruniversity_board_applied.jinja2 +++ b/dispatcher/templates/summeruniversity_board_applied.jinja2 @@ -45,7 +45,7 @@
Here are the application details: - <%= render(OmsmailerWeb.SnippetsView, "summeruniversity_application_info.html", parameters: @parameters ) %> + {% include "snippets/summeruniversity_application_info.jinja2" %}
diff --git a/dispatcher/templates/summeruniversity_board_edited.jinja2 b/dispatcher/templates/summeruniversity_board_edited.jinja2 index 21ddea90..7f7ec9d0 100644 --- a/dispatcher/templates/summeruniversity_board_edited.jinja2 +++ b/dispatcher/templates/summeruniversity_board_edited.jinja2 @@ -46,7 +46,7 @@
Here are the application details: - <%= render(OmsmailerWeb.SnippetsView, "summeruniversity_application_info.html", parameters: @parameters ) %> + {% include "snippets/summeruniversity_application_info.jinja2" %}
diff --git a/dispatcher/templates/summeruniversity_event_created.jinja2 b/dispatcher/templates/summeruniversity_event_created.jinja2 new file mode 100644 index 00000000..b8c4b1e9 --- /dev/null +++ b/dispatcher/templates/summeruniversity_event_created.jinja2 @@ -0,0 +1,82 @@ + + + + +{% import "snippets/macros.jinja2" as macros %} +{{ macros.head("MyAEGEE: The event has been created") }} + + + + + + + + + + +
  + + + + + + +
+ + + + + + +
+
 
+
+ + + + + + + + + + + + +
+ Logo +
+
+ The event "{{event["name"]}}" has been created. +
+
+ You are receiving this as one of the organizers. +
+
+
+ See event +
+
+ + + + + + +
+
This email was autogenerated by MyAEGEE mailer.
+
+
+
+ + + + + + diff --git a/dispatcher/templates/summeruniversity_event_updated.jinja2 b/dispatcher/templates/summeruniversity_event_updated.jinja2 new file mode 100644 index 00000000..54f25363 --- /dev/null +++ b/dispatcher/templates/summeruniversity_event_updated.jinja2 @@ -0,0 +1,82 @@ + + + + +{% import "snippets/macros.jinja2" as macros %} +{{ macros.head("MyAEGEE: The event has been updated") }} + + + + + + + + + + +
  + + + + + + +
+ + + + + + +
+
 
+
+ + + + + + + + + + + + +
+ Logo +
+
+ The event "{{event["name"]}}" has been updated. +
+
+ You are receiving this as one of the organizers. +
+
+
+ See event +
+
+ + + + + + +
+
This email was autogenerated by MyAEGEE mailer.
+
+
+
+ + + + + + diff --git a/dispatcher/templates/summeruniversity_organizer_applied.jinja2 b/dispatcher/templates/summeruniversity_organizer_applied.jinja2 index c6f9ceed..f71f247f 100644 --- a/dispatcher/templates/summeruniversity_organizer_applied.jinja2 +++ b/dispatcher/templates/summeruniversity_organizer_applied.jinja2 @@ -46,7 +46,7 @@
Here are the application details: - <%= render(OmsmailerWeb.SnippetsView, "summeruniversity_application_info.html", parameters: @parameters ) %> + {% include "snippets/summeruniversity_application_info.jinja2" %}
diff --git a/dispatcher/templates/summeruniversity_organizer_edited.jinja2 b/dispatcher/templates/summeruniversity_organizer_edited.jinja2 index ca683208..d4b2ba93 100644 --- a/dispatcher/templates/summeruniversity_organizer_edited.jinja2 +++ b/dispatcher/templates/summeruniversity_organizer_edited.jinja2 @@ -46,7 +46,7 @@
Here are the application details: - <%= render(OmsmailerWeb.SnippetsView, "summeruniversity_application_info.html", parameters: @parameters ) %> + {% include "snippets/summeruniversity_application_info.jinja2" %}
diff --git a/dispatcher/templates/summeruniversity_status_changed.jinja2 b/dispatcher/templates/summeruniversity_status_changed.jinja2 new file mode 100644 index 00000000..bc59dc67 --- /dev/null +++ b/dispatcher/templates/summeruniversity_status_changed.jinja2 @@ -0,0 +1,83 @@ + + + + +{% with title="MyAEGEE: Summer University status changed" %} + {% include "snippets/head.jinja2" %} +{% endwith %} + + + + + + + + + +
  + + + + + + +
+ + + + + + +
+
 
+
+ + + + + + + + + + + + +
+ Logo +
+
Your Summer University's status was changed
+
+ The status of your Summer University, + "{{event["name"]}}", + has changed from {{old_status}} + to {{event["status"]}}. +
+
+ +
+ + + + + + +
+
This email was autogenerated by MyAEGEE mailer.
+
+
+
+ + + + + + diff --git a/dispatcher/templates/summeruniversity_submitted.jinja2 b/dispatcher/templates/summeruniversity_submitted.jinja2 new file mode 100644 index 00000000..cec961bd --- /dev/null +++ b/dispatcher/templates/summeruniversity_submitted.jinja2 @@ -0,0 +1,82 @@ + + + + +{% with title="MyAEGEE: New event submitted" %} + {% include "snippets/head.jinja2" %} +{% endwith %} + + + + + + + + + +
  + + + + + + +
+ + + + + + +
+
 
+
+ + + + + + + + + + + + +
+ Logo +
+
A new event is submitted for {{event["status"]}}
+
+ A new event + "{{event["name"]}}", + is submitted as {{event["status"]}}. +
+
+ +
+ + + + + + +
+
This email was autogenerated by MyAEGEE mailer.
+
+
+
+ + + + + + From 415f6538a611d4cfe4f91d7698881c130f63b363 Mon Sep 17 00:00:00 2001 From: linuxbandit Date: Fri, 21 Apr 2023 11:14:52 -0400 Subject: [PATCH 14/32] chore: add events templates --- dispatcher/helpers/send.py | 19 ++++- dispatcher/templates/events_applied.jinja2 | 76 +++++++++++++++++ dispatcher/templates/events_edited.jinja2 | 76 +++++++++++++++++ .../templates/events_event_created.jinja2 | 81 ++++++++++++++++++ .../templates/events_event_updated.jinja2 | 81 ++++++++++++++++++ .../templates/events_status_changed.jinja2 | 82 +++++++++++++++++++ dispatcher/templates/events_submitted.jinja2 | 81 ++++++++++++++++++ .../snippets/events_application_info.jinja2 | 12 +++ 8 files changed, 506 insertions(+), 2 deletions(-) create mode 100644 dispatcher/templates/events_applied.jinja2 create mode 100644 dispatcher/templates/events_edited.jinja2 create mode 100644 dispatcher/templates/events_event_created.jinja2 create mode 100644 dispatcher/templates/events_event_updated.jinja2 create mode 100644 dispatcher/templates/events_status_changed.jinja2 create mode 100644 dispatcher/templates/events_submitted.jinja2 create mode 100644 dispatcher/templates/snippets/events_application_info.jinja2 diff --git a/dispatcher/helpers/send.py b/dispatcher/helpers/send.py index 86745ef2..5e7baf19 100644 --- a/dispatcher/helpers/send.py +++ b/dispatcher/helpers/send.py @@ -29,7 +29,14 @@ def su_sentence(): "NEW_JOIN_REQUEST": 'MyAEGEE: New join request for your body', "NEW_MEMBER": 'MyAEGEE: Welcome to AEGEE' }, - #"EVENTS": {}, + "EVENTS": { + "MAIL_APPLIED": f"You've successfully applied for {su_sentence()}", + "MAIL_UPDATED": f"Your application for {su_sentence()} was updated", + "MAIL_EVENT_CREATED": "The event was created", + "MAIL_EVENT_UPDATED": "The event was updated", + "MAIL_EVENT_STATUS_CHANGED": "Your event's status was changed", + "MAIL_EVENT_SUBMITTED": "An event was submitted", + }, "SUMMERUNIVERSITIES": { "MAIL_APPLIED_MEMBER": f"You've successfully applied for {su_sentence()}", "MAIL_APPLIED_ORGANISERS": f"Somebody has applied for {su_sentence()}", @@ -55,7 +62,14 @@ def su_sentence(): "NEW_JOIN_REQUEST": 'member_joined', "NEW_MEMBER": 'new_member' }, - #"EVENTS": {}, + "EVENTS": { + "MAIL_APPLIED": "events_applied", + "MAIL_UPDATED": "events_edited", + "MAIL_EVENT_CREATED": "events_event_created", + "MAIL_EVENT_UPDATED": "events_event_updated", + "MAIL_EVENT_STATUS_CHANGED": "events_status_changed", + "MAIL_EVENT_SUBMITTED": "events_submitted", + }, "SUMMERUNIVERSITIES": { "MAIL_APPLIED_MEMBER": "summeruniversity_applied", "MAIL_APPLIED_ORGANISERS": "summeruniversity_organizer_applied", @@ -120,6 +134,7 @@ def generate_fake_payload(subj="", template=""): "ideal_su": "I get to suck", "motivation": "I wanna suck", "status": "totally snafu", + "email": "someone@example.org", "answers": [ "ho-hoo, ho-hoo", "42", diff --git a/dispatcher/templates/events_applied.jinja2 b/dispatcher/templates/events_applied.jinja2 new file mode 100644 index 00000000..98a64198 --- /dev/null +++ b/dispatcher/templates/events_applied.jinja2 @@ -0,0 +1,76 @@ + + + + +{% import "snippets/macros.jinja2" as macros %} +{{ macros.head("MyAEGEE: Your application for VAR"|replace("VAR",event["name"])) }} + + + + + + + + + +
  + + + + + + +
+ + + + + + +
+
 
+
+ + + + + + + + + + + + +
+ AEGEE-Europe +
+
You've successfully applied to {{ event["name"] }}
+
+
+ Here are your application details: + {% include "snippets/summeruniversity_application_info.jinja2" %} +
+
+ + + + + + +
+
This email was autogenerated by MyAEGEE mailer.
+
+
+
+ + + + + + diff --git a/dispatcher/templates/events_edited.jinja2 b/dispatcher/templates/events_edited.jinja2 new file mode 100644 index 00000000..d78e7d5f --- /dev/null +++ b/dispatcher/templates/events_edited.jinja2 @@ -0,0 +1,76 @@ + + + + +{% import "snippets/macros.jinja2" as macros %} +{{ macros.head("MyAEGEE: Your application for VAR"|replace("VAR",event["name"])) }} + + + + + + + + + +
  + + + + + + +
+ + + + + + +
+
 
+
+ + + + + + + + + + + + +
+ AEGEE-Europe +
+
Your application for {{ event["name"] }} was successfully updated.
+
+
+ Here are your application details: + {% include "snippets/events_application_info.jinja2" %} +
+
+ + + + + + +
+
This email was autogenerated by MyAEGEE mailer.
+
+
+
+ + + + + + diff --git a/dispatcher/templates/events_event_created.jinja2 b/dispatcher/templates/events_event_created.jinja2 new file mode 100644 index 00000000..0a2def3d --- /dev/null +++ b/dispatcher/templates/events_event_created.jinja2 @@ -0,0 +1,81 @@ + + + + +{% import "snippets/macros.jinja2" as macros %} +{{ macros.head("MyAEGEE: The event has been created") }} + + + + + + + + + +
  + + + + + + +
+ + + + + + +
+
 
+
+ + + + + + + + + + + + +
+ Logo +
+
+ The event "{{ event["name"] }}" has been created. +
+
+ You are receiving this as one of the organizers. +
+
+
+ See event +
+
+ + + + + + +
+
This email was autogenerated by MyAEGEE mailer.
+
+
+
+ + + + + + diff --git a/dispatcher/templates/events_event_updated.jinja2 b/dispatcher/templates/events_event_updated.jinja2 new file mode 100644 index 00000000..0f77b5d0 --- /dev/null +++ b/dispatcher/templates/events_event_updated.jinja2 @@ -0,0 +1,81 @@ + + + + +{% import "snippets/macros.jinja2" as macros %} +{{ macros.head("MyAEGEE: The event has been updated") }} + + + + + + + + + +
  + + + + + + +
+ + + + + + +
+
 
+
+ + + + + + + + + + + + +
+ Logo +
+
+ The event "{{ event["name"] }}" has been updated. +
+
+ You are receiving this as one of the organizers. +
+
+
+ See event +
+
+ + + + + + +
+
This email was autogenerated by MyAEGEE mailer.
+
+
+
+ + + + + + diff --git a/dispatcher/templates/events_status_changed.jinja2 b/dispatcher/templates/events_status_changed.jinja2 new file mode 100644 index 00000000..2f86aa31 --- /dev/null +++ b/dispatcher/templates/events_status_changed.jinja2 @@ -0,0 +1,82 @@ + + + + +{% import "snippets/macros.jinja2" as macros %} +{{ macros.head("MyAEGEE: Event status changed") }} + + + + + + + + + +
  + + + + + + +
+ + + + + + +
+
 
+
+ + + + + + + + + + + + +
+ Logo +
+
Your event's status was changed
+
+ The status of your event, + "{{ event["name"] }}", + has changed from {{ old_status }} + to {{ event["status"] }}. +
+
+ +
+ + + + + + +
+
This email was autogenerated by MyAEGEE mailer.
+
+
+
+ + + + + + diff --git a/dispatcher/templates/events_submitted.jinja2 b/dispatcher/templates/events_submitted.jinja2 new file mode 100644 index 00000000..d402e0f6 --- /dev/null +++ b/dispatcher/templates/events_submitted.jinja2 @@ -0,0 +1,81 @@ + + + + +{% import "snippets/macros.jinja2" as macros %} +{{ macros.head("MyAEGEE: New event submitted") }} + + + + + + + + + +
  + + + + + + +
+ + + + + + +
+
 
+
+ + + + + + + + + + + + +
+ Logo +
+
A new event is submitted
+
+ A new event + "{{ event["name"] }}", + is submitted. +
+
+ +
+ + + + + + +
+
This email was autogenerated by MyAEGEE mailer.
+
+
+
+ + + + + + diff --git a/dispatcher/templates/snippets/events_application_info.jinja2 b/dispatcher/templates/snippets/events_application_info.jinja2 new file mode 100644 index 00000000..cd702dec --- /dev/null +++ b/dispatcher/templates/snippets/events_application_info.jinja2 @@ -0,0 +1,12 @@ +
    +
  • Body name: {{ application["body_name"] }}
    • +
  • Applied on: {{ application["created_at"] }}
    • +
  • Updated on: {{ application["updated_at"] }}
    • +
  • First name: {{ application["first_name"] }}
    • +
  • Last name: {{ application["last_name"] }}
    • +
  • Email: {{ application["email"] }}
    • + + {% for question in event["questions"] %} +
  • {{ question["description"] }} {{ application["answers"][loop.index0] }}
    • + {% endfor %} +
From a7b9b1f0b3ad26a311e14002e0d627414bd639d1 Mon Sep 17 00:00:00 2001 From: linuxbandit Date: Mon, 24 Apr 2023 15:47:56 -0400 Subject: [PATCH 15/32] chore: add stuff for statutory --- dispatcher/helpers/send.py | 42 +++++++++- dispatcher/templates/candidate_applied.jinja2 | 82 ++++++++++++++++++ .../statutory_application_info.jinja2 | 30 +++++++ dispatcher/templates/statutory_applied.jinja2 | 83 ++++++++++++++++++ .../templates/statutory_board_applied.jinja2 | 84 +++++++++++++++++++ .../templates/statutory_board_edited.jinja2 | 84 +++++++++++++++++++ dispatcher/templates/statutory_edited.jinja2 | 76 +++++++++++++++++ .../statutory_memberslist_edited.jinja2 | 71 ++++++++++++++++ .../statutory_memberslist_submitted.jinja2 | 71 ++++++++++++++++ 9 files changed, 621 insertions(+), 2 deletions(-) create mode 100644 dispatcher/templates/candidate_applied.jinja2 create mode 100644 dispatcher/templates/snippets/statutory_application_info.jinja2 create mode 100644 dispatcher/templates/statutory_applied.jinja2 create mode 100644 dispatcher/templates/statutory_board_applied.jinja2 create mode 100644 dispatcher/templates/statutory_board_edited.jinja2 create mode 100644 dispatcher/templates/statutory_edited.jinja2 create mode 100644 dispatcher/templates/statutory_memberslist_edited.jinja2 create mode 100644 dispatcher/templates/statutory_memberslist_submitted.jinja2 diff --git a/dispatcher/helpers/send.py b/dispatcher/helpers/send.py index 5e7baf19..f4fcd5b8 100644 --- a/dispatcher/helpers/send.py +++ b/dispatcher/helpers/send.py @@ -20,7 +20,10 @@ su_words_list = ['summer', 'awesome', 'your', 'now', 'sustainability', 'culture', 'europe', 'balkan', 'russia', 'adventure', 'ukraine', 'capital', 'montenegro', 'ireland', 'serbia', 'crimea', 'amazing', 'slavaukraini', 'heroiamslava'] def su_sentence(): return faker.sentence(nb_words=5, ext_word_list=su_words_list) -# from constants.js +def agora_sentence(): + return f"Agora {faker.sentence(nb_words=1)}" + +# from constants.js (at least for core..) MAIL_SUBJECTS = { "CORE": { "MAIL_CONFIRMATION": 'MyAEGEE: Please confirm your account', @@ -37,6 +40,15 @@ def su_sentence(): "MAIL_EVENT_STATUS_CHANGED": "Your event's status was changed", "MAIL_EVENT_SUBMITTED": "An event was submitted", }, + "STATUTORY": { + "MAIL_APPLIED": f"You've successfully applied for {agora_sentence()}", + "MAIL_UPDATED": f"Your application for {agora_sentence()} was updated", + "MAIL_APPLIED_BOARD": f"One of your body members has applied to {agora_sentence()}", + "MAIL_UPDATED_BOARD": f"One of your body members has updated their application to {agora_sentence()}", + "MAIL_CANDIDATE_APPLIED": "A new application was submitted", + "MAIL_MEMBERSLIST_SUBMITTED": "The event was updated", + "MAIL_MEMBERSLIST_EDITED": f"A memberslist has been edited for {agora_sentence()}", + }, "SUMMERUNIVERSITIES": { "MAIL_APPLIED_MEMBER": f"You've successfully applied for {su_sentence()}", "MAIL_APPLIED_ORGANISERS": f"Somebody has applied for {su_sentence()}", @@ -70,6 +82,15 @@ def su_sentence(): "MAIL_EVENT_STATUS_CHANGED": "events_status_changed", "MAIL_EVENT_SUBMITTED": "events_submitted", }, + "STATUTORY": { + "MAIL_APPLIED": "statutory_applied", + "MAIL_UPDATED": "statutory_edited", + "MAIL_APPLIED_BOARD": "statutory_board_applied", + "MAIL_UPDATED_BOARD": "statutory_board_edited", + "MAIL_CANDIDATE_APPLIED": "candidate_applied", + "MAIL_MEMBERSLIST_SUBMITTED": "statutory_memberslist_submitted", + "MAIL_MEMBERSLIST_EDITED": "statutory_memberslist_edited", + }, "SUMMERUNIVERSITIES": { "MAIL_APPLIED_MEMBER": "summeruniversity_applied", "MAIL_APPLIED_ORGANISERS": "summeruniversity_organizer_applied", @@ -85,7 +106,7 @@ def su_sentence(): }, } -RABBIT_HOST='172.18.0.8' #FIXME +RABBIT_HOST='172.18.0.5' #FIXME connection = pika.BlockingConnection(pika.ConnectionParameters(RABBIT_HOST)) channel = connection.channel() @@ -114,6 +135,8 @@ def generate_fake_payload(subj="", template=""): "place": faker.city(), "token": faker.md5(), "old_status": "snafu", + "event_name": agora_sentence(), + "membership_fee": "15⎊", "event": { "name": su_sentence(), "location": faker.city(), @@ -135,11 +158,26 @@ def generate_fake_payload(subj="", template=""): "motivation": "I wanna suck", "status": "totally snafu", "email": "someone@example.org", + "gender": "Robot", + "date_of_birth": "1985-04-16", + "nationality": "Europe", + "meals": "Steak", + "allergies": "wallopers", + "number_of_events_visited": "over 9000", + "visa_required": "No", "answers": [ "ho-hoo, ho-hoo", "42", ], }, + "candidate": { + "first_name": faker.first_name(), + "last_name": faker.last_name(), + }, + "position": { + "event_id": "42", + "name": "This gran C commissioner", + }, } } return email diff --git a/dispatcher/templates/candidate_applied.jinja2 b/dispatcher/templates/candidate_applied.jinja2 new file mode 100644 index 00000000..e2c92d07 --- /dev/null +++ b/dispatcher/templates/candidate_applied.jinja2 @@ -0,0 +1,82 @@ + + + + +{% import "snippets/macros.jinja2" as macros %} +{{ macros.head("MyAEGEE: New candidature submitted") }} + + + + + + + + + +
  + + + + + + +
+ + + + + + +
+
 
+
+ + + + + + + + + + + + +
+ Logo +
+
A new candidature is submitted
+
+ A new candidature to position + {{ position["name"] }} + is submitted by + {{ candidate["first_name"] }} {{candidate["last_name"] }}. +
+
+ +
+ + + + + + +
+
This email was autogenerated by MyAEGEE mailer.
+
+
+
+ + + + + + diff --git a/dispatcher/templates/snippets/statutory_application_info.jinja2 b/dispatcher/templates/snippets/statutory_application_info.jinja2 new file mode 100644 index 00000000..57dae5ef --- /dev/null +++ b/dispatcher/templates/snippets/statutory_application_info.jinja2 @@ -0,0 +1,30 @@ +
    +
  • Body name: {{ application["body_name"] }}
    • +
  • Applied on: {{ application["created_at"] }}
    • +
  • Updated on: {{ application["updated_at"] }}
    • +
  • First name: {{ application["first_name"] }}
    • +
  • Last name: {{ application["last_name"] }}
    • +
  • Gender: {{ application["gender"] }}
    • +
  • Date of birth: {{ application["date_of_birth"] }}
    • +
  • Email: {{ application["email"] }}
    • +
  • Nationality: {{ application["nationality"] }}
    • +
  • Meals: {{ application["meals"] }}
    • +
  • Allergies: {{ application["allergies"] }}
    • +
  • Number of events visited: {{ application["number_of_events_visited"] }}
    • +
  • Visa required?: {{ application["visa_required"] }}
    • +
  • Visa passport number: {{ application["visa_passport_number"] }}
    • +
  • Visa passport issue date: {{ application["visa_passport_issue_date"] }}
    • +
  • Visa passport expiration date: {{ application["visa_passport_expiration_date"] }}
    • +
  • Visa passport issue authority: {{ application["visa_passport_issue_authority"] }}
    • +
  • Place of birth: {{ application["visa_place_of_birth"] }}
    • +
  • Street and house number: {{ application["visa_street_and_house"] }}
    • +
  • Postal code: {{ application["visa_postal_code"] }}
    • +
  • City: {{ application["visa_city"] }}
    • +
  • Country: {{ application["visa_country"] }}
    • +
  • Embassy: {{ application["visa_embassy"] }}
    • + + {% for question in event["questions"] %} +
  • {{ question["description"] }} {{ application["answers"][loop.index0] }}
    • + {% endfor %} + +
diff --git a/dispatcher/templates/statutory_applied.jinja2 b/dispatcher/templates/statutory_applied.jinja2 new file mode 100644 index 00000000..9c754e9c --- /dev/null +++ b/dispatcher/templates/statutory_applied.jinja2 @@ -0,0 +1,83 @@ + + + + +{% import "snippets/macros.jinja2" as macros %} +{{ macros.head("MyAEGEE: Your application for VAR"|replace("VAR", event["name"])) }} + + + + + + + + + +
  + + + + + + +
+ + + + + + +
+
 
+
+ + + + + + + + + + + + + + + +
+ AEGEE-Europe +
+
You've successfully applied to {{ event["name"] }}
+
+
+ Here are your application details: + {% include "snippets/statutory_application_info.jinja2" %} +
+
+
+ Have you already subscribed to our internal communication channels? Don’t miss out on any AEGEE news or announcements, subscribe to our mailing lists! +
+
+ + + + + + +
+
This email was autogenerated by MyAEGEE mailer.
+
+
+
+ + + + + + diff --git a/dispatcher/templates/statutory_board_applied.jinja2 b/dispatcher/templates/statutory_board_applied.jinja2 new file mode 100644 index 00000000..095ac010 --- /dev/null +++ b/dispatcher/templates/statutory_board_applied.jinja2 @@ -0,0 +1,84 @@ + + + + +{% import "snippets/macros.jinja2" as macros %} +{{ macros.head("MyAEGEE: A member of your body applied for VAR"|replace("VAR", event["name"])) }} + + + + + + + + + +
  + + + + + + +
+ + + + + + +
+
 
+
+ + + + + + + + + + + + + + + +
+ +
+
A member of your body ({{ application["body_name"] }}) applied for {{ event["name"] }}
+
+
+ Here are the application details: + {% include "snippets/statutory_application_info.jinja2" %} +
+
+ +
+ + + + + + +
+
This email was autogenerated by MyAEGEE mailer.
+
+
+
+ + + + + + diff --git a/dispatcher/templates/statutory_board_edited.jinja2 b/dispatcher/templates/statutory_board_edited.jinja2 new file mode 100644 index 00000000..12312b98 --- /dev/null +++ b/dispatcher/templates/statutory_board_edited.jinja2 @@ -0,0 +1,84 @@ + + + + +{% import "snippets/macros.jinja2" as macros %} +{{ macros.head("MyAEGEE: A member of your body edited the application for VAR"|replace("VAR", event["name"])) }} + + + + + + + + + +
  + + + + + + +
+ + + + + + +
+
 
+
+ + + + + + + + + + + + + + + +
+ +
+
A member of your body ({{ application["body_name"] }}) edited the application for {{ event["name"] }}
+
+
+ Here are the application details: + {% include "snippets/statutory_application_info.jinja2" %} +
+
+ +
+ + + + + + +
+
This email was autogenerated by MyAEGEE mailer.
+
+
+
+ + + + + + diff --git a/dispatcher/templates/statutory_edited.jinja2 b/dispatcher/templates/statutory_edited.jinja2 new file mode 100644 index 00000000..31fbf47b --- /dev/null +++ b/dispatcher/templates/statutory_edited.jinja2 @@ -0,0 +1,76 @@ + + + + +{% import "snippets/macros.jinja2" as macros %} +{{ macros.head("MyAEGEE: Your application for VAR"|replace("VAR", event["name"])) }} + + + + + + + + + +
  + + + + + + +
+ + + + + + +
+
 
+
+ + + + + + + + + + + + +
+ AEGEE-Europe +
+
Your application for {{ event["name"] }} was successfully updated.
+
+
+ Here are your application details: + {% include "snippets/statutory_application_info.jinja2" %} +
+
+ + + + + + +
+
This email was autogenerated by MyAEGEE mailer.
+
+
+
+ + + + + + diff --git a/dispatcher/templates/statutory_memberslist_edited.jinja2 b/dispatcher/templates/statutory_memberslist_edited.jinja2 new file mode 100644 index 00000000..0ab859d1 --- /dev/null +++ b/dispatcher/templates/statutory_memberslist_edited.jinja2 @@ -0,0 +1,71 @@ + + + + +{% import "snippets/macros.jinja2" as macros %} +{{ macros.head("MyAEGEE: A memberslist has been edited for VAR"|replace("VAR", event_name)) }} + + + + + + + + + +
  + + + + + + +
+ + + + + + +
+
 
+
+ + + + + + + + + +
+ Logo +
+
+ "{{ body_name }}" has edited their memberslist for "{{ event_name }}". + Their new membership fee is "{{ membership_fee }}". +
+
+ + + + + + +
+
This email was autogenerated by MyAEGEE mailer.
+
+
+
+ + + + + + diff --git a/dispatcher/templates/statutory_memberslist_submitted.jinja2 b/dispatcher/templates/statutory_memberslist_submitted.jinja2 new file mode 100644 index 00000000..7dfd0d3e --- /dev/null +++ b/dispatcher/templates/statutory_memberslist_submitted.jinja2 @@ -0,0 +1,71 @@ + + + + +{% import "snippets/macros.jinja2" as macros %} +{{ macros.head("MyAEGEE: A memberslist has been submitted for VAR"|replace("VAR", event_name)) }} + + + + + + + + + +
  + + + + + + +
+ + + + + + +
+
 
+
+ + + + + + + + + +
+ Logo +
+
+ "{{ body_name }}" has submitted their memberslist for "{{ event_name }}". + Their membership fee is "{{ membership_fee }}". +
+
+ + + + + + +
+
This email was autogenerated by MyAEGEE mailer.
+
+
+
+ + + + + + From 74ee32eebcd5ae1947740516fb457d4922d8ec66 Mon Sep 17 00:00:00 2001 From: linuxbandit Date: Tue, 25 Apr 2023 10:22:52 -0400 Subject: [PATCH 16/32] chore: add last templates --- dispatcher/helpers/send.py | 26 +++++++ dispatcher/templates/custom.jinja2 | 2 + dispatcher/templates/member_expired.jinja2 | 6 +- dispatcher/templates/network_new_board.jinja2 | 76 +++++++++++++++++++ dispatcher/templates/new_member.jinja2 | 5 +- .../templates/snippets/board_info.jinja2 | 12 +++ dispatcher/templates/snippets/head.jinja2 | 9 --- .../summeruniversity_status_changed.jinja2 | 5 +- .../summeruniversity_submitted.jinja2 | 5 +- 9 files changed, 124 insertions(+), 22 deletions(-) create mode 100644 dispatcher/templates/custom.jinja2 create mode 100644 dispatcher/templates/network_new_board.jinja2 create mode 100644 dispatcher/templates/snippets/board_info.jinja2 delete mode 100644 dispatcher/templates/snippets/head.jinja2 diff --git a/dispatcher/helpers/send.py b/dispatcher/helpers/send.py index f4fcd5b8..3392ed0c 100644 --- a/dispatcher/helpers/send.py +++ b/dispatcher/helpers/send.py @@ -62,6 +62,13 @@ def agora_sentence(): "MAIL_SU_STATUS_CHANGED": "Your event's status was changed", "MAIL_SU_SUBMITTED": "An event was submitted", }, + "NETWORK": { + "MAIL_NEW_BOARD": f'A new board was added for { random.choice(BODIES_LIST) }', + }, + "OTHER": { + "MAIL_EXPIRED_MEMBERSHIP": 'member_expired', + "MAIL_CUSTOM": 'custom', + }, } # should exist in constants.js but it does not yet. # anyway here one could #TODO a smarter way: look into the filesystem @@ -104,6 +111,13 @@ def agora_sentence(): "MAIL_SU_STATUS_CHANGED": "summeruniversity_status_changed", "MAIL_SU_SUBMITTED": "summeruniversity_submitted", }, + "NETWORK": { + "MAIL_NEW_BOARD": 'network_new_board', + }, + "OTHER": { + "MAIL_EXPIRED_MEMBERSHIP": 'membership_expired', + "MAIL_CUSTOM": 'custom', + }, } RABBIT_HOST='172.18.0.5' #FIXME @@ -137,6 +151,12 @@ def generate_fake_payload(subj="", template=""): "old_status": "snafu", "event_name": agora_sentence(), "membership_fee": "15⎊", + "board": { + "elected_date": "yesteeeerday", + "start_date": "tomooorrow", + "end_date": "next weeeeek", + "message": "chattanooga", + }, "event": { "name": su_sentence(), "location": faker.city(), @@ -178,6 +198,12 @@ def generate_fake_payload(subj="", template=""): "event_id": "42", "name": "This gran C commissioner", }, + "body": """ +
    +
  • Who is cool: Accountable people
    • +
  • Who is not: People hiding behind the excuse of 'volunteer'
    • +
+ """ } } return email diff --git a/dispatcher/templates/custom.jinja2 b/dispatcher/templates/custom.jinja2 new file mode 100644 index 00000000..1e2dccc9 --- /dev/null +++ b/dispatcher/templates/custom.jinja2 @@ -0,0 +1,2 @@ + +{{ body }} diff --git a/dispatcher/templates/member_expired.jinja2 b/dispatcher/templates/member_expired.jinja2 index fd5d80dd..cfcb80f0 100644 --- a/dispatcher/templates/member_expired.jinja2 +++ b/dispatcher/templates/member_expired.jinja2 @@ -2,10 +2,8 @@ -{% with title="MyAEGEE: Membership expired" %} - {% include "snippets/head.jinja2" %} -{% endwith %} - +{% import "snippets/macros.jinja2" as macros %} +{{ macros.head("MyAEGEE: Membership expired") }} + + +{% import "snippets/macros.jinja2" as macros %} +{{ macros.head("MyAEGEE: New board added for VAR"|replace("VAR",body_name)) }} + + +
+ + + + + + +
  + + + + + + +
+ + + + + + +
+
 
+
+ + + + + + + + + + + + +
+ AEGEE-Europe +
+
A new board has successfully been added for {{ body_name }}
+
+
+ Here are their details: + {% include "snippets/board_info.jinja2" %} +
+
+ + + + + + +
+
This email was autogenerated by MyAEGEE mailer.
+
+
+
+ + + + + + diff --git a/dispatcher/templates/new_member.jinja2 b/dispatcher/templates/new_member.jinja2 index c6e14564..63842d3a 100644 --- a/dispatcher/templates/new_member.jinja2 +++ b/dispatcher/templates/new_member.jinja2 @@ -2,9 +2,8 @@ -{% with title="MyAEGEE: Welcome to AEGEE" %} - {% include "snippets/head.jinja2" %} -{% endwith %} +{% import "snippets/macros.jinja2" as macros %} +{{ macros.head("MyAEGEE: Welcome to AEGEE") }} +
  • Body name: {{ body_name }}
    • +
  • Elected date: {{ board["elected_date"] }}
    • +
  • Start date: {{ board["start_date"] }}
    • +
  • End date: {{ board["end_date"] }}
    • + + {% for position in positions %} +
  • {{ position["function"] }} {{ position["name"] }}
    • + {% endfor %} + +
  • Message: {{ board["message"] }}
    • + diff --git a/dispatcher/templates/snippets/head.jinja2 b/dispatcher/templates/snippets/head.jinja2 deleted file mode 100644 index b8367653..00000000 --- a/dispatcher/templates/snippets/head.jinja2 +++ /dev/null @@ -1,9 +0,0 @@ - - - - {{ title }} - - - - {% include "snippets/css.jinja2" %} - diff --git a/dispatcher/templates/summeruniversity_status_changed.jinja2 b/dispatcher/templates/summeruniversity_status_changed.jinja2 index bc59dc67..78e08924 100644 --- a/dispatcher/templates/summeruniversity_status_changed.jinja2 +++ b/dispatcher/templates/summeruniversity_status_changed.jinja2 @@ -2,9 +2,8 @@ -{% with title="MyAEGEE: Summer University status changed" %} - {% include "snippets/head.jinja2" %} -{% endwith %} +{% import "snippets/macros.jinja2" as macros %} +{{ macros.head("MyAEGEE: Summer University status changed") }}
    -{% with title="MyAEGEE: New event submitted" %} - {% include "snippets/head.jinja2" %} -{% endwith %} +{% import "snippets/macros.jinja2" as macros %} +{{ macros.head("MyAEGEE: New event submitted") }}
    Date: Tue, 25 Apr 2023 10:24:51 -0400 Subject: [PATCH 17/32] chore: add checker for thorough testing --- dispatcher/helpers/send.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/dispatcher/helpers/send.py b/dispatcher/helpers/send.py index 3392ed0c..f32e0435 100644 --- a/dispatcher/helpers/send.py +++ b/dispatcher/helpers/send.py @@ -1,6 +1,7 @@ import pika import json import random +import os """ creates between 1 and 8 fake emails and puts in the queue @@ -71,8 +72,9 @@ def agora_sentence(): }, } # should exist in constants.js but it does not yet. -# anyway here one could #TODO a smarter way: look into the filesystem -# but then you miss the correspondence between the subject and the template +# anyway here one could make it in a smarter way: look into the filesystem +# but then you miss the correspondence between the subject and the template. +# So we only look in the fs to cross-check if we were thorough MAIL_TEMPLATES = { "CORE": { "MAIL_CONFIRMATION": 'confirm_email', @@ -120,7 +122,7 @@ def agora_sentence(): }, } -RABBIT_HOST='172.18.0.5' #FIXME +RABBIT_HOST='172.18.0.2' #FIXME connection = pika.BlockingConnection(pika.ConnectionParameters(RABBIT_HOST)) channel = connection.channel() @@ -223,6 +225,7 @@ def generate_fake_payload(subj="", template=""): print(f" Gee, I sent all {amount} ") if(ALL_TEMPLATES_TEST): + templates_tested=0 for ms in MAIL_TEMPLATES: for case in MAIL_TEMPLATES[ms]: email = generate_fake_payload(MAIL_SUBJECTS[ms][case], MAIL_TEMPLATES[ms][case]) @@ -233,5 +236,11 @@ def generate_fake_payload(subj="", template=""): delivery_mode = pika.spec.PERSISTENT_DELIVERY_MODE )) print(f" [x] Sent {email['subject']}") + templates_tested+=1 + file_count = len([f for f in os.listdir("../templates/") if os.path.isfile(os.path.join("../templates/", f))]) + if(file_count != templates_tested): + print() + print() + print(f"Tested {templates_tested} templates but the folder contains {file_count}") connection.close() From de2f6fcb6e75da1e6eb7871b56c6e138a71b759c Mon Sep 17 00:00:00 2001 From: linuxbandit Date: Tue, 25 Apr 2023 10:26:26 -0400 Subject: [PATCH 18/32] docs: add dockerisation --- dispatcher/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dispatcher/README.md b/dispatcher/README.md index cc94331a..a937022a 100644 --- a/dispatcher/README.md +++ b/dispatcher/README.md @@ -16,7 +16,7 @@ In a console, run `python3 dispatcher/main.py`. In another console, run `python3 helpers/send.py`. Control on `appserver.test:8025` the emails sent. It is also possible to control rabbit's stats on `appserver.test:8080` -This method is not dockerised. Currently unsure about adding the docker way. +This method is now dockerised. Using the docker way is useful for the 'DNS' feature of docker (i.e. not hardcoding the IP address of the rabbit host) ### Rationale We do not need a web service for this, only a worker. Doing it this way only means it cannot be scaled (unless precautions are taken for the ack of the message, but pika should already give this out of the box). From 1d63fda6087f1fabbb2ca2a1ed79ec074d513d84 Mon Sep 17 00:00:00 2001 From: linuxbandit Date: Tue, 25 Apr 2023 10:29:08 -0400 Subject: [PATCH 19/32] chore: forgot another update --- dispatcher/helpers/send.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dispatcher/helpers/send.py b/dispatcher/helpers/send.py index f32e0435..30146880 100644 --- a/dispatcher/helpers/send.py +++ b/dispatcher/helpers/send.py @@ -117,7 +117,7 @@ def agora_sentence(): "MAIL_NEW_BOARD": 'network_new_board', }, "OTHER": { - "MAIL_EXPIRED_MEMBERSHIP": 'membership_expired', + "MAIL_EXPIRED_MEMBERSHIP": 'member_expired', "MAIL_CUSTOM": 'custom', }, } From c43582216c7db95ae304637a6cc7aa4b443c9289 Mon Sep 17 00:00:00 2001 From: linuxbandit Date: Tue, 25 Apr 2023 13:18:27 -0400 Subject: [PATCH 20/32] chore: add quickstart --- dispatcher/Makefile | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 dispatcher/Makefile diff --git a/dispatcher/Makefile b/dispatcher/Makefile new file mode 100644 index 00000000..b8fcb3bf --- /dev/null +++ b/dispatcher/Makefile @@ -0,0 +1,14 @@ + +default: build-dev + +build-dev: + BASE_URL=appserver.vgr SUBDOMAIN_RABBITMQ=rabbit. SUBDOMAIN_MAILHOG=mailhog. PATH_DISPATCHER=./ COMPOSE_PROJECT_NAME=myaegee docker-compose -f ./docker/docker-compose.yml -f ./docker/docker-compose.dev.yml up -d + +build: + BASE_URL=appserver.vgr SUBDOMAIN_RABBITMQ=rabbit. COMPOSE_PROJECT_NAME=myaegee docker-compose -f ./docker/docker-compose.yml up -d + +down: + COMPOSE_PROJECT_NAME=myaegee docker-compose -f ./docker/docker-compose.yml -f ./docker/docker-compose.dev.yml down + +test: + python helpers/send.py From f35b2479553f44fa710dca90593da47132cd82ad Mon Sep 17 00:00:00 2001 From: linuxbandit Date: Tue, 25 Apr 2023 13:31:51 -0400 Subject: [PATCH 21/32] fix: correct make test --- dispatcher/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dispatcher/Makefile b/dispatcher/Makefile index b8fcb3bf..c86bc677 100644 --- a/dispatcher/Makefile +++ b/dispatcher/Makefile @@ -11,4 +11,4 @@ down: COMPOSE_PROJECT_NAME=myaegee docker-compose -f ./docker/docker-compose.yml -f ./docker/docker-compose.dev.yml down test: - python helpers/send.py + @sed -i "s/'172.18.0.2'/$$(docker inspect myaegee_rabbit_1 | jq .[0].NetworkSettings.Networks.OMS.IPAddress)/" helpers/send.py ; cd helpers && python send.py From 26a277b534608b56243d4dbb15beaa88b2822683 Mon Sep 17 00:00:00 2001 From: linuxbandit Date: Tue, 25 Apr 2023 14:10:30 -0400 Subject: [PATCH 22/32] chore: add extra checks for errors during render --- dispatcher/dispatcher/main.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/dispatcher/dispatcher/main.py b/dispatcher/dispatcher/main.py index ba44d811..33fd0c1d 100755 --- a/dispatcher/dispatcher/main.py +++ b/dispatcher/dispatcher/main.py @@ -33,11 +33,17 @@ def send_email(ch, method, properties, body): template = environment.get_template(f"{msg['template']}.jinja2") except exceptions.TemplateNotFound: # TODO: send a notification to someone about adding a template - print(f"template {msg['template']}.jinja2 not found") + print(f"Template {msg['template']}.jinja2 not found") return # TODO: check if there is an auto-delete after 30 minutes for stuck un-acked messages - rendered = template.render(msg['parameters'], altro=msg['subject']) - + try: + rendered = template.render(msg['parameters'], altro=msg['subject']) + except exceptions.UndefinedError: + print(f"Error in rendering: some parameter is undefined") + return + except exceptions.TemplateNotFound: + print(f"A sub-template in {msg['template']}.jinja2 was not found") + return email = EmailMessage() email.set_content(rendered, subtype='html') email['From'] = msg['from'] From 262580b58e1ec809938e9ca1e192589581f1c327 Mon Sep 17 00:00:00 2001 From: linuxbandit Date: Fri, 7 Jul 2023 12:26:03 +0300 Subject: [PATCH 23/32] feat: add requeue with a delay --- dispatcher/Makefile | 7 +- dispatcher/dispatcher/main.py | 134 ++++++++++++++++++++++++++- dispatcher/docker/Dockerfile.rabbit | 6 ++ dispatcher/docker/docker-compose.yml | 2 +- dispatcher/helpers/send.py | 10 +- 5 files changed, 150 insertions(+), 9 deletions(-) create mode 100644 dispatcher/docker/Dockerfile.rabbit diff --git a/dispatcher/Makefile b/dispatcher/Makefile index c86bc677..f90658cf 100644 --- a/dispatcher/Makefile +++ b/dispatcher/Makefile @@ -10,5 +10,8 @@ build: down: COMPOSE_PROJECT_NAME=myaegee docker-compose -f ./docker/docker-compose.yml -f ./docker/docker-compose.dev.yml down -test: - @sed -i "s/'172.18.0.2'/$$(docker inspect myaegee_rabbit_1 | jq .[0].NetworkSettings.Networks.OMS.IPAddress)/" helpers/send.py ; cd helpers && python send.py +test: build-dev + @sleep 5; sed -i "s/'172.18.0.2'/$$(docker inspect myaegee_rabbit_1 | jq .[0].NetworkSettings.Networks.OMS.IPAddress)/" helpers/send.py ; cd helpers && python send.py + +rabbit: + @DOCKER_BUILDKIT=0 docker build -t aegee/rabbit:latest -f docker/Dockerfile.rabbit . && docker push aegee/rabbit diff --git a/dispatcher/dispatcher/main.py b/dispatcher/dispatcher/main.py index 33fd0c1d..eef382ef 100755 --- a/dispatcher/dispatcher/main.py +++ b/dispatcher/dispatcher/main.py @@ -21,29 +21,86 @@ channel.exchange_declare(exchange='eml', exchange_type='direct', durable=True) -channel.queue_declare(queue='email', durable=True) +channel.queue_declare(queue='email', + arguments={ + 'x-dead-letter-exchange': "dead_letter_exchange", + 'x-dead-letter-routing-key': "dead_letterl_routing_key", + 'x-death-header': True, + }, + durable=True) channel.queue_bind(exchange='eml', queue='email', routing_key='mail') + #channel.basic_qos(prefetch_count=1) #TODO: notice that an error processing a message will BLOCK the others from being processed +channel.exchange_declare(exchange="dead_letter_exchange", + exchange_type='direct', + durable=True) +channel.queue_declare(queue='error_queue', + durable=True) +channel.queue_bind(exchange='dead_letter_exchange', + queue='error_queue', + routing_key='dead_letterl_routing_key') + +channel.exchange_declare(exchange="wait_exchange", + exchange_type='x-delayed-message', + durable=True, + arguments={"x-delayed-type": "direct"} + ) +channel.queue_declare(queue='requeue_queue', + durable=True) +channel.queue_bind(exchange='wait_exchange', + queue='requeue_queue', + routing_key='wait') + def send_email(ch, method, properties, body): + """ + Callback for the NORMAL MESSAGE + Output: send an email + OR + Output: Dead-letter queue + """ msg = json.loads(body) + try: template = environment.get_template(f"{msg['template']}.jinja2") except exceptions.TemplateNotFound: # TODO: send a notification to someone about adding a template + # NOTE: this is a requeuable message print(f"Template {msg['template']}.jinja2 not found") + ch.basic_nack(delivery_tag=method.delivery_tag, requeue=False) return - # TODO: check if there is an auto-delete after 30 minutes for stuck un-acked messages + try: rendered = template.render(msg['parameters'], altro=msg['subject']) - except exceptions.UndefinedError: - print(f"Error in rendering: some parameter is undefined") + except exceptions.UndefinedError as e: + # NOTE: this is a NON-requeuable message + print(f"Error in rendering: some parameter is undefined (error: {e}; message: {msg})") +# TODO: check if there is no x-header about DLQ if +# i send directly to queue from this (i.e. without passing from DLQ), + # headers = { + # 'reason': 'parameter_undefined', + # 'x-delay': '60000', + # } + # prop = pika.BasicProperties( + # headers=headers, + # delivery_mode = pika.spec.PERSISTENT_DELIVERY_MODE, + # ) + # channel.basic_publish(exchange='wait_exchange', + # routing_key='wait', + # body=body, + # properties=prop) + # ch.basic_ack(delivery_tag=method.delivery_tag) + + ch.basic_nack(delivery_tag=method.delivery_tag, requeue=False) return except exceptions.TemplateNotFound: + # NOTE: this is a requeuable message print(f"A sub-template in {msg['template']}.jinja2 was not found") + ch.basic_nack(delivery_tag=method.delivery_tag, requeue=False) return + email = EmailMessage() email.set_content(rendered, subtype='html') email['From'] = msg['from'] @@ -53,10 +110,79 @@ def send_email(ch, method, properties, body): smtpObj.send_message(email) #TODO handle case in which smtp not ready ch.basic_ack(delivery_tag = method.delivery_tag) +def process_dead_letter_messages(ch, method, properties, body): + """ + Callback for the ERROR MESSAGE + Output: Requeue on delayed exchange (if error about missing template) + OR + Output: Remove (if error about unretrievable data) + #TODO I can't know which is the case, with the current design + """ + REQUEUE_DELAY_DURATIONS = [ + 5 * 60000, # 5 mins + 50 * 60000, # 50 mins + 5*60 * 60000, # 5 hrs + 5*60*10 * 60000, # 50 hrs + 5*60*20 * 60000, # 100 hrs + ] + wait_for = REQUEUE_DELAY_DURATIONS[4] + wait_for = '15000' + + print(f'DLQ') + print(properties.headers) + print(f'Message was in "{properties.headers["x-first-death-exchange"]}" (specifically "{properties.headers["x-first-death-queue"]}" queue) and was rejected because: {properties.headers["x-first-death-reason"]}') + print() + + headers = { + #'reason': 'parameter_undefined', + 'x-delay': wait_for, + } + fullheaders = {**properties.headers, **headers} + prop = pika.BasicProperties( + headers=fullheaders, + delivery_mode = pika.spec.PERSISTENT_DELIVERY_MODE, + ) + channel.basic_publish(exchange='wait_exchange', + routing_key='wait', + body=body, + properties=prop) + + ch.basic_ack(delivery_tag = method.delivery_tag) + +def process_requeue(ch, method, properties, body): + """ + Callback for the WAITING MESSAGES + Output: Requeue on normal exchange (if error about missing template) + OR + Output: Remove (if limit time reached) #TODO + """ + + print(f'REQUEUE-WAIT') + print(properties.headers) + print(f'Message was in "{properties.headers["x-first-death-exchange"]}" (specifically "{properties.headers["x-first-death-queue"]}" queue) and was rejected because: {properties.headers["x-first-death-reason"]}') + print() + + channel.basic_publish(exchange='eml', + routing_key='mail', + body=body, + properties=pika.BasicProperties( + delivery_mode = pika.spec.PERSISTENT_DELIVERY_MODE, + )) + ch.basic_ack(delivery_tag = method.delivery_tag) + channel.basic_consume(queue='email', auto_ack=False, on_message_callback=send_email) +channel.basic_consume(queue='error_queue', + auto_ack=False, + on_message_callback=process_dead_letter_messages) + +channel.basic_consume(queue='requeue_queue', + auto_ack=False, + on_message_callback=process_requeue) + + print(' [*] Waiting for messages. To exit press CTRL+C') channel.start_consuming() diff --git a/dispatcher/docker/Dockerfile.rabbit b/dispatcher/docker/Dockerfile.rabbit new file mode 100644 index 00000000..7901d4d1 --- /dev/null +++ b/dispatcher/docker/Dockerfile.rabbit @@ -0,0 +1,6 @@ +FROM rabbitmq:3.11-management + +ADD https://github.com/rabbitmq/rabbitmq-delayed-message-exchange/releases/download/3.11.1/rabbitmq_delayed_message_exchange-3.11.1.ez /opt/rabbitmq/plugins +RUN chown rabbitmq:rabbitmq /opt/rabbitmq/plugins/rabbitmq_delayed_message_exchange-3.11.1.ez \ + && chmod 755 /opt/rabbitmq/plugins/rabbitmq_delayed_message_exchange-3.11.1.ez \ + && rabbitmq-plugins enable rabbitmq_delayed_message_exchange --offline diff --git a/dispatcher/docker/docker-compose.yml b/dispatcher/docker/docker-compose.yml index 1acc1067..4cd3fb00 100644 --- a/dispatcher/docker/docker-compose.yml +++ b/dispatcher/docker/docker-compose.yml @@ -3,7 +3,7 @@ version: "3.4" services: rabbit: - image: rabbitmq:3.11-management + image: aegee/rabbit restart: always expose: - 5672 diff --git a/dispatcher/helpers/send.py b/dispatcher/helpers/send.py index 30146880..08bb85ee 100644 --- a/dispatcher/helpers/send.py +++ b/dispatcher/helpers/send.py @@ -122,14 +122,20 @@ def agora_sentence(): }, } -RABBIT_HOST='172.18.0.2' #FIXME +RABBIT_HOST="172.18.0.2" #FIXME connection = pika.BlockingConnection(pika.ConnectionParameters(RABBIT_HOST)) channel = connection.channel() channel.exchange_declare(exchange='eml', exchange_type='direct', durable=True) -channel.queue_declare(queue='email', durable=True) +channel.queue_declare(queue='email', + arguments={ + 'x-dead-letter-exchange': "dead_letter_exchange", + 'x-dead-letter-routing-key': "dead_letterl_routing_key", + 'x-death-header': True, + }, + durable=True) channel.queue_bind(exchange='eml', queue='email', routing_key='mail') From a174fa9569750f532fe26fbba40a30777defa991 Mon Sep 17 00:00:00 2001 From: linuxbandit Date: Wed, 2 Aug 2023 18:40:25 +0300 Subject: [PATCH 24/32] chore: correct template, add comments --- dispatcher/dispatcher/main.py | 12 ++++++------ .../templates/summeruniversity_submitted.jinja2 | 6 ++++++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/dispatcher/dispatcher/main.py b/dispatcher/dispatcher/main.py index eef382ef..db055b2f 100755 --- a/dispatcher/dispatcher/main.py +++ b/dispatcher/dispatcher/main.py @@ -24,7 +24,7 @@ channel.queue_declare(queue='email', arguments={ 'x-dead-letter-exchange': "dead_letter_exchange", - 'x-dead-letter-routing-key': "dead_letterl_routing_key", + 'x-dead-letter-routing-key': "dead_letter_routing_key", 'x-death-header': True, }, durable=True) @@ -32,7 +32,7 @@ queue='email', routing_key='mail') -#channel.basic_qos(prefetch_count=1) #TODO: notice that an error processing a message will BLOCK the others from being processed +#channel.basic_qos(prefetch_count=1) #TODO: notice that with this enabled, an error processing a message will BLOCK the others from being processed channel.exchange_declare(exchange="dead_letter_exchange", exchange_type='direct', @@ -41,7 +41,7 @@ durable=True) channel.queue_bind(exchange='dead_letter_exchange', queue='error_queue', - routing_key='dead_letterl_routing_key') + routing_key='dead_letter_routing_key') channel.exchange_declare(exchange="wait_exchange", exchange_type='x-delayed-message', @@ -79,6 +79,7 @@ def send_email(ch, method, properties, body): print(f"Error in rendering: some parameter is undefined (error: {e}; message: {msg})") # TODO: check if there is no x-header about DLQ if # i send directly to queue from this (i.e. without passing from DLQ), + # headers = { # 'reason': 'parameter_undefined', # 'x-delay': '60000', @@ -125,8 +126,7 @@ def process_dead_letter_messages(ch, method, properties, body): 5*60*10 * 60000, # 50 hrs 5*60*20 * 60000, # 100 hrs ] - wait_for = REQUEUE_DELAY_DURATIONS[4] - wait_for = '15000' + wait_for = REQUEUE_DELAY_DURATIONS[4] # TODO make it dynamic print(f'DLQ') print(properties.headers) @@ -188,7 +188,7 @@ def process_requeue(ch, method, properties, body): if __name__ == '__main__': try: - main() + main() #fixme lol except KeyboardInterrupt: print('Interrupted') smtpObj.quit() diff --git a/dispatcher/templates/summeruniversity_submitted.jinja2 b/dispatcher/templates/summeruniversity_submitted.jinja2 index 9ea7dca6..add2288f 100644 --- a/dispatcher/templates/summeruniversity_submitted.jinja2 +++ b/dispatcher/templates/summeruniversity_submitted.jinja2 @@ -2,6 +2,12 @@ +{# +{% with title="MyAEGEE: New event submitted" %} + {% include "snippets/head.jinja2" %} +{% endwith %} +#} + {% import "snippets/macros.jinja2" as macros %} {{ macros.head("MyAEGEE: New event submitted") }} From c0da07b7a0f58e9e5117e47766d647f000a80389 Mon Sep 17 00:00:00 2001 From: linuxbandit Date: Wed, 2 Aug 2023 19:10:56 +0300 Subject: [PATCH 25/32] feat: requeue with dynamic delay --- dispatcher/Makefile | 2 +- dispatcher/README.md | 12 ++++ dispatcher/dispatcher/main.py | 91 ++++++++++++++---------- dispatcher/docker/docker-compose.dev.yml | 6 ++ dispatcher/docker/docker-compose.yml | 2 +- dispatcher/helpers/send.py | 32 +++++++-- 6 files changed, 102 insertions(+), 43 deletions(-) diff --git a/dispatcher/Makefile b/dispatcher/Makefile index f90658cf..a834fbc3 100644 --- a/dispatcher/Makefile +++ b/dispatcher/Makefile @@ -11,7 +11,7 @@ down: COMPOSE_PROJECT_NAME=myaegee docker-compose -f ./docker/docker-compose.yml -f ./docker/docker-compose.dev.yml down test: build-dev - @sleep 5; sed -i "s/'172.18.0.2'/$$(docker inspect myaegee_rabbit_1 | jq .[0].NetworkSettings.Networks.OMS.IPAddress)/" helpers/send.py ; cd helpers && python send.py + @sleep 5; sed -i "s/'172.18.0.X'/$$(docker inspect myaegee_rabbit_1 | jq .[0].NetworkSettings.Networks.OMS.IPAddress)/" helpers/send.py ; cd helpers && python send.py rabbit: @DOCKER_BUILDKIT=0 docker build -t aegee/rabbit:latest -f docker/Dockerfile.rabbit . && docker push aegee/rabbit diff --git a/dispatcher/README.md b/dispatcher/README.md index a937022a..8c56bf73 100644 --- a/dispatcher/README.md +++ b/dispatcher/README.md @@ -44,3 +44,15 @@ rather in order: 1. [x] When RabbitMQ quits or crashes it will forget the queues and messages unless you tell it not to: we need to mark both the queue and messages as durable 1. [ ] Add auto-retry (DLQ). rabbit is smart and doesn't let me process a message again unless i force it.. https://devcorner.digitalpress.blog/rabbitmq-retries-the-new-full-story/ 1. [ ] add the telegram queue +1. investigate the massmailer queue: a queue which picks every message, and creates a list of "bcc" to send only one email? (danger: queue needs something like batch ack..) - OR it is not feasible at all because "mass"mailer is still "personalised" mailer? + +1. why do we even have a `<`title`>` (which is dynamic), why not using directly the subject? (re: the body of the email) +1. remove extension Jinja2 (into jinja) +1. make it such that templates list is read from fs (for dynamic tests) + + + +https://www.rabbitmq.com/publishers.html#unroutable + + +Each consumer (subscription) has an identifier called a consumer tag. It can be used to unsubscribe from messages. Consumer tags are just strings. diff --git a/dispatcher/dispatcher/main.py b/dispatcher/dispatcher/main.py index db055b2f..2ab7be36 100755 --- a/dispatcher/dispatcher/main.py +++ b/dispatcher/dispatcher/main.py @@ -54,12 +54,48 @@ queue='requeue_queue', routing_key='wait') +def requeue_wait(ch, method, properties, body, reason): + REQUEUE_DELAY_DURATIONS = [ + 5 * 60000, # 5 mins + 50 * 60000, # 50 mins + 5*60 * 60000, # 5 hrs + 5*60*10 * 60000, # 50 hrs + 5*60*20 * 60000, # 100 hrs + ] + + if (properties.headers != None and "x-delay" in properties.headers): + index = REQUEUE_DELAY_DURATIONS.index(int(properties.headers["x-delay"])) + if (index+1 == len(REQUEUE_DELAY_DURATIONS) ): + print('Max retry time hit, dropping message') + # TODO: notify someone that they've been sloppy + ch.basic_ack(delivery_tag = method.delivery_tag) + return + else: + print(f'Attempt {index+1}/{len(REQUEUE_DELAY_DURATIONS)-1}') + wait = REQUEUE_DELAY_DURATIONS[index+1] + else: + wait = REQUEUE_DELAY_DURATIONS[0] + + headers = { + 'reason': reason, + 'x-delay': wait, + } + prop = pika.BasicProperties( + headers=headers, + delivery_mode = pika.spec.PERSISTENT_DELIVERY_MODE, + ) + channel.basic_publish(exchange='wait_exchange', + routing_key='wait', + body=body, + properties=prop) #NOTE it completely ignores the previous properties (and it's fine) + ch.basic_ack(delivery_tag=method.delivery_tag) + def send_email(ch, method, properties, body): """ Callback for the NORMAL MESSAGE Output: send an email OR - Output: Dead-letter queue + Output: Wait-exchange """ msg = json.loads(body) @@ -69,7 +105,7 @@ def send_email(ch, method, properties, body): # TODO: send a notification to someone about adding a template # NOTE: this is a requeuable message print(f"Template {msg['template']}.jinja2 not found") - ch.basic_nack(delivery_tag=method.delivery_tag, requeue=False) + requeue_wait(ch, method, properties, body, reason="template_not_found") return try: @@ -77,29 +113,12 @@ def send_email(ch, method, properties, body): except exceptions.UndefinedError as e: # NOTE: this is a NON-requeuable message print(f"Error in rendering: some parameter is undefined (error: {e}; message: {msg})") -# TODO: check if there is no x-header about DLQ if -# i send directly to queue from this (i.e. without passing from DLQ), - - # headers = { - # 'reason': 'parameter_undefined', - # 'x-delay': '60000', - # } - # prop = pika.BasicProperties( - # headers=headers, - # delivery_mode = pika.spec.PERSISTENT_DELIVERY_MODE, - # ) - # channel.basic_publish(exchange='wait_exchange', - # routing_key='wait', - # body=body, - # properties=prop) - # ch.basic_ack(delivery_tag=method.delivery_tag) - - ch.basic_nack(delivery_tag=method.delivery_tag, requeue=False) + requeue_wait(ch, method, properties, body, reason="parameter_undefined") return except exceptions.TemplateNotFound: # NOTE: this is a requeuable message print(f"A sub-template in {msg['template']}.jinja2 was not found") - ch.basic_nack(delivery_tag=method.delivery_tag, requeue=False) + requeue_wait(ch, method, properties, body, reason="sub-template_not_found") return email = EmailMessage() @@ -114,10 +133,12 @@ def send_email(ch, method, properties, body): def process_dead_letter_messages(ch, method, properties, body): """ Callback for the ERROR MESSAGE - Output: Requeue on delayed exchange (if error about missing template) - OR - Output: Remove (if error about unretrievable data) - #TODO I can't know which is the case, with the current design + Output: none yet. I don't expect for messages to fall here, I keep the DLQ for safety + + @see https://stackoverflow.com/a/58500336 + "The way to do this is not to use NACK at all but to generate and return a 'new' message + (which is simply the current message you are handling, but adding new headers to it). + It appears that a NACK is basically doing this anyway according to the AMQP spec." """ REQUEUE_DELAY_DURATIONS = [ 5 * 60000, # 5 mins @@ -126,15 +147,9 @@ def process_dead_letter_messages(ch, method, properties, body): 5*60*10 * 60000, # 50 hrs 5*60*20 * 60000, # 100 hrs ] - wait_for = REQUEUE_DELAY_DURATIONS[4] # TODO make it dynamic - - print(f'DLQ') - print(properties.headers) - print(f'Message was in "{properties.headers["x-first-death-exchange"]}" (specifically "{properties.headers["x-first-death-queue"]}" queue) and was rejected because: {properties.headers["x-first-death-reason"]}') - print() + wait_for = REQUEUE_DELAY_DURATIONS[-1] headers = { - #'reason': 'parameter_undefined', 'x-delay': wait_for, } fullheaders = {**properties.headers, **headers} @@ -154,18 +169,20 @@ def process_requeue(ch, method, properties, body): Callback for the WAITING MESSAGES Output: Requeue on normal exchange (if error about missing template) OR - Output: Remove (if limit time reached) #TODO + Output: Remove (if unfixable error) """ - print(f'REQUEUE-WAIT') - print(properties.headers) - print(f'Message was in "{properties.headers["x-first-death-exchange"]}" (specifically "{properties.headers["x-first-death-queue"]}" queue) and was rejected because: {properties.headers["x-first-death-reason"]}') - print() + if (properties.headers["reason"] == 'parameter_undefined'): + print('Impossible to fix error, dropping message') + #TODO output something/notify to leave a trail for better debugging on what was missing + ch.basic_ack(delivery_tag = method.delivery_tag) + return channel.basic_publish(exchange='eml', routing_key='mail', body=body, properties=pika.BasicProperties( + headers = properties.headers, # propagation to avoid endless loop delivery_mode = pika.spec.PERSISTENT_DELIVERY_MODE, )) ch.basic_ack(delivery_tag = method.delivery_tag) diff --git a/dispatcher/docker/docker-compose.dev.yml b/dispatcher/docker/docker-compose.dev.yml index 6ad4d22d..af8f6075 100644 --- a/dispatcher/docker/docker-compose.dev.yml +++ b/dispatcher/docker/docker-compose.dev.yml @@ -2,6 +2,12 @@ version: "3.4" services: + rabbit: + build: + context: ./${PATH_DISPATCHER}/.. + dockerfile: ./docker/Dockerfile.rabbit + image: aegee/rabbit:3.11-mgmt-delayxch + dispatcher: build: context: ./${PATH_DISPATCHER}/.. diff --git a/dispatcher/docker/docker-compose.yml b/dispatcher/docker/docker-compose.yml index 4cd3fb00..921bce53 100644 --- a/dispatcher/docker/docker-compose.yml +++ b/dispatcher/docker/docker-compose.yml @@ -3,7 +3,7 @@ version: "3.4" services: rabbit: - image: aegee/rabbit + image: aegee/rabbit:3.11-mgmt-delayxch restart: always expose: - 5672 diff --git a/dispatcher/helpers/send.py b/dispatcher/helpers/send.py index 08bb85ee..5092a2e0 100644 --- a/dispatcher/helpers/send.py +++ b/dispatcher/helpers/send.py @@ -12,6 +12,7 @@ from faker import Faker faker = Faker() +ERROR_TEST=True RANDOM_AMOUNT_TEST=False MIN_MSG=1 MAX_MSG=8 @@ -122,7 +123,7 @@ def agora_sentence(): }, } -RABBIT_HOST="172.18.0.2" #FIXME +RABBIT_HOST="172.18.0.X" #FIXME (as this is a python script launched on host we cant use docker's dns) connection = pika.BlockingConnection(pika.ConnectionParameters(RABBIT_HOST)) channel = connection.channel() @@ -132,7 +133,7 @@ def agora_sentence(): channel.queue_declare(queue='email', arguments={ 'x-dead-letter-exchange': "dead_letter_exchange", - 'x-dead-letter-routing-key': "dead_letterl_routing_key", + 'x-dead-letter-routing-key': "dead_letter_routing_key", 'x-death-header': True, }, durable=True) @@ -140,7 +141,7 @@ def agora_sentence(): queue='email', routing_key='mail') -def generate_fake_payload(subj="", template=""): +def generate_fake_payload(subj="", template="", return_malformed_mail=False): email = { "from": "mailer@aegee.eu", "to": [faker.email() for _ in range(random.randrange(1,3))], @@ -214,6 +215,9 @@ def generate_fake_payload(subj="", template=""): """ } } + if(return_malformed_mail): + email["template"] = MAIL_TEMPLATES["EVENTS"]["MAIL_EVENT_UPDATED"] + del email["parameters"]["event"] return email @@ -228,7 +232,27 @@ def generate_fake_payload(subj="", template=""): delivery_mode = pika.spec.PERSISTENT_DELIVERY_MODE )) print(f" [x] Sent {email['subject']} (to {email['to']})") - print(f" Gee, I sent all {amount} ") + print(f" Gee, I sent all {amount} ") + +if(ERROR_TEST): + #ERROR 1: no template + email = generate_fake_payload(template="notexisting", subj="will never be delivered") + channel.basic_publish(exchange='eml', + routing_key='mail', + body=json.dumps(email), + properties=pika.BasicProperties( + delivery_mode = pika.spec.PERSISTENT_DELIVERY_MODE + )) + print(f" [x] Sent {email['subject']} (to {email['to']})") + #ERROR 2: missing field + email = generate_fake_payload(return_malformed_mail=True, subj="will never be delivered") + channel.basic_publish(exchange='eml', + routing_key='mail', + body=json.dumps(email), + properties=pika.BasicProperties( + delivery_mode = pika.spec.PERSISTENT_DELIVERY_MODE + )) + print(f" [x] Sent {email['subject']} (to {email['to']})") if(ALL_TEMPLATES_TEST): templates_tested=0 From bdf6d32984489d95dd6cf484a79351f104ac8a9f Mon Sep 17 00:00:00 2001 From: linuxbandit Date: Sat, 5 Aug 2023 11:15:04 +0300 Subject: [PATCH 26/32] chore: make USER non-root --- dispatcher/docker/Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dispatcher/docker/Dockerfile b/dispatcher/docker/Dockerfile index 0c94e39b..b5c3de01 100644 --- a/dispatcher/docker/Dockerfile +++ b/dispatcher/docker/Dockerfile @@ -8,9 +8,8 @@ RUN pip install --no-cache-dir -r requirements.txt COPY ./templates /usr/app/templates COPY ./dispatcher/main.py /usr/app/src/main.py -#USER python - -#ENV FOO=BAR +RUN useradd -s /bin/bash python +USER python ENTRYPOINT [ "python", "-u" ] CMD ["main.py"] From fcc5950d973cebf44d1461f2b5d58d1a11fe3eff Mon Sep 17 00:00:00 2001 From: linuxbandit Date: Wed, 20 Sep 2023 19:47:10 +0300 Subject: [PATCH 27/32] chore: differentiate between environments --- .env.example | 5 ++ dispatcher/dispatcher/main.py | 69 ++++++++++++++++++++++------ dispatcher/docker/docker-compose.yml | 11 ++++- 3 files changed, 70 insertions(+), 15 deletions(-) mode change 100755 => 100644 dispatcher/dispatcher/main.py diff --git a/.env.example b/.env.example index 9b990d00..7705ffe8 100644 --- a/.env.example +++ b/.env.example @@ -35,6 +35,11 @@ SENDGRID_SMTP_HOST=smtp.sendgrid.net SENDGRID_USER= SENDGRID_PW=5ecr3t +ETHEREAL_SMTP_HOST=smtp.ethereal.email +ETHEREAL_PORT=587 +ETHEREAL_USER= +ETHEREAL_PW= + ELASTIC_HOST=https://example.com:9243 ELASTIC_USER=5ecr3t ELASTIC_PASSWORD=5ecr3t diff --git a/dispatcher/dispatcher/main.py b/dispatcher/dispatcher/main.py old mode 100755 new mode 100644 index 2ab7be36..655d2fc5 --- a/dispatcher/dispatcher/main.py +++ b/dispatcher/dispatcher/main.py @@ -3,16 +3,48 @@ import smtplib from email.message import EmailMessage from jinja2 import Environment, FileSystemLoader, exceptions +import os +import sys """ continuously polls(*) the email queue and renders+sends the template on every acked message (*) = waits for the queue to push a message onto the app """ -environment = Environment(loader=FileSystemLoader("../templates/")) +tpl_environment = Environment(loader=FileSystemLoader("../templates/")) +env = os.environ.get("ENV") or 'development' + +EMAIL_HOST='mailhog' +EMAIL_PORT=1025 +EMAIL_ADDRESS=None +EMAIL_PASSWORD=None + + +if env == 'production': + EMAIL_HOST= os.environ.get("EMAIL_HOST") + EMAIL_PORT= os.environ.get("EMAIL_PORT") + EMAIL_ADDRESS= os.environ.get("EMAIL_ADDRESS") + EMAIL_PASSWORD= os.environ.get("EMAIL_PASSWORD") + +smtpObj = None + +def connect_to_smtp(): + global smtpObj + try: + smtpObj = smtplib.SMTP( EMAIL_HOST, EMAIL_PORT ) + + if env == 'production': + # we have to upgrade the connection and login + smtpObj.starttls() + smtpObj.login(EMAIL_ADDRESS, EMAIL_PASSWORD) + print(" -> Connected") + except smtplib.SMTPConnectError: + print("Could not connect to the SMTP server.") + except smtplib.SMTPAuthenticationError: + print("Failed to authenticate with given credentials.") + except Exception as e: + print(f"Could not connect to SMTP server for generic reason: {e}") -EMAIL_HOST='mailhog' #FIXME differentiate between dev and prod -smtpObj = smtplib.SMTP( EMAIL_HOST, 1025 ) RABBIT_HOST='rabbit' connection = pika.BlockingConnection(pika.ConnectionParameters(RABBIT_HOST)) @@ -100,7 +132,7 @@ def send_email(ch, method, properties, body): msg = json.loads(body) try: - template = environment.get_template(f"{msg['template']}.jinja2") + template = tpl_environment.get_template(f"{msg['template']}.jinja2") except exceptions.TemplateNotFound: # TODO: send a notification to someone about adding a template # NOTE: this is a requeuable message @@ -120,15 +152,23 @@ def send_email(ch, method, properties, body): print(f"A sub-template in {msg['template']}.jinja2 was not found") requeue_wait(ch, method, properties, body, reason="sub-template_not_found") return - - email = EmailMessage() - email.set_content(rendered, subtype='html') - email['From'] = msg['from'] - email['Reply-To'] = msg['reply_to'] - email['To'] = msg['to'] - email['Subject'] = msg['subject'] - smtpObj.send_message(email) #TODO handle case in which smtp not ready - ch.basic_ack(delivery_tag = method.delivery_tag) + try: + email = EmailMessage() + email.set_content(rendered, subtype='html') + email['From'] = msg['from'] + email['Reply-To'] = msg['reply_to'] + email['To'] = msg['to'] + email['Subject'] = msg['subject'] + smtpObj.send_message(email) + ch.basic_ack(delivery_tag = method.delivery_tag) + except smtplib.SMTPServerDisconnected: + print("Server unexpectedly disconnected. Attempting to reconnect") + connect_to_smtp() + except smtplib.SMTPResponseException as e: + print(f"SMTP error occurred: {e.smtp_code} - {e.smtp_error}") + except Exception as e: + print(f"An unexpected error occurred: {e}") + # TODO add logging (in info mode) def process_dead_letter_messages(ch, method, properties, body): """ @@ -199,7 +239,8 @@ def process_requeue(ch, method, properties, body): auto_ack=False, on_message_callback=process_requeue) - +print(' [*] Connecting to smtp') +connect_to_smtp() print(' [*] Waiting for messages. To exit press CTRL+C') channel.start_consuming() diff --git a/dispatcher/docker/docker-compose.yml b/dispatcher/docker/docker-compose.yml index 921bce53..a06d3173 100644 --- a/dispatcher/docker/docker-compose.yml +++ b/dispatcher/docker/docker-compose.yml @@ -2,12 +2,15 @@ version: "3.4" services: - rabbit: + rabbit: # login with guest:guest image: aegee/rabbit:3.11-mgmt-delayxch restart: always expose: - 5672 - 15672 + # environment: + # RABBITMQ_DEFAULT_USER: "admin" + # RABBITMQ_DEFAULT_PASS: "5ecr3t" labels: - "traefik.frontend.rule=Host:${SUBDOMAIN_RABBITMQ}${BASE_URL};PathPrefix:/" - "traefik.backend=rabbit" @@ -18,6 +21,12 @@ services: dispatcher: image: aegee/dispatcher restart: always + environment: + ENV: "${MYAEGEE_ENV}" + EMAIL_HOST: "${ETHEREAL_SMTP_HOST}" + EMAIL_PORT: "${ETHEREAL_PORT}" + EMAIL_ADDRESS: "${ETHEREAL_USER}" + EMAIL_PASSWORD: "${ETHEREAL_PW}" networks: From 7a905206f14ada8c9c82647aec00df2e682a6397 Mon Sep 17 00:00:00 2001 From: linuxbandit Date: Wed, 20 Sep 2023 19:53:30 +0300 Subject: [PATCH 28/32] chore: add everything into a main to be modular --- dispatcher/dispatcher/main.py | 155 ++++++++++++++++++---------------- 1 file changed, 80 insertions(+), 75 deletions(-) diff --git a/dispatcher/dispatcher/main.py b/dispatcher/dispatcher/main.py index 655d2fc5..d7252660 100644 --- a/dispatcher/dispatcher/main.py +++ b/dispatcher/dispatcher/main.py @@ -11,30 +11,26 @@ (*) = waits for the queue to push a message onto the app """ -tpl_environment = Environment(loader=FileSystemLoader("../templates/")) -env = os.environ.get("ENV") or 'development' - -EMAIL_HOST='mailhog' -EMAIL_PORT=1025 -EMAIL_ADDRESS=None -EMAIL_PASSWORD=None - +def connect_to_smtp(): + global smtpObj -if env == 'production': - EMAIL_HOST= os.environ.get("EMAIL_HOST") - EMAIL_PORT= os.environ.get("EMAIL_PORT") - EMAIL_ADDRESS= os.environ.get("EMAIL_ADDRESS") - EMAIL_PASSWORD= os.environ.get("EMAIL_PASSWORD") + EMAIL_HOST='mailhog' + EMAIL_PORT=1025 + EMAIL_ADDRESS=None + EMAIL_PASSWORD=None -smtpObj = None + if env == 'production': + EMAIL_HOST= os.environ.get("EMAIL_HOST") + EMAIL_PORT= os.environ.get("EMAIL_PORT") + EMAIL_ADDRESS= os.environ.get("EMAIL_ADDRESS") + EMAIL_PASSWORD= os.environ.get("EMAIL_PASSWORD") -def connect_to_smtp(): - global smtpObj try: smtpObj = smtplib.SMTP( EMAIL_HOST, EMAIL_PORT ) if env == 'production': # we have to upgrade the connection and login + # (At least with ethereal.email.. didn't try with gmail!) #TODO smtpObj.starttls() smtpObj.login(EMAIL_ADDRESS, EMAIL_PASSWORD) print(" -> Connected") @@ -45,47 +41,6 @@ def connect_to_smtp(): except Exception as e: print(f"Could not connect to SMTP server for generic reason: {e}") - -RABBIT_HOST='rabbit' -connection = pika.BlockingConnection(pika.ConnectionParameters(RABBIT_HOST)) -channel = connection.channel() - -channel.exchange_declare(exchange='eml', - exchange_type='direct', - durable=True) -channel.queue_declare(queue='email', - arguments={ - 'x-dead-letter-exchange': "dead_letter_exchange", - 'x-dead-letter-routing-key': "dead_letter_routing_key", - 'x-death-header': True, - }, - durable=True) -channel.queue_bind(exchange='eml', - queue='email', - routing_key='mail') - -#channel.basic_qos(prefetch_count=1) #TODO: notice that with this enabled, an error processing a message will BLOCK the others from being processed - -channel.exchange_declare(exchange="dead_letter_exchange", - exchange_type='direct', - durable=True) -channel.queue_declare(queue='error_queue', - durable=True) -channel.queue_bind(exchange='dead_letter_exchange', - queue='error_queue', - routing_key='dead_letter_routing_key') - -channel.exchange_declare(exchange="wait_exchange", - exchange_type='x-delayed-message', - durable=True, - arguments={"x-delayed-type": "direct"} - ) -channel.queue_declare(queue='requeue_queue', - durable=True) -channel.queue_bind(exchange='wait_exchange', - queue='requeue_queue', - routing_key='wait') - def requeue_wait(ch, method, properties, body, reason): REQUEUE_DELAY_DURATIONS = [ 5 * 60000, # 5 mins @@ -152,6 +107,7 @@ def send_email(ch, method, properties, body): print(f"A sub-template in {msg['template']}.jinja2 was not found") requeue_wait(ch, method, properties, body, reason="sub-template_not_found") return + try: email = EmailMessage() email.set_content(rendered, subtype='html') @@ -186,7 +142,7 @@ def process_dead_letter_messages(ch, method, properties, body): 5*60 * 60000, # 5 hrs 5*60*10 * 60000, # 50 hrs 5*60*20 * 60000, # 100 hrs - ] + ] #TODO: why is this here again? wait_for = REQUEUE_DELAY_DURATIONS[-1] headers = { @@ -227,26 +183,75 @@ def process_requeue(ch, method, properties, body): )) ch.basic_ack(delivery_tag = method.delivery_tag) -channel.basic_consume(queue='email', - auto_ack=False, - on_message_callback=send_email) - -channel.basic_consume(queue='error_queue', - auto_ack=False, - on_message_callback=process_dead_letter_messages) - -channel.basic_consume(queue='requeue_queue', - auto_ack=False, - on_message_callback=process_requeue) - -print(' [*] Connecting to smtp') -connect_to_smtp() -print(' [*] Waiting for messages. To exit press CTRL+C') -channel.start_consuming() +def main(): + global smtpObj + global tpl_environment + global env + global channel + + tpl_environment = Environment(loader=FileSystemLoader("../templates/")) + env = os.environ.get("ENV") or 'development' + + RABBIT_HOST='rabbit' + connection = pika.BlockingConnection(pika.ConnectionParameters(RABBIT_HOST)) + channel = connection.channel() + + channel.exchange_declare(exchange='eml', + exchange_type='direct', + durable=True) + channel.queue_declare(queue='email', + arguments={ + 'x-dead-letter-exchange': "dead_letter_exchange", + 'x-dead-letter-routing-key': "dead_letter_routing_key", + 'x-death-header': True, + }, + durable=True) + channel.queue_bind(exchange='eml', + queue='email', + routing_key='mail') + + #channel.basic_qos(prefetch_count=1) #TODO: notice that with this enabled, an error processing a message will BLOCK the others from being processed + + channel.exchange_declare(exchange="dead_letter_exchange", + exchange_type='direct', + durable=True) + channel.queue_declare(queue='error_queue', + durable=True) + channel.queue_bind(exchange='dead_letter_exchange', + queue='error_queue', + routing_key='dead_letter_routing_key') + + channel.exchange_declare(exchange="wait_exchange", + exchange_type='x-delayed-message', + durable=True, + arguments={"x-delayed-type": "direct"} + ) + channel.queue_declare(queue='requeue_queue', + durable=True) + channel.queue_bind(exchange='wait_exchange', + queue='requeue_queue', + routing_key='wait') + + channel.basic_consume(queue='email', + auto_ack=False, + on_message_callback=send_email) + + channel.basic_consume(queue='error_queue', + auto_ack=False, + on_message_callback=process_dead_letter_messages) + + channel.basic_consume(queue='requeue_queue', + auto_ack=False, + on_message_callback=process_requeue) + + print(' [*] Connecting to smtp') + connect_to_smtp() + print(' [*] Waiting for messages. To exit press CTRL+C') + channel.start_consuming() if __name__ == '__main__': try: - main() #fixme lol + main() except KeyboardInterrupt: print('Interrupted') smtpObj.quit() From 526bcd8b948b9fb3a747e56dbc3256c4527d8aee Mon Sep 17 00:00:00 2001 From: linuxbandit Date: Wed, 20 Sep 2023 20:19:18 +0300 Subject: [PATCH 29/32] chore: add loglevels --- dispatcher/dispatcher/main.py | 38 +++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/dispatcher/dispatcher/main.py b/dispatcher/dispatcher/main.py index d7252660..77008afc 100644 --- a/dispatcher/dispatcher/main.py +++ b/dispatcher/dispatcher/main.py @@ -5,6 +5,7 @@ from jinja2 import Environment, FileSystemLoader, exceptions import os import sys +import logging """ continuously polls(*) the email queue and renders+sends the template on every acked message @@ -33,13 +34,13 @@ def connect_to_smtp(): # (At least with ethereal.email.. didn't try with gmail!) #TODO smtpObj.starttls() smtpObj.login(EMAIL_ADDRESS, EMAIL_PASSWORD) - print(" -> Connected") + logging.info(" -> Connected") except smtplib.SMTPConnectError: - print("Could not connect to the SMTP server.") + logging.error("Could not connect to the SMTP server.") except smtplib.SMTPAuthenticationError: - print("Failed to authenticate with given credentials.") + logging.error("Failed to authenticate with given credentials.") except Exception as e: - print(f"Could not connect to SMTP server for generic reason: {e}") + logging.error(f"Could not connect to SMTP server for generic reason: {e}") def requeue_wait(ch, method, properties, body, reason): REQUEUE_DELAY_DURATIONS = [ @@ -53,12 +54,12 @@ def requeue_wait(ch, method, properties, body, reason): if (properties.headers != None and "x-delay" in properties.headers): index = REQUEUE_DELAY_DURATIONS.index(int(properties.headers["x-delay"])) if (index+1 == len(REQUEUE_DELAY_DURATIONS) ): - print('Max retry time hit, dropping message') + logging.warn('Max retry time hit, dropping message') # TODO: notify someone that they've been sloppy ch.basic_ack(delivery_tag = method.delivery_tag) return else: - print(f'Attempt {index+1}/{len(REQUEUE_DELAY_DURATIONS)-1}') + logging.info(f'Attempt {index+1}/{len(REQUEUE_DELAY_DURATIONS)-1}') wait = REQUEUE_DELAY_DURATIONS[index+1] else: wait = REQUEUE_DELAY_DURATIONS[0] @@ -91,7 +92,7 @@ def send_email(ch, method, properties, body): except exceptions.TemplateNotFound: # TODO: send a notification to someone about adding a template # NOTE: this is a requeuable message - print(f"Template {msg['template']}.jinja2 not found") + logging.error(f"Template {msg['template']}.jinja2 not found") requeue_wait(ch, method, properties, body, reason="template_not_found") return @@ -99,12 +100,12 @@ def send_email(ch, method, properties, body): rendered = template.render(msg['parameters'], altro=msg['subject']) except exceptions.UndefinedError as e: # NOTE: this is a NON-requeuable message - print(f"Error in rendering: some parameter is undefined (error: {e}; message: {msg})") + logging.error(f"Error in rendering: some parameter is undefined (error: {e}; message: {msg})") requeue_wait(ch, method, properties, body, reason="parameter_undefined") return except exceptions.TemplateNotFound: # NOTE: this is a requeuable message - print(f"A sub-template in {msg['template']}.jinja2 was not found") + logging.error(f"A sub-template in {msg['template']}.jinja2 was not found") requeue_wait(ch, method, properties, body, reason="sub-template_not_found") return @@ -118,13 +119,12 @@ def send_email(ch, method, properties, body): smtpObj.send_message(email) ch.basic_ack(delivery_tag = method.delivery_tag) except smtplib.SMTPServerDisconnected: - print("Server unexpectedly disconnected. Attempting to reconnect") + logging.error("Server unexpectedly disconnected. Attempting to reconnect") connect_to_smtp() except smtplib.SMTPResponseException as e: - print(f"SMTP error occurred: {e.smtp_code} - {e.smtp_error}") + logging.error(f"SMTP error occurred: {e.smtp_code} - {e.smtp_error}") except Exception as e: - print(f"An unexpected error occurred: {e}") - # TODO add logging (in info mode) + logging.error(f"An unexpected error occurred: {e}") def process_dead_letter_messages(ch, method, properties, body): """ @@ -169,7 +169,7 @@ def process_requeue(ch, method, properties, body): """ if (properties.headers["reason"] == 'parameter_undefined'): - print('Impossible to fix error, dropping message') + logging.warn('Impossible to fix error, dropping message') #TODO output something/notify to leave a trail for better debugging on what was missing ch.basic_ack(delivery_tag = method.delivery_tag) return @@ -189,6 +189,10 @@ def main(): global env global channel + # Configure logging for this app, and remove "info" of pika + logging.basicConfig(level=logging.INFO) + logging.getLogger('pika').setLevel(logging.WARNING) + tpl_environment = Environment(loader=FileSystemLoader("../templates/")) env = os.environ.get("ENV") or 'development' @@ -244,16 +248,16 @@ def main(): auto_ack=False, on_message_callback=process_requeue) - print(' [*] Connecting to smtp') + logging.info(' [*] Connecting to smtp') connect_to_smtp() - print(' [*] Waiting for messages. To exit press CTRL+C') + logging.info(' [*] Waiting for messages. To exit press CTRL+C') channel.start_consuming() if __name__ == '__main__': try: main() except KeyboardInterrupt: - print('Interrupted') + logging.error('Interrupted') smtpObj.quit() try: sys.exit(0) From c405a67f0e83387af81e29fbc519c40ef5de951f Mon Sep 17 00:00:00 2001 From: linuxbandit Date: Thu, 21 Sep 2023 15:55:12 +0300 Subject: [PATCH 30/32] chore: increase clarity of incremental retry block --- dispatcher/dispatcher/main.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/dispatcher/dispatcher/main.py b/dispatcher/dispatcher/main.py index 77008afc..7e1ce914 100644 --- a/dispatcher/dispatcher/main.py +++ b/dispatcher/dispatcher/main.py @@ -51,18 +51,24 @@ def requeue_wait(ch, method, properties, body, reason): 5*60*20 * 60000, # 100 hrs ] - if (properties.headers != None and "x-delay" in properties.headers): - index = REQUEUE_DELAY_DURATIONS.index(int(properties.headers["x-delay"])) - if (index+1 == len(REQUEUE_DELAY_DURATIONS) ): - logging.warn('Max retry time hit, dropping message') - # TODO: notify someone that they've been sloppy - ch.basic_ack(delivery_tag = method.delivery_tag) - return - else: - logging.info(f'Attempt {index+1}/{len(REQUEUE_DELAY_DURATIONS)-1}') - wait = REQUEUE_DELAY_DURATIONS[index+1] - else: - wait = REQUEUE_DELAY_DURATIONS[0] + current_delay = properties.headers.get("x-delay") if properties.headers else 0 + try: + index = REQUEUE_DELAY_DURATIONS.index(int(current_delay)) + except ValueError: + index = -1 + + next_index = index + 1 + + if next_index >= len(REQUEUE_DELAY_DURATIONS): + logging.warning('Max retry time hit, dropping message') + # TODO: notify someone that they've been sloppy + ch.basic_ack(delivery_tag=method.delivery_tag) + return + + wait = REQUEUE_DELAY_DURATIONS[next_index] + logging.info(f'Retry attempt {next_index + 1}/{len(REQUEUE_DELAY_DURATIONS)} in {int(wait/1000)} sec') + if next_index + 1 == len(REQUEUE_DELAY_DURATIONS): + logging.error(f'LAST ATTEMPT TO FIX: within {int(wait/1000)} sec') headers = { 'reason': reason, From 083b1ade6c014e6f4ecb78559c1708b674f7968f Mon Sep 17 00:00:00 2001 From: linuxbandit Date: Thu, 21 Sep 2023 17:10:23 +0300 Subject: [PATCH 31/32] feat: add notification system had to downgrade python of one minor version to make it work with the simple library slack_notifications --- dispatcher/dispatcher/main.py | 31 ++- dispatcher/dispatcher/notify.py | 65 ++++++ dispatcher/docker/Dockerfile | 2 +- dispatcher/docker/docker-compose.yml | 3 + dispatcher/poetry.lock | 296 +++++++++++++++++++++------ dispatcher/pyproject.toml | 3 +- dispatcher/requirements.txt | 1 + 7 files changed, 322 insertions(+), 79 deletions(-) create mode 100755 dispatcher/dispatcher/notify.py diff --git a/dispatcher/dispatcher/main.py b/dispatcher/dispatcher/main.py index 7e1ce914..cd4a8e30 100644 --- a/dispatcher/dispatcher/main.py +++ b/dispatcher/dispatcher/main.py @@ -6,12 +6,15 @@ import os import sys import logging +from notify import slack_alert """ continuously polls(*) the email queue and renders+sends the template on every acked message (*) = waits for the queue to push a message onto the app """ +ONCALL_HANDLER = "@grasshopper" + def connect_to_smtp(): global smtpObj @@ -61,14 +64,20 @@ def requeue_wait(ch, method, properties, body, reason): if next_index >= len(REQUEUE_DELAY_DURATIONS): logging.warning('Max retry time hit, dropping message') - # TODO: notify someone that they've been sloppy + slack_alert(f"Time over, a message was dropped ({reason})", submessage = ":poop:") ch.basic_ack(delivery_tag=method.delivery_tag) return wait = REQUEUE_DELAY_DURATIONS[next_index] - logging.info(f'Retry attempt {next_index + 1}/{len(REQUEUE_DELAY_DURATIONS)} in {int(wait/1000)} sec') + retry_message = f'Retry attempt {next_index + 1}/{len(REQUEUE_DELAY_DURATIONS)} will happen in {int(wait/1000)} sec' + logging.info(retry_message) + last_chance = '' if next_index + 1 == len(REQUEUE_DELAY_DURATIONS): - logging.error(f'LAST ATTEMPT TO FIX: within {int(wait/1000)} sec') + last_chance = f'-- LAST ATTEMPT TO FIX: within {int(wait/1000)} sec' + f' {ONCALL_HANDLER}' + logging.error(last_chance) + slack_alert(f"A template is missing! ({reason})", + submessage = retry_message + " " + last_chance + ) headers = { 'reason': reason, @@ -96,23 +105,22 @@ def send_email(ch, method, properties, body): try: template = tpl_environment.get_template(f"{msg['template']}.jinja2") except exceptions.TemplateNotFound: - # TODO: send a notification to someone about adding a template - # NOTE: this is a requeuable message logging.error(f"Template {msg['template']}.jinja2 not found") - requeue_wait(ch, method, properties, body, reason="template_not_found") + # NOTE: this is a requeuable message + requeue_wait(ch, method, properties, body, reason=f"template_not_found-{msg['template']}") return try: rendered = template.render(msg['parameters'], altro=msg['subject']) except exceptions.UndefinedError as e: - # NOTE: this is a NON-requeuable message logging.error(f"Error in rendering: some parameter is undefined (error: {e}; message: {msg})") + # NOTE: this is a NON-requeuable message requeue_wait(ch, method, properties, body, reason="parameter_undefined") return except exceptions.TemplateNotFound: - # NOTE: this is a requeuable message logging.error(f"A sub-template in {msg['template']}.jinja2 was not found") - requeue_wait(ch, method, properties, body, reason="sub-template_not_found") + # NOTE: this is a requeuable message + requeue_wait(ch, method, properties, body, reason=f"subtemplate_not_found-{msg['template']}") return try: @@ -151,6 +159,9 @@ def process_dead_letter_messages(ch, method, properties, body): ] #TODO: why is this here again? wait_for = REQUEUE_DELAY_DURATIONS[-1] + logging.error("For some reason there's the DLQ handler that was triggered!") + slack_alert("For some reason there's the DLQ handler that was triggered!") + headers = { 'x-delay': wait_for, } @@ -175,7 +186,7 @@ def process_requeue(ch, method, properties, body): """ if (properties.headers["reason"] == 'parameter_undefined'): - logging.warn('Impossible to fix error, dropping message') + logging.warning('Impossible to fix error, dropping message') #TODO output something/notify to leave a trail for better debugging on what was missing ch.basic_ack(delivery_tag = method.delivery_tag) return diff --git a/dispatcher/dispatcher/notify.py b/dispatcher/dispatcher/notify.py new file mode 100755 index 00000000..1f9551c4 --- /dev/null +++ b/dispatcher/dispatcher/notify.py @@ -0,0 +1,65 @@ +import os +import json +import datetime +import slack_notifications as slack + + +#the ?? notification channels: +# - slack +# - telegram (wip) + +# example of tokens.json: +# { +# "slack": "xoxb-123", # a single token +# "telegram": ["456", "789"], # an array of tokens +# } + +tokens = None +NOTIFY_CHANNEL = os.environ.get("NOTIFY_CHANNEL") or "----monitoring" + +payload_nice = { + "title": 'Something going on with the dispatcher', + # "author_name": f'Host: {os.uname()[1]}', + #"text": "REPLACEME", + "footer": f'Error happened SOMETIME', + "color": '#FF5A36', + # "fields": [ + # slack.Attachment.Field( + # title='Error happened at:', + # value=f'{datetime.datetime.now().strftime("%d %b %Y, %H:%M:%S %Z")}', + # short=False + # ), + # ] + } + +def open_credentials_file(): + token_file = f'{os.path.realpath(os.path.dirname(__file__))}/../tokens.json' + with open(token_file, encoding="utf-8") as tokens_json: + return json.load(tokens_json) + + +def slack_alert(message_title, submessage = None): + global tokens + if tokens is None: + tokens = open_credentials_file() + api_key = tokens["slack"] + + slack.ACCESS_TOKEN = api_key + + payload_nice["title"] = f"Dispatcher says: {message_title}" + payload_nice["footer"] = f'Error happened {datetime.datetime.now().strftime("%d %b %Y, %H:%M:%S %Z")}' + if submessage: + payload_nice["text"] = submessage + + attachment = slack.Attachment( + **payload_nice + ) + + slack.send_notify(NOTIFY_CHANNEL, icon_emoji=':hole:', username='Dispatcher template notifier', + attachments=[attachment]) + +if __name__ == "__main__": + import sys + tokens = open_credentials_file() + EVENT = sys.argv[1] + slack_alert(EVENT) diff --git a/dispatcher/docker/Dockerfile b/dispatcher/docker/Dockerfile index b5c3de01..3c819a10 100644 --- a/dispatcher/docker/Dockerfile +++ b/dispatcher/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10-slim +FROM python:3.9-slim WORKDIR /usr/app/src diff --git a/dispatcher/docker/docker-compose.yml b/dispatcher/docker/docker-compose.yml index a06d3173..3c953f83 100644 --- a/dispatcher/docker/docker-compose.yml +++ b/dispatcher/docker/docker-compose.yml @@ -27,6 +27,9 @@ services: EMAIL_PORT: "${ETHEREAL_PORT}" EMAIL_ADDRESS: "${ETHEREAL_USER}" EMAIL_PASSWORD: "${ETHEREAL_PW}" + NOTIFY_CHANNEL: "slack-tests-programmatic-stuff-other" + volumes: + - ./secrets/tokens.json:/usr/app/tokens.json networks: diff --git a/dispatcher/poetry.lock b/dispatcher/poetry.lock index 255b0182..3f03e702 100644 --- a/dispatcher/poetry.lock +++ b/dispatcher/poetry.lock @@ -1,25 +1,129 @@ -# This file is automatically @generated by Poetry 1.4.0 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "certifi" +version = "2023.7.22" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2023.7.22-py3-none-any.whl", hash = "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"}, + {file = "certifi-2023.7.22.tar.gz", hash = "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.2.0" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.2.0.tar.gz", hash = "sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-win32.whl", hash = "sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-win32.whl", hash = "sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-win32.whl", hash = "sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-win32.whl", hash = "sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-win32.whl", hash = "sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80"}, + {file = "charset_normalizer-3.2.0-py3-none-any.whl", hash = "sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6"}, +] [[package]] name = "faker" -version = "18.4.0" +version = "18.13.0" description = "Faker is a Python package that generates fake data for you." -category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "Faker-18.4.0-py3-none-any.whl", hash = "sha256:170ead9d0d140916168b142df69c44722b8f622ced2070802d0af9c476f0cb84"}, - {file = "Faker-18.4.0.tar.gz", hash = "sha256:977ad0b7aa7a61ed57287d6a0723a827e9d3dd1f8cc82aaf08707f281b33bacc"}, + {file = "Faker-18.13.0-py3-none-any.whl", hash = "sha256:801d1a2d71f1fc54d332de2ab19de7452454309937233ea2f7485402882d67b3"}, + {file = "Faker-18.13.0.tar.gz", hash = "sha256:84bcf92bb725dd7341336eea4685df9a364f16f2470c4d29c1d7e6c5fd5a457d"}, ] [package.dependencies] python-dateutil = ">=2.4" +[[package]] +name = "idna" +version = "3.4" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, + {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, +] + [[package]] name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -35,74 +139,82 @@ i18n = ["Babel (>=2.7)"] [[package]] name = "markupsafe" -version = "2.1.2" +version = "2.1.3" description = "Safely add untrusted strings to HTML/XML markup." -category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:340bea174e9761308703ae988e982005aedf427de816d1afe98147668cc03036"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22152d00bf4a9c7c83960521fc558f55a1adbc0631fbb00a9471e097b19d72e1"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28057e985dace2f478e042eaa15606c7efccb700797660629da387eb289b9323"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca244fa73f50a800cf8c3ebf7fd93149ec37f5cb9596aa8873ae2c1d23498601"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d9d971ec1e79906046aa3ca266de79eac42f1dbf3612a05dc9368125952bd1a1"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7e007132af78ea9df29495dbf7b5824cb71648d7133cf7848a2a5dd00d36f9ff"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7313ce6a199651c4ed9d7e4cfb4aa56fe923b1adf9af3b420ee14e6d9a73df65"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-win32.whl", hash = "sha256:c4a549890a45f57f1ebf99c067a4ad0cb423a05544accaf2b065246827ed9603"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:835fb5e38fd89328e9c81067fd642b3593c33e1e17e2fdbf77f5676abb14a156"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2ec4f2d48ae59bbb9d1f9d7efb9236ab81429a764dedca114f5fdabbc3788013"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:608e7073dfa9e38a85d38474c082d4281f4ce276ac0010224eaba11e929dd53a"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65608c35bfb8a76763f37036547f7adfd09270fbdbf96608be2bead319728fcd"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2bfb563d0211ce16b63c7cb9395d2c682a23187f54c3d79bfec33e6705473c6"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:da25303d91526aac3672ee6d49a2f3db2d9502a4a60b55519feb1a4c7714e07d"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9cad97ab29dfc3f0249b483412c85c8ef4766d96cdf9dcf5a1e3caa3f3661cf1"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:085fd3201e7b12809f9e6e9bc1e5c96a368c8523fad5afb02afe3c051ae4afcc"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1bea30e9bf331f3fef67e0a3877b2288593c98a21ccb2cf29b74c581a4eb3af0"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-win32.whl", hash = "sha256:7df70907e00c970c60b9ef2938d894a9381f38e6b9db73c5be35e59d92e06625"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:e55e40ff0cc8cc5c07996915ad367fa47da6b3fc091fdadca7f5403239c5fec3"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a6e40afa7f45939ca356f348c8e23048e02cb109ced1eb8420961b2f40fb373a"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf877ab4ed6e302ec1d04952ca358b381a882fbd9d1b07cccbfd61783561f98a"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63ba06c9941e46fa389d389644e2d8225e0e3e5ebcc4ff1ea8506dce646f8c8a"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f1cd098434e83e656abf198f103a8207a8187c0fc110306691a2e94a78d0abb2"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:55f44b440d491028addb3b88f72207d71eeebfb7b5dbf0643f7c023ae1fba619"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a6f2fcca746e8d5910e18782f976489939d54a91f9411c32051b4aab2bd7c513"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0b462104ba25f1ac006fdab8b6a01ebbfbce9ed37fd37fd4acd70c67c973e460"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-win32.whl", hash = "sha256:7668b52e102d0ed87cb082380a7e2e1e78737ddecdde129acadb0eccc5423859"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6d6607f98fcf17e534162f0709aaad3ab7a96032723d8ac8750ffe17ae5a0666"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a806db027852538d2ad7555b203300173dd1b77ba116de92da9afbc3a3be3eed"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a4abaec6ca3ad8660690236d11bfe28dfd707778e2442b45addd2f086d6ef094"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f03a532d7dee1bed20bc4884194a16160a2de9ffc6354b3878ec9682bb623c54"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4cf06cdc1dda95223e9d2d3c58d3b178aa5dacb35ee7e3bbac10e4e1faacb419"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:22731d79ed2eb25059ae3df1dfc9cb1546691cc41f4e3130fe6bfbc3ecbbecfa"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f8ffb705ffcf5ddd0e80b65ddf7bed7ee4f5a441ea7d3419e861a12eaf41af58"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8db032bf0ce9022a8e41a22598eefc802314e81b879ae093f36ce9ddf39ab1ba"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2298c859cfc5463f1b64bd55cb3e602528db6fa0f3cfd568d3605c50678f8f03"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-win32.whl", hash = "sha256:50c42830a633fa0cf9e7d27664637532791bfc31c731a87b202d2d8ac40c3ea2"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:bb06feb762bade6bf3c8b844462274db0c76acc95c52abe8dbed28ae3d44a147"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:99625a92da8229df6d44335e6fcc558a5037dd0a760e11d84be2260e6f37002f"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8bca7e26c1dd751236cfb0c6c72d4ad61d986e9a41bbf76cb445f69488b2a2bd"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40627dcf047dadb22cd25ea7ecfe9cbf3bbbad0482ee5920b582f3809c97654f"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40dfd3fefbef579ee058f139733ac336312663c6706d1163b82b3003fb1925c4"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:090376d812fb6ac5f171e5938e82e7f2d7adc2b629101cec0db8b267815c85e2"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2e7821bffe00aa6bd07a23913b7f4e01328c3d5cc0b40b36c0bd81d362faeb65"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c0a33bc9f02c2b17c3ea382f91b4db0e6cde90b63b296422a939886a7a80de1c"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b8526c6d437855442cdd3d87eede9c425c4445ea011ca38d937db299382e6fa3"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-win32.whl", hash = "sha256:137678c63c977754abe9086a3ec011e8fd985ab90631145dfb9294ad09c102a7"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:0576fe974b40a400449768941d5d0858cc624e3249dfd1e0c33674e5c7ca7aed"}, - {file = "MarkupSafe-2.1.2.tar.gz", hash = "sha256:abcabc8c2b26036d62d4c746381a6f7cf60aafcc653198ad678306986b09450d"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-win32.whl", hash = "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-win_amd64.whl", hash = "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-win32.whl", hash = "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-win_amd64.whl", hash = "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-win32.whl", hash = "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-win_amd64.whl", hash = "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-win32.whl", hash = "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-win_amd64.whl", hash = "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba"}, + {file = "MarkupSafe-2.1.3.tar.gz", hash = "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad"}, ] [[package]] name = "pika" -version = "1.3.1" +version = "1.3.2" description = "Pika Python AMQP Client Library" -category = "main" optional = false -python-versions = ">=3.4" +python-versions = ">=3.7" files = [ - {file = "pika-1.3.1-py3-none-any.whl", hash = "sha256:89f5e606646caebe3c00cbdbc4c2c609834adde45d7507311807b5775edac8e0"}, - {file = "pika-1.3.1.tar.gz", hash = "sha256:beb19ff6dd1547f99a29acc2c6987ebb2ba7c44bf44a3f8e305877c5ef7d2fdc"}, + {file = "pika-1.3.2-py3-none-any.whl", hash = "sha256:0779a7c1fafd805672796085560d290213a465e4f6f76a6fb19e378d8041a14f"}, + {file = "pika-1.3.2.tar.gz", hash = "sha256:b2a327ddddf8570b4965b3576ac77091b850262d34ce8c1d8cb4e4146aa4145f"}, ] [package.extras] @@ -114,7 +226,6 @@ twisted = ["twisted"] name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" -category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -125,11 +236,31 @@ files = [ [package.dependencies] six = ">=1.5" +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + [[package]] name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -137,7 +268,38 @@ files = [ {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] +[[package]] +name = "slack-notifications" +version = "0.2.2" +description = "Send notifications to slack channel with supporting attachments and fields" +optional = false +python-versions = "*" +files = [ + {file = "slack-notifications-0.2.2.tar.gz", hash = "sha256:cb3930c01f7625b93fa3da239942161a2a9407894314688f4fde6a7634543df2"}, + {file = "slack_notifications-0.2.2-py3-none-any.whl", hash = "sha256:1a0e2fd382b43ef3a0f9828df35e2a69ce711f84c8fda7f8ab3c159861c75cf1"}, +] + +[package.dependencies] +requests = "*" + +[[package]] +name = "urllib3" +version = "2.0.5" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.7" +files = [ + {file = "urllib3-2.0.5-py3-none-any.whl", hash = "sha256:ef16afa8ba34a1f989db38e1dbbe0c302e4289a47856990d0682e374563ce35e"}, + {file = "urllib3-2.0.5.tar.gz", hash = "sha256:13abf37382ea2ce6fb744d4dad67838eec857c9f4f57009891805e0b5e123594"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + [metadata] lock-version = "2.0" -python-versions = "^3.10" -content-hash = "c8c63462889297aa1d01f2c1b74a2d56f3fa034994b301d3302c455ba7257887" +python-versions = "^3.9" +content-hash = "cfd3ce33c11517121dc4dad8f62cb06ef74a82d92be8f9408b3e4237f1059938" diff --git a/dispatcher/pyproject.toml b/dispatcher/pyproject.toml index cb28b021..08d928cd 100644 --- a/dispatcher/pyproject.toml +++ b/dispatcher/pyproject.toml @@ -6,9 +6,10 @@ authors = ["linuxbandit "] readme = "README.md" [tool.poetry.dependencies] -python = "^3.10" +python = "^3.9" pika = "^1.3.1" jinja2 = "^3.1.2" +slack-notifications = "^0.2.2" [tool.poetry.group.dev.dependencies] diff --git a/dispatcher/requirements.txt b/dispatcher/requirements.txt index 147529a3..5310b7be 100644 --- a/dispatcher/requirements.txt +++ b/dispatcher/requirements.txt @@ -3,3 +3,4 @@ requests==2.18.4 pika==1.3.1 Jinja2==3.0.3 Faker==14.2.1 +slack-notifications==0.2.2 From 89e4f477c129d9b97603a94edf1b3585fe682422 Mon Sep 17 00:00:00 2001 From: linuxbandit Date: Sat, 30 Sep 2023 12:27:59 +0300 Subject: [PATCH 32/32] chore: apply suggestions from code review for templates Co-authored-by: Rik Smale <13023439+WikiRik@users.noreply.github.com> --- dispatcher/templates/confirm_email.jinja2 | 2 +- dispatcher/templates/events_applied.jinja2 | 2 +- dispatcher/templates/events_status_changed.jinja2 | 4 ++-- dispatcher/templates/events_submitted.jinja2 | 2 +- dispatcher/templates/member_joined.jinja2 | 2 +- dispatcher/templates/snippets/board_info.jinja2 | 2 +- dispatcher/templates/snippets/events_application_info.jinja2 | 2 +- .../templates/snippets/statutory_application_info.jinja2 | 2 +- .../snippets/summeruniversity_application_info.jinja2 | 2 +- dispatcher/templates/summeruniversity_status_changed.jinja2 | 2 +- dispatcher/templates/summeruniversity_submitted.jinja2 | 2 +- 11 files changed, 12 insertions(+), 12 deletions(-) diff --git a/dispatcher/templates/confirm_email.jinja2 b/dispatcher/templates/confirm_email.jinja2 index d27154f9..2be2dbb9 100644 --- a/dispatcher/templates/confirm_email.jinja2 +++ b/dispatcher/templates/confirm_email.jinja2 @@ -39,7 +39,7 @@ diff --git a/dispatcher/templates/events_applied.jinja2 b/dispatcher/templates/events_applied.jinja2 index 98a64198..2071f38c 100644 --- a/dispatcher/templates/events_applied.jinja2 +++ b/dispatcher/templates/events_applied.jinja2 @@ -45,7 +45,7 @@ diff --git a/dispatcher/templates/events_status_changed.jinja2 b/dispatcher/templates/events_status_changed.jinja2 index 2f86aa31..043e767f 100644 --- a/dispatcher/templates/events_status_changed.jinja2 +++ b/dispatcher/templates/events_status_changed.jinja2 @@ -41,9 +41,9 @@
    Your event's status was changed
    The status of your event, - "{{ event["name"] }}", + {{ event["name"] }}, has changed from {{ old_status }} - to {{ event["status"] }}. + to {{ event["status"] }}.
    diff --git a/dispatcher/templates/events_submitted.jinja2 b/dispatcher/templates/events_submitted.jinja2 index d402e0f6..365f9df6 100644 --- a/dispatcher/templates/events_submitted.jinja2 +++ b/dispatcher/templates/events_submitted.jinja2 @@ -41,7 +41,7 @@
    A new event is submitted
    A new event - "{{ event["name"] }}", + "{{ event["name"] }}" is submitted.
    diff --git a/dispatcher/templates/member_joined.jinja2 b/dispatcher/templates/member_joined.jinja2 index 7db54f6f..c8767a64 100644 --- a/dispatcher/templates/member_joined.jinja2 +++ b/dispatcher/templates/member_joined.jinja2 @@ -39,7 +39,7 @@ diff --git a/dispatcher/templates/snippets/board_info.jinja2 b/dispatcher/templates/snippets/board_info.jinja2 index 378741a0..53642360 100644 --- a/dispatcher/templates/snippets/board_info.jinja2 +++ b/dispatcher/templates/snippets/board_info.jinja2 @@ -5,7 +5,7 @@
  • End date: {{ board["end_date"] }}
    • {% for position in positions %} -
  • {{ position["function"] }} {{ position["name"] }}
    • +
  • {{ position["function"] }}: {{ position["name"] }}
    • {% endfor %}
  • Message: {{ board["message"] }}
    • diff --git a/dispatcher/templates/snippets/events_application_info.jinja2 b/dispatcher/templates/snippets/events_application_info.jinja2 index cd702dec..f425a425 100644 --- a/dispatcher/templates/snippets/events_application_info.jinja2 +++ b/dispatcher/templates/snippets/events_application_info.jinja2 @@ -7,6 +7,6 @@
  • Email: {{ application["email"] }}
    • {% for question in event["questions"] %} -
  • {{ question["description"] }} {{ application["answers"][loop.index0] }}
    • +
  • {{ question["description"] }}: {{ application["answers"][loop.index0] }}
    • {% endfor %} diff --git a/dispatcher/templates/snippets/statutory_application_info.jinja2 b/dispatcher/templates/snippets/statutory_application_info.jinja2 index 57dae5ef..90782211 100644 --- a/dispatcher/templates/snippets/statutory_application_info.jinja2 +++ b/dispatcher/templates/snippets/statutory_application_info.jinja2 @@ -24,7 +24,7 @@
  • Embassy: {{ application["visa_embassy"] }}
    • {% for question in event["questions"] %} -
  • {{ question["description"] }} {{ application["answers"][loop.index0] }}
    • +
  • {{ question["description"] }}: {{ application["answers"][loop.index0] }}
    • {% endfor %} diff --git a/dispatcher/templates/snippets/summeruniversity_application_info.jinja2 b/dispatcher/templates/snippets/summeruniversity_application_info.jinja2 index 7919428e..914e4673 100644 --- a/dispatcher/templates/snippets/summeruniversity_application_info.jinja2 +++ b/dispatcher/templates/snippets/summeruniversity_application_info.jinja2 @@ -9,7 +9,7 @@
  • Motivation: {{ application["motivation"] }}
    • {% for question in event["questions"] %} -
  • {{ question["description"] }} {{ application["answers"][loop.index0] }}
    • +
  • {{ question["description"] }}: {{ application["answers"][loop.index0] }}
    • {% endfor %} diff --git a/dispatcher/templates/summeruniversity_status_changed.jinja2 b/dispatcher/templates/summeruniversity_status_changed.jinja2 index 78e08924..2545fe3e 100644 --- a/dispatcher/templates/summeruniversity_status_changed.jinja2 +++ b/dispatcher/templates/summeruniversity_status_changed.jinja2 @@ -41,7 +41,7 @@
    Your Summer University's status was changed
    The status of your Summer University, - "{{event["name"]}}", + {{event["name"]}}, has changed from {{old_status}} to {{event["status"]}}.
    diff --git a/dispatcher/templates/summeruniversity_submitted.jinja2 b/dispatcher/templates/summeruniversity_submitted.jinja2 index add2288f..98e09dc1 100644 --- a/dispatcher/templates/summeruniversity_submitted.jinja2 +++ b/dispatcher/templates/summeruniversity_submitted.jinja2 @@ -47,7 +47,7 @@
    A new event is submitted for {{event["status"]}}
    A new event - "{{event["name"]}}", + {{event["name"]}} is submitted as {{event["status"]}}.
    Hello {{member_firstname}} {{member_surname}}
    -
    You've recently registered on AEGEE Intranet. To confirm your account, follow this link
    +
    You've recently registered on MyAEGEE. To confirm your account, follow this link
    Here are your application details: - {% include "snippets/summeruniversity_application_info.jinja2" %} + {% include "snippets/events_application_info.jinja2" %}
    You might have a new member soon!
    -
    Recently member {{member_firstname}} {{member_surname}} requested to join your body {{body_name}}
    +
    {{member_firstname}} {{member_surname}} requested to join {{body_name}}