Merge pull request #68 from dlcs/feature/sqs_broker

Allow SQS broker

.env.dist

@@ -34,6 +34,9 @@ ENGINE_WORKER_TIMEOUT=3600
 ENGINE_WORKER_RETRY=4500
 ENGINE_WORKER_MAX_ATTEMPTS=0
 
+# Django Q SQS Broker
+SQS_BROKER_QUEUE_NAME=composite-handler-queue
+
 # Run migrations
 MIGRATE=True
 

.github/workflows/build.yml

@@ -16,12 +16,12 @@ jobs:
         uses: actions/checkout@v4
 
       - id: setup-python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
+          python-version: '3.12'
 
       - id: pre-commit
-        uses: pre-commit/[email protected].0
+        uses: pre-commit/[email protected].1
 
       - id: docker-setup-buildx
         uses: docker/setup-buildx-action@v3

.pre-commit-config.yaml

@@ -15,4 +15,4 @@ repos:
     rev: 23.11.0
     hooks:
       - id: black
-        language_version: python3.11
+        language_version: python3.12

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11
+FROM python:3.12
 LABEL maintainer="Donald Gray <[email protected]>"
 LABEL org.opencontainers.image.source=https://github.com/dlcs/composite-handler
 

README.md

@@ -91,9 +91,18 @@ The following list of environment variables are supported:
 | `MIGRATE`                     | None                           | API, Engine  | If "True" will run migrations + createcachetable on startup if entrypoint used.                                                                                                                                                                                              |
 | `INIT_SUPERUSER`              | None                           | API, Engine  | If "True" will attempt to create superuser. Needs standard Django envvars to be set (e.g. `DJANGO_SUPERUSER_USERNAME`, `DJANGO_SUPERUSER_EMAIL`, `DJANGO_SUPERUSER_PASSWORD`) if entrypoint used.                                                                            |
 | `GUNICORN_WORKERS`            | `2`                            | API          | The value of [`--workers`](https://docs.gunicorn.org/en/stable/run.html) arg when running gunicorn                                                                                                                                                                           |
+| `SQS_BROKER_QUEUE_NAME`       | None                           | API, Engine  | If set, django-q [SQS broker](https://django-q.readthedocs.io/en/latest/brokers.html#amazon-sqs) will be used. Queue created if doesn't exist. If empty default [Django ORM broker](https://django-q.readthedocs.io/en/latest/brokers.html#django-orm) is used               |
 
 Note that in order to access the S3 bucket, the Composite Handler assumes that valid AWS credentials are available in the environment - this can be in the former of [environment variables](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html), or in the form of ambient credentials.
 
+### Django Q Broker
+
+By default Django Q will use the default [Django ORM](https://django-q.readthedocs.io/en/latest/brokers.html#django-orm) broker.
+
+The [SQS broker](https://django-q.readthedocs.io/en/latest/brokers.html#amazon-sqs) can be configured by specifying the `SQS_BROKER_QUEUE_NAME` environment variable. Default SQS broker behaviour is to create this queue if it is not found.
+
+As with S3, above, Composite Handler assumes that valid AWS credentials are available in the environment.
+
 ## Building
 
 The project ships with a [`Dockerfile`](./Dockerfile):

docker-compose.yml

@@ -15,6 +15,8 @@ services:
       - INIT_SUPERUSER=True
     ports:
       - "8000:8000"
+    volumes:
+      - $HOME/.aws:/srv/dlcs/.aws:ro
   engine:
     build: .
     command: /srv/dlcs/entrypoint-worker.sh

src/app/settings.py

@@ -156,9 +156,14 @@
     "timeout": env("ENGINE_WORKER_TIMEOUT", cast=int, default=3600),
     "retry": env("ENGINE_WORKER_RETRY", cast=int, default=4500),
     "max_attempts": env("ENGINE_WORKER_MAX_ATTEMPTS", cast=int, default=0),
-    "orm": "default",
 }
 
+if sqs_queue_name := env.str("SQS_BROKER_QUEUE_NAME", default=""):
+    Q_CLUSTER["name"] = sqs_queue_name
+    Q_CLUSTER["sqs"] = {}
+else:
+    Q_CLUSTER["orm"] = "default"
+
 SCRATCH_DIRECTORY = env.path("SCRATCH_DIRECTORY", default="/tmp/scratch")
 
 WEB_SERVER = {

src/requirements.txt

@@ -1,18 +1,18 @@
-asgiref==3.7.2
-attrs==23.1.0
-boto3==1.29.4
-Django==4.2.7
+asgiref==3.8.1
+attrs==23.2.0
+boto3==1.34.143
+Django==4.2.14
 django-environ==0.11.2
-django-health-check==3.17.0
+django-health-check==3.18.3
 django-q==1.3.9
-djangorestframework==3.14.0
-gunicorn==21.2.0
-jsonschema==4.20.0
-pdf2image==1.16.3
-psutil==5.9.6
+djangorestframework==3.15.2
+gunicorn==22.0.0
+jsonschema==4.23.0
+pdf2image==1.17.0
+psutil==5.9.8
 psycopg2-binary==2.9.9
 pyrsistent==0.20.0
-pytz==2021.3
-requests==2.31.0
+pytz==2024.1
+requests==2.32.3
 sqlparse==0.4.4
-tqdm==4.66.1
+tqdm==4.66.4