diff --git a/.env.dist b/.env.dist index d2886e2..323f21f 100644 --- a/.env.dist +++ b/.env.dist @@ -34,6 +34,9 @@ ENGINE_WORKER_TIMEOUT=3600 ENGINE_WORKER_RETRY=4500 ENGINE_WORKER_MAX_ATTEMPTS=0 +# Django Q SQS Broker +SQS_BROKER_QUEUE_NAME=composite-handler-queue + # Run migrations MIGRATE=True diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bc54124..432c0ea 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,12 +16,12 @@ jobs: uses: actions/checkout@v4 - id: setup-python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: '3.12' - id: pre-commit - uses: pre-commit/action@v3.0.0 + uses: pre-commit/action@v3.0.1 - id: docker-setup-buildx uses: docker/setup-buildx-action@v3 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 48bf524..5d80306 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,4 +15,4 @@ repos: rev: 23.11.0 hooks: - id: black - language_version: python3.11 + language_version: python3.12 diff --git a/Dockerfile b/Dockerfile index b9f1e9c..3a273ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11 +FROM python:3.12 LABEL maintainer="Donald Gray " LABEL org.opencontainers.image.source=https://github.com/dlcs/composite-handler diff --git a/README.md b/README.md index c81ab80..f635ad9 100644 --- a/README.md +++ b/README.md @@ -91,9 +91,18 @@ The following list of environment variables are supported: | `MIGRATE` | None | API, Engine | If "True" will run migrations + createcachetable on startup if entrypoint used. | | `INIT_SUPERUSER` | None | API, Engine | If "True" will attempt to create superuser. Needs standard Django envvars to be set (e.g. `DJANGO_SUPERUSER_USERNAME`, `DJANGO_SUPERUSER_EMAIL`, `DJANGO_SUPERUSER_PASSWORD`) if entrypoint used. | | `GUNICORN_WORKERS` | `2` | API | The value of [`--workers`](https://docs.gunicorn.org/en/stable/run.html) arg when running gunicorn | +| `SQS_BROKER_QUEUE_NAME` | None | API, Engine | If set, django-q [SQS broker](https://django-q.readthedocs.io/en/latest/brokers.html#amazon-sqs) will be used. Queue created if doesn't exist. If empty default [Django ORM broker](https://django-q.readthedocs.io/en/latest/brokers.html#django-orm) is used | Note that in order to access the S3 bucket, the Composite Handler assumes that valid AWS credentials are available in the environment - this can be in the former of [environment variables](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html), or in the form of ambient credentials. +### Django Q Broker + +By default Django Q will use the default [Django ORM](https://django-q.readthedocs.io/en/latest/brokers.html#django-orm) broker. + +The [SQS broker](https://django-q.readthedocs.io/en/latest/brokers.html#amazon-sqs) can be configured by specifying the `SQS_BROKER_QUEUE_NAME` environment variable. Default SQS broker behaviour is to create this queue if it is not found. + +As with S3, above, Composite Handler assumes that valid AWS credentials are available in the environment. + ## Building The project ships with a [`Dockerfile`](./Dockerfile): diff --git a/docker-compose.yml b/docker-compose.yml index fe7918c..247f757 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,8 @@ services: - INIT_SUPERUSER=True ports: - "8000:8000" + volumes: + - $HOME/.aws:/srv/dlcs/.aws:ro engine: build: . command: /srv/dlcs/entrypoint-worker.sh diff --git a/src/app/settings.py b/src/app/settings.py index af2ff3d..58f1c28 100644 --- a/src/app/settings.py +++ b/src/app/settings.py @@ -156,9 +156,14 @@ "timeout": env("ENGINE_WORKER_TIMEOUT", cast=int, default=3600), "retry": env("ENGINE_WORKER_RETRY", cast=int, default=4500), "max_attempts": env("ENGINE_WORKER_MAX_ATTEMPTS", cast=int, default=0), - "orm": "default", } +if sqs_queue_name := env.str("SQS_BROKER_QUEUE_NAME", default=""): + Q_CLUSTER["name"] = sqs_queue_name + Q_CLUSTER["sqs"] = {} +else: + Q_CLUSTER["orm"] = "default" + SCRATCH_DIRECTORY = env.path("SCRATCH_DIRECTORY", default="/tmp/scratch") WEB_SERVER = { diff --git a/src/requirements.txt b/src/requirements.txt index f689800..59d85d6 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,18 +1,18 @@ -asgiref==3.7.2 -attrs==23.1.0 -boto3==1.29.4 -Django==4.2.7 +asgiref==3.8.1 +attrs==23.2.0 +boto3==1.34.143 +Django==4.2.14 django-environ==0.11.2 -django-health-check==3.17.0 +django-health-check==3.18.3 django-q==1.3.9 -djangorestframework==3.14.0 -gunicorn==21.2.0 -jsonschema==4.20.0 -pdf2image==1.16.3 -psutil==5.9.6 +djangorestframework==3.15.2 +gunicorn==22.0.0 +jsonschema==4.23.0 +pdf2image==1.17.0 +psutil==5.9.8 psycopg2-binary==2.9.9 pyrsistent==0.20.0 -pytz==2021.3 -requests==2.31.0 +pytz==2024.1 +requests==2.32.3 sqlparse==0.4.4 -tqdm==4.66.1 +tqdm==4.66.4