Merge pull request #35 from dlcs/feature/slf_updates

Various updates prior to SLF release

.dockerignore

@@ -41,4 +41,4 @@ Utils/
 **/bld/
 **/[Bb]in/
 **/[Oo]bj/
-**/[Ll]og/
+**/[Ll]og/

.env.dist

@@ -41,4 +41,4 @@ MIGRATE=True
 INIT_SUPERUSER=True
 DJANGO_SUPERUSER_EMAIL=x.y@z
 DJANGO_SUPERUSER_PASSWORD=composite-handler-password
-DJANGO_SUPERUSER_USERNAME=Admin
+DJANGO_SUPERUSER_USERNAME=Admin

.github/workflows/build.yml

@@ -13,21 +13,21 @@ jobs:
 
     steps:
       - id: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - id: setup-python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.9'
+          python-version: '3.11'
 
       - id: pre-commit
         uses: pre-commit/[email protected]
 
       - id: docker-setup-buildx
-        uses: docker/setup-buildx-action@v2
-            
+        uses: docker/setup-buildx-action@v3
+
       - id: docker-meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: ghcr.io/dlcs/compositehandler
           tags: |
@@ -39,18 +39,17 @@ jobs:
             type=semver,pattern={{major}}
 
       - id: docker-login
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - id: docker-build-push
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           context: .
           builder: ${{ steps.docker-setup-buildx.outputs.name }}
           tags: ${{ steps.docker-meta.outputs.tags }}
           labels: ${{ steps.docker-meta.outputs.labels }}
           push: ${{ github.actor != 'dependabot[bot]' }}
-

.gitignore

@@ -183,4 +183,4 @@ src/.idea/workspace.xml
 
 # Temp working files
 src/scratch/
-scratch/
+scratch/

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks.git
-    rev: v4.0.1
+    rev: v4.5.0
     hooks:
       - id: check-added-large-files
       - id: check-ast
@@ -12,7 +12,7 @@ repos:
       - id: requirements-txt-fixer
       - id: trailing-whitespace
   - repo: https://github.com/ambv/black
-    rev: stable
+    rev: 23.11.0
     hooks:
       - id: black
-        language_version: python3.9
+        language_version: python3.11

Dockerfile

@@ -4,20 +4,34 @@ LABEL org.opencontainers.image.source=https://github.com/dlcs/composite-handler
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
+ENV GUNICORN_WORKERS=2
 ENV DEBIAN_FRONTEND=noninteractive
 ENV PYTHONUNBUFFERED=1
 
 RUN apt-get update && apt-get --yes install apt-utils && apt-get --yes upgrade \
+    && apt-get install -y nginx \
     && apt-get --yes install poppler-data poppler-utils \
     && apt-get --yes autoremove && apt-get --yes autoclean && apt-get --yes clean \
     && useradd --create-home --home-dir /srv/dlcs --shell /bin/bash --uid 1000 dlcs \
     && python -m pip install --upgrade pip
 
+# Copy nginx config and create appropriate folders
+COPY --chown=dlcs:dlcs ./nginx.conf /etc/nginx/nginx.conf
+RUN mkdir -p /var/cache/nginx && chown -R dlcs:dlcs /var/cache/nginx && \
+    mkdir -p /var/log/nginx  && chown -R dlcs:dlcs /var/log/nginx && \
+    mkdir -p /var/lib/nginx  && chown -R dlcs:dlcs /var/lib/nginx && \
+    touch /run/nginx.pid && chown -R dlcs:dlcs /run/nginx.pid && \
+    chown -R dlcs:dlcs /etc/nginx && \
+    chmod -R 777 /etc/nginx/conf.d
+
+RUN mkdir /srv/dlcs/app_static && chown dlcs:dlcs /srv/dlcs/app_static/
+
 COPY --chown=dlcs:dlcs ./src/requirements.txt /srv/dlcs
 RUN pip install --no-warn-script-location --requirement /srv/dlcs/requirements.txt
 
 COPY --chown=dlcs:dlcs ./src /srv/dlcs
 COPY --chown=dlcs:dlcs ./entrypoints /srv/dlcs
+
 RUN chmod +x /srv/dlcs/entrypoint.sh
 RUN chmod +x /srv/dlcs/entrypoint-api.sh
 RUN chmod +x /srv/dlcs/entrypoint-worker.sh

entrypoints/entrypoint-api.sh

@@ -4,4 +4,12 @@ set -o errexit
 set -o pipefail
 
 bash entrypoint.sh
-python manage.py runserver 0.0.0.0:8000
+
+echo "collecting static files.."
+python manage.py collectstatic --noinput
+
+echo "starting nginx.."
+nginx
+
+echo "starting gunicorn on :5000 with $GUNICORN_WORKERS workers"
+gunicorn app.wsgi:application --workers $GUNICORN_WORKERS --bind "0.0.0.0:5000"

entrypoints/entrypoint-worker.sh

@@ -4,4 +4,4 @@ set -o errexit
 set -o pipefail
 
 bash entrypoint.sh
-python manage.py qcluster
+python manage.py qcluster

nginx.conf

@@ -0,0 +1,94 @@
+# Set number of worker processes automatically based on number of CPU cores.
+worker_processes auto;
+
+# Enables the use of JIT for regular expressions to speed-up their processing.
+pcre_jit on;
+
+# Configures default error logger.
+error_log     /var/log/nginx/error.log warn;
+
+events {
+    # The maximum number of simultaneous connections that can be opened by
+    # a worker process.
+    worker_connections 1024;
+}
+
+http {
+    # Includes mapping of file name extensions to MIME types of responses
+    # and defines the default type.
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    # Don't tell nginx version to clients.
+    server_tokens off;
+
+    # Specifies the maximum accepted body size of a client request, as
+    # indicated by the request header Content-Length. If the stated content
+    # length is greater than this size, then the client receives the HTTP
+    # error code 413. Set to 0 to disable.
+    client_max_body_size 5M;
+
+    # Timeout for keep-alive connections. Server will close connections after
+    # this time.
+    keepalive_timeout 65;
+
+    # Sendfile copies data between one FD and other from within the kernel,
+    # which is more efficient than read() + write().
+    sendfile on;
+
+    # Don't buffer data-sends (disable Nagle algorithm).
+    # Good for sending frequent small bursts of data in real time.
+    tcp_nodelay on;
+
+    # Specifies that our cipher suits should be preferred over client ciphers.
+    ssl_prefer_server_ciphers on;
+
+    # Enables a shared SSL cache with size that can hold around 8000 sessions.
+    ssl_session_cache shared:SSL:2m;
+
+    # Enable gzipping of responses.
+    gzip on;
+
+    # Set the Vary HTTP header as defined in the RFC 2616.
+    gzip_vary on;
+
+    # Enable checking the existence of precompressed files.
+    gzip_static on;
+
+    # Specifies the main log format.
+    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
+                    '$status $body_bytes_sent "$http_referer" '
+                    '"$http_user_agent" "$http_x_forwarded_for"';
+
+    # Sets the path, format, and configuration for a buffered log write.
+    access_log /var/log/nginx/access.log main;
+
+    proxy_connect_timeout 300s;
+
+    proxy_read_timeout 300s;
+
+    server {
+        listen        8000 default_server;
+        server_name   "";
+
+        error_log     /var/log/nginx/error.log;
+        access_log    /var/log/nginx/access.log;
+
+        location / {
+            proxy_pass	http://127.0.0.1:5000;
+            proxy_http_version 1.1;
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+        }
+
+        # Pass through the static files so they aren't served from Django
+        location /static {
+            alias /srv/dlcs/app_static;
+        }
+        location /media {
+            alias /app_media;
+        }
+    }
+}

src/app/api/views.py

@@ -28,7 +28,8 @@ def __init__(self, **kwargs):
     def _validate_credentials(self, customer, headers):
         if not customer or "Authorization" not in headers:
             raise PermissionDenied
-        self._dlcs.test_credentials(customer, headers["Authorization"])
+        if not self._dlcs.test_credentials(customer, headers["Authorization"]):
+            raise PermissionDenied
 
     def _build_collection_response_body(self, collection):
         return {
@@ -61,7 +62,6 @@ def _build_member_response_body(self, member):
 
 class QueryCollectionAPIView(AbstractAPIView):
     def get(self, request, *args, **kwargs):
-
         try:
             collection = Collection.objects.get(id=kwargs["collection_id"])
         except Collection.DoesNotExist:
@@ -98,7 +98,6 @@ def get(self, request, *args, **kwargs):
 
 class CollectionAPIView(AbstractAPIView):
     def post(self, request, *args, **kwargs):
-
         self._validate_credentials(kwargs["customer"], request.headers)
 
         serializer = CollectionSerializer(

src/app/common/dlcs.py

@@ -15,8 +15,11 @@ def test_credentials(self, customer, auth):
             f"{self._api_root}customers/{customer}",
             headers={"Content-Type": "application/json", "Authorization": auth},
         )
+
         if response.status_code == requests.codes.ok:
-            return response.json()
+            return True
+        elif response.status_code in [401, 403]:
+            return False
         else:
             response.raise_for_status()
 
@@ -29,5 +32,7 @@ def ingest(self, customer, json, auth):
         if response.status_code == requests.codes.created:
             return response.json()
         else:
-            logger.info(f"ingest failed with status {response.status_code}: {response.text}")
+            logger.info(
+                f"ingest failed with status {response.status_code}: {response.text}"
+            )
             response.raise_for_status()

src/app/common/migrations/0001_initial.py

@@ -6,7 +6,6 @@
 
 
 class Migration(migrations.Migration):
-
     initial = True
 
     dependencies = []

src/app/common/migrations/0002_alter_member_status.py

@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("common", "0001_initial"),
     ]

src/app/common/migrations/0003_auto_20220112_1454.py

@@ -6,7 +6,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("common", "0002_alter_member_status"),
     ]

src/app/common/migrations/0004_auto_20220113_1314.py

@@ -6,7 +6,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("common", "0003_auto_20220112_1454"),
     ]

src/app/engine/tasks.py

@@ -26,6 +26,7 @@ def cleanup_scratch(folder_path):
 
 def process_member(args):
     member = Member.objects.get(id=args["id"])
+    folder_path = None
     try:
         folder_path = __fetch_origin(member, member.json_data["origin"])
         images = __rasterize_composite(member, folder_path)
@@ -36,11 +37,12 @@ def process_member(args):
     except Exception as error:
         __process_error(member, error)
     finally:
-        async_task(
-            "app.engine.tasks.cleanup_scratch",
-            folder_path,
-            task_name="Scavenger: [{0}]".format(args["id"]),
-        )
+        if folder_path:
+            async_task(
+                "app.engine.tasks.cleanup_scratch",
+                folder_path,
+                task_name="Scavenger: [{0}]".format(args["id"]),
+            )
 
 
 def __fetch_origin(member, origin_uri):

src/app/settings.py

@@ -123,7 +123,8 @@
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/3.2/howto/static-files/
 
-STATIC_URL = "/static/"
+STATIC_URL = "static/"
+STATIC_ROOT = BASE_DIR.parent / "dlcs/app_static"
 
 # Default primary key field type
 # https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field

src/requirements.txt

@@ -1,17 +1,18 @@
 asgiref==3.7.2
 attrs==23.1.0
-boto3==1.26.144
-Django==3.2.19
-django-environ==0.10.0
-django-health-check==3.16.4
+boto3==1.29.4
+Django==4.2.7
+django-environ==0.11.2
+django-health-check==3.17.0
 django-q==1.3.9
-djangorestframework==3.12.4
-jsonschema==4.2.1
+djangorestframework==3.14.0
+gunicorn==21.2.0
+jsonschema==4.20.0
 pdf2image==1.16.3
-psutil==5.9.5
-psycopg2-binary==2.9.6
-pyrsistent==0.18.0
+psutil==5.9.6
+psycopg2-binary==2.9.9
+pyrsistent==0.20.0
 pytz==2021.3
-requests==2.26.0
+requests==2.31.0
 sqlparse==0.4.4
-tqdm==4.65.0
+tqdm==4.66.1