From 753081baf84f406ba6c22967cf6f56a5343fb094 Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Wed, 9 Aug 2023 16:02:57 +0100 Subject: [PATCH 01/14] initial commit --- .gitignore | 165 +++++++++++++++++++ etc/default.vcl | 46 ++++-- start.sh | 4 +- varnish-cleanup/Test/Json/FullRequest.json | 60 +++++++ varnish-cleanup/Test/conftest.py | 23 +++ varnish-cleanup/Test/test_cleanup_handler.py | 75 +++++++++ varnish-cleanup/app/aws_factory.py | 35 ++++ varnish-cleanup/app/settings.py | 18 ++ varnish-cleanup/app/signal_handler.py | 27 +++ varnish-cleanup/cleanup_handler.py | 84 ++++++++++ varnish-cleanup/requirements.txt | 77 +++++++++ 11 files changed, 601 insertions(+), 13 deletions(-) create mode 100644 .gitignore create mode 100644 varnish-cleanup/Test/Json/FullRequest.json create mode 100644 varnish-cleanup/Test/conftest.py create mode 100644 varnish-cleanup/Test/test_cleanup_handler.py create mode 100644 varnish-cleanup/app/aws_factory.py create mode 100644 varnish-cleanup/app/settings.py create mode 100644 varnish-cleanup/app/signal_handler.py create mode 100644 varnish-cleanup/cleanup_handler.py create mode 100644 varnish-cleanup/requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d4a2f95 --- /dev/null +++ b/.gitignore @@ -0,0 +1,165 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +Scripts/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +pyvenv.cfg + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +.idea/ + +# General +*.exe \ No newline at end of file diff --git a/etc/default.vcl b/etc/default.vcl index 1c86f61..7556fba 100644 --- a/etc/default.vcl +++ b/etc/default.vcl @@ -1,13 +1,24 @@ +vcl 4.1; # See: https://www.varnish-software.com/developers/tutorials/#vcl backend default { - .host = "${BACKEND_HOST}"; - .port = "${BACKEND_PORT}"; + .host = "127.0.0.1"; + .port = "80"; } sub vcl_recv { - set req.backend = default; - return(lookup); + #set req.backend = default; + + if (req.method == "BAN") { + + ban("obj.http.x-cache-tags ~ " + req.url); + + # Throw a synthetic page so the + # request won't go to the backend. + return(synth(200, "Ban added")); + } + + return(pass); } sub vcl_miss { @@ -18,17 +29,28 @@ sub vcl_hit { return(deliver); } -sub vcl_fetch { - # Get the response. Set the cache lifetime of the response to 1 hour. - set beresp.ttl = 1h; +# sub vcl_fetch { +# # Get the response. Set the cache lifetime of the response to 1 hour. +# set beresp.ttl = 1h; - # Indicate that this response is cacheable. This is important. - set beresp.http.X-Cacheable = "YES"; +# # Indicate that this response is cacheable. This is important. +# set beresp.http.X-Cacheable = "YES"; - # Now pass this backend response along to the cache to be stored and served. - return(deliver); -} +# # Now pass this backend response along to the cache to be stored and served. +# return(deliver); +# } sub vcl_deliver { + + # Add debug header to see if it's a HIT/MISS and the number of hits, disable when not needed + if (obj.hits > 0) { + set resp.http.X-Cache = "HIT"; + } else { + set resp.http.X-Cache = "MISS"; + } + + # Set hits in X-Cache-Hits header + set resp.http.X-Cache-Hits = obj.hits; + return(deliver); } diff --git a/start.sh b/start.sh index 41162e8..717a324 100755 --- a/start.sh +++ b/start.sh @@ -1,6 +1,8 @@ #!/bin/bash -aws s3 cp ${S3_VCL_FILE} /etc/varnish/default.vcl +#aws s3 cp ${S3_VCL_FILE} /etc/varnish/new.vcl + +cp /mnt/varnish/default.vcl /etc/varnish/default.vcl RELOAD_VCL=1 diff --git a/varnish-cleanup/Test/Json/FullRequest.json b/varnish-cleanup/Test/Json/FullRequest.json new file mode 100644 index 0000000..c05fb37 --- /dev/null +++ b/varnish-cleanup/Test/Json/FullRequest.json @@ -0,0 +1,60 @@ +{ + "asset": { + "batch": 0, + "created": "2023-08-07T09:27:35.487625Z", + "customer": 26, + "deliveryChannels": [ + "iiif-img", + "iiif-av", + "file" + ], + "duration": 0, + "error": "", + "family": 73, + "finished": "2023-08-07T09:27:37.248125Z", + "fullImageOptimisationPolicy": { + "customer": 0, + "global": false, + "id": null, + "name": null, + "technicalDetails": null + }, + "fullThumbnailPolicy": null, + "height": 4149, + "id": { + "asset": "54378677", + "customer": 26, + "space": 18 + }, + "imageOptimisationPolicy": "fast-higher", + "ingesting": false, + "initialOrigin": null, + "maxUnauthorised": -1, + "mediaType": "image/jpg", + "notForDelivery": false, + "numberReference1": 0, + "numberReference2": 0, + "numberReference3": 0, + "origin": "https://www.nasa.gov/sites/default/files/thumbnails/image/as11-44-6552.jpeg", + "preservedUri": "", + "reference1": "", + "reference2": "", + "reference3": "", + "requiresAuth": false, + "roles": "", + "rolesList": [ + ], + "space": 18, + "tags": "sb", + "tagsList": [ + "sb" + ], + "thumbnailPolicy": "default", + "width": 4149 + }, + "customerPathElement": { + "id": 26, + "name": "jacklewis" + } + } + diff --git a/varnish-cleanup/Test/conftest.py b/varnish-cleanup/Test/conftest.py new file mode 100644 index 0000000..8c29465 --- /dev/null +++ b/varnish-cleanup/Test/conftest.py @@ -0,0 +1,23 @@ +import os +import boto3 +from moto import mock_sqs +import cleanup_handler +import pytest +@pytest.fixture(scope="function") +def aws_credentials(): + """Mocked AWS Credentials for moto.""" + os.environ["AWS_ACCESS_KEY_ID"] = "testing" + os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" + os.environ["AWS_SECURITY_TOKEN"] = "testing" + os.environ["AWS_SESSION_TOKEN"] = "testing" + os.environ["AWS_DEFAULT_REGION"] = "us-east-1" + +@pytest.fixture(scope="function") +def sqs(aws_credentials): + with mock_sqs(): + yield boto3.resource("sqs", region_name="us-east-1") + +@pytest.fixture(scope="function") +def sqs_client(aws_credentials): + with mock_sqs(): + yield boto3.client("sqs", region_name="us-east-1") \ No newline at end of file diff --git a/varnish-cleanup/Test/test_cleanup_handler.py b/varnish-cleanup/Test/test_cleanup_handler.py new file mode 100644 index 0000000..e221586 --- /dev/null +++ b/varnish-cleanup/Test/test_cleanup_handler.py @@ -0,0 +1,75 @@ +import os +import pytest +import requests_mock + +import boto3 +from time import sleep +from threading import Thread +from moto import mock_sqs +from unittest.mock import patch +import cleanup_handler +import mock +import json +from app.signal_handler import SignalHandler +from threading import Thread + +@mock_sqs +def test_write_message_valid(sqs): + "Test the start_monitoring_queue method with a valid message" + # Arrange + name = 'test-delete-notifications' + queue = sqs.create_queue(QueueName=name) + cleanup_handler.INCOMING_QUEUE = name + cleanup_handler.MONITOR_SLEEP_SECS = 1 + with open('Json/FullRequest.json', 'r') as file: + data = file.read().replace('\n', '') + queue.send_message(QueueUrl=queue.url, MessageBody=(data)) + signal_handler = SignalHandler() + + # Act + t = Thread(target=cleanup_handler.start_monitoring_queue, args=(sqs, signal_handler,)) + t.start() + # wait for messages to be handled + sleep(2) + + # Assert + sqs_messages = queue.receive_messages() + assert len(sqs_messages) == 0, "queue should have no messages" + + t.join(0.1) + +def test_receive_message_bans(): + "Test the receive_message method with a valid message" + # Arrange + with open('Json/FullRequest.json', 'r') as file: + data = file.read().replace('\n', '') + + sqs_mock_message = mock.Mock() + sqs_mock_message.body = data + + with requests_mock.Mocker() as mo: + mo.request("BAN", url="http://localhost:65345/26/18/54378677") + + # Act + response = cleanup_handler._handle_message(sqs_mock_message) + + # Assert + assert response == True, "response should be a success" + +def test_receive_message_bans_fail(): + "Test the receive_message method with a valid message" + # Arrange + with open('Json/FullRequest.json', 'r') as file: + data = file.read().replace('\n', '') + + sqs_mock_message = mock.Mock() + sqs_mock_message.body = data + + with requests_mock.Mocker() as mo: + mo.request("BAN", url="http://localhost:65345/26/18/54378677", status_code=400) + + # Act + response = cleanup_handler._handle_message(sqs_mock_message) + + # Assert + assert response == False, "response should be a failure" \ No newline at end of file diff --git a/varnish-cleanup/app/aws_factory.py b/varnish-cleanup/app/aws_factory.py new file mode 100644 index 0000000..10ff5b4 --- /dev/null +++ b/varnish-cleanup/app/aws_factory.py @@ -0,0 +1,35 @@ +import boto3 +from moto import sqs + +from app.settings import LOCALSTACK, REGION, LOCALSTACK_ADDRESS +from logzero import logger + + +def get_aws_client(client_type: str): + """Get an aws client configured to use LocalStack if env var is set""" + if LOCALSTACK: + logger.warn(f"Using localstack for {client_type} client") + return boto3.client( + client_type, + region_name=REGION, + endpoint_url=LOCALSTACK_ADDRESS, + aws_access_key_id="foo", + aws_secret_access_key="bar", + ) + else: + return boto3.client(client_type, REGION) + + +def get_aws_resource(resource_type: str): + """Get an aws resource configured to use LocalStack if env var is set""" + if LOCALSTACK: + logger.warn(f"Using localstack for {resource_type} resource") + return boto3.resource( + resource_type, + region_name=REGION, + endpoint_url=LOCALSTACK_ADDRESS, + aws_access_key_id="foo", + aws_secret_access_key="bar", + ) + else: + return boto3.resource(resource_type, REGION) \ No newline at end of file diff --git a/varnish-cleanup/app/settings.py b/varnish-cleanup/app/settings.py new file mode 100644 index 0000000..cce4e5d --- /dev/null +++ b/varnish-cleanup/app/settings.py @@ -0,0 +1,18 @@ +import os + +def _get_boolean(env_name: str, fallback: str) -> bool: + return os.environ.get(env_name, fallback).lower() in ("true", "t", "1") + +MONITOR_SLEEP_SECS = float(os.environ.get("MONITOR_SLEEP_SECS", 30)) + +# AWS +REGION = os.environ.get("AWS_REGION", "eu-west-1") +INCOMING_QUEUE = os.environ.get("INCOMING_QUEUE") +COMPLETED_TOPIC_ARN = os.environ.get("COMPLETED_TOPIC_ARN") + +# LocalStack +LOCALSTACK = _get_boolean("LOCALSTACK", "False") +LOCALSTACK_ADDRESS = os.environ.get("LOCALSTACK_ADDRESS", "http://localhost:4566") + +# varnish +VARNISH_ADDRESS = os.environ.get("VARNISH_ADDRESS", "http://localhost:65345") \ No newline at end of file diff --git a/varnish-cleanup/app/signal_handler.py b/varnish-cleanup/app/signal_handler.py new file mode 100644 index 0000000..614e00e --- /dev/null +++ b/varnish-cleanup/app/signal_handler.py @@ -0,0 +1,27 @@ +import signal + +from logzero import logger + + +class SignalHandler: + """Handles sigterm and sigint events""" + + def __init__(self): + self._cancellation_requested = False + self._setup_signal_handling() + + def cancellation_requested(self): + """ + Verify if lifecycle is to continue + :return: True if cancellation requested, else False + """ + return self._cancellation_requested + + def _signal_handler(self, signum, frame): + logger.info(f"Caught signal {signum}. Cancellation requested") + self._cancellation_requested = True + + def _setup_signal_handling(self): + logger.info("setting up signal handling") + signal.signal(signal.SIGTERM, self._signal_handler) + signal.signal(signal.SIGINT, self._signal_handler) \ No newline at end of file diff --git a/varnish-cleanup/cleanup_handler.py b/varnish-cleanup/cleanup_handler.py new file mode 100644 index 0000000..d95ce2a --- /dev/null +++ b/varnish-cleanup/cleanup_handler.py @@ -0,0 +1,84 @@ +import json +import traceback +import time +import requests + +from logzero import logger +from app.aws_factory import get_aws_resource +from app.settings import INCOMING_QUEUE, MONITOR_SLEEP_SECS, COMPLETED_TOPIC_ARN, VARNISH_ADDRESS +from app.signal_handler import SignalHandler + + +def start_monitoring(): + sqs = get_aws_resource("sqs") + signal_handler = SignalHandler() + start_monitoring_queue(sqs, signal_handler) + + +def start_monitoring_queue(sqs, signal_handler): + incoming_queue = sqs.get_queue_by_name(QueueName=INCOMING_QUEUE) + + logger.info(f"starting monitoring queue '{INCOMING_QUEUE}'") + + try: + while not signal_handler.cancellation_requested(): + message_received = False + for message in _get_messages_from_queue(incoming_queue): + if message and not signal_handler.cancellation_requested(): + message_received = True + try: + if _handle_message(message): + message.delete() + except Exception: + e = traceback.format_exc() + logger.error(f"Error processing message: {e}") + else: + message_received = False + + if not message_received and not signal_handler.cancellation_requested(): + time.sleep(MONITOR_SLEEP_SECS) + except Exception as e: + logger.error(f"Error getting messages: {e}") + raise e + + logger.info(f"stopped monitoring queue '{INCOMING_QUEUE}'...") + + +def _get_messages_from_queue(queue): + return queue.receive_messages(WaitTimeSeconds=20, MaxNumberOfMessages=10) + + +def _handle_message(received_message): + logger.debug(received_message) + message = json.loads(received_message.body) + delivery_channels = message["asset"]["deliveryChannels"] + + id = _convert_asset_id(message["asset"]["id"], message["customerPathElement"]["name"]) + success = True + + varnishUrl = f"{VARNISH_ADDRESS}/{id}" + + response = requests.request( + "BAN", + url=varnishUrl + ) + + if response.ok: + logger.debug(f"banned {id}") + else: + success = False + logger.error(f"failed to ban {id} - {response.content}") + + return success + + +def _convert_asset_id(id, name): + customer = id["customer"] + space = id["space"] + asset = id["asset"] + + return str(customer) + "/" + str(space) + "/" + str(asset) + + +if __name__ == "__main__": + start_monitoring() diff --git a/varnish-cleanup/requirements.txt b/varnish-cleanup/requirements.txt new file mode 100644 index 0000000..dbadad7 --- /dev/null +++ b/varnish-cleanup/requirements.txt @@ -0,0 +1,77 @@ +argcomplete==3.1.1 +aws_xray_sdk==2.12.0 +awscrt==0.17.0 +brotli==1.0.9 +brotlicffi==1.0.9.2 +cffi==1.15.1 +cfn-lint==0.79.6 +ConfigParser==6.0.0 +contextlib2==21.6.0 +crc32c==2.3.post0 +Cython==3.0.0 +defusedexpat==0.4 +dl==0.1.0 +docker_py==1.10.6 +docutils==0.20.1 +exceptiongroup==1.1.2 +Flask==2.3.2 +flask_cors==4.0.0 +graphql==0.0.4 +HTMLParser==0.0.2 +ipython==8.14.0 +ipywidgets==8.1.0 +jnius==1.1.0 +jose==1.0.0 +jsondiff==2.0.0 +keyring==24.2.0 +lockfile==0.12.2 +numpy==1.25.2 +openapi_spec_validator==0.6.0 +pexpect==4.8.0 +Pillow==10.0.0 +protobuf==4.24.0 +py_partiql_parser==0.3.7 +pyOpenSSL==23.2.0 +python_bcrypt==0.3.2 +railroad==0.5.0 +simplejson==3.19.1 +Sphinx==7.1.2 +sshpubkeys==3.3.1 +tomli_w==1.0.0 +tornado==6.3.2 +trove_classifiers==2023.8.7 +truststore==0.7.0 +Twisted==22.10.0 +urllib3_secure_extra==0.1.0 +watchdog==3.0.0 +xmlrpclib==1.0.1 +zope==5.8.3 + +pip~=23.1.2 +attrs~=23.1.0 +DateTime~=5.2 +argparse~=1.4.0 +cryptography~=41.0.3 +Jinja2~=3.1.2 +setuptools~=65.5.0 +packaging~=23.1 +zipp~=3.16.2 +boto3~=1.26.161 +logzero~=1.7.0 +requests~=2.31.0 +botocore~=1.29.161 +python-dateutil~=2.8.2 +PyYAML~=6.0 +Werkzeug~=2.3.6 +urllib3~=1.26.16 +docker-py~=1.10.6 +s3transfer~=0.6.1 +jmespath~=1.0.1 +Pygments~=2.16.1 +MarkupSafe~=2.1.3 +wcwidth~=0.2.6 +certifi~=2023.7.22 +colorama~=0.4.6 +six~=1.16.0 +idna~=3.4 +pycparser~=2.21 \ No newline at end of file From c3a729539c3ccb630eef2b519decede8608493f9 Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Wed, 9 Aug 2023 16:22:57 +0100 Subject: [PATCH 02/14] updating requirements.txt --- varnish-cleanup/requirements.txt | Bin 1285 -> 1150 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/varnish-cleanup/requirements.txt b/varnish-cleanup/requirements.txt index dbadad7f8ba7bae85ee598941bf766ca56811f5e..9da4e97556cc8cfc2e5fc4a06027deab5f79c642 100644 GIT binary patch literal 1150 zcmaKsPfx;75XAS)cY)A?s0RCc)6aTqt%MSX=-E=A1XH1GwI@QUOBZ= z-^7Cpoaouw-Wpe4I!YrJd(RuYHqb`OywK@Jv0A(m3u3T%)G5w@lNd}CL+@PCs$&N8 zv7c2sr=oH%O0Nn{fq1ESWiP7B58YxWJ)N$kWUX#fXGBM0u*rmuA7$rP?bSq44cBW` zwYIRW>TmV?sdI(y`RX!d~h#*G>QV4-kF0Nq63V;d{lTm|^IT fo3@kMrTW%#nce=r37HT>_1f&U487~WRPaYW+Jg_nhG7_5tUwD4I~j^hJ8Be3mENXik&*NQI0(cM|&O8)jiZp2sbMl(IDm?(DL230Ddy~}h)*-(0x&kUZb2c_{M1xKQoi&AQyqlt* zLx}_R?^dPKc?)A?qt`wnYgI6(sKWlez*KT2e!M9kYF)B^o!uTuB~-MNgx#hzI=!SZ+RE8*J9Fd?2TS z=~E!x|5eruDticw=8+sA6KUwQlJ_v5jkW#+WM++5jMVRDKm&?thouiHX7g_Ow+yPn z7`Ax$1)P_cN6>;qC6tHK?{r*3nFA{qm9f#Qf#KH(N{080J1m>_0R3OW-netuZ6QP> zQrd@khn68t3#3|Pf3&7&j)fw6acc0qFbt5R|4Y5bfRY&v@r0TRjPAiP$2sMMgd!EM z?CZ&-OkmSsrMclbTI^ceC(?7I9;+8s(G_nemGyq`5dOMn=cA25Bm7FG79Wth>(;<( zg$O2GSPE`JC;y>D3(Rf>6B0sVI3tKdf+(%a?sxF5QZoY{gIG-c)lb#|TX* z&mntnQFF5{ItHZ=YQk1rQN)J{fCu^p5IYR9<@(JAjv5OyeNgo5VG$y|&jewRDqCJY zj>mF?AW|?Aq+GZ64y!|aEgX7}%FoN6FHetPfvn#Y4<9r$159G`Ddm6M+wI*2v~|0^ z)IwgDMv9lRj Date: Thu, 10 Aug 2023 09:42:29 +0100 Subject: [PATCH 03/14] updating build process --- .github/actions/action.yml | 53 ++++++++++++++++++++++++++++ .github/workflows/build-deploy.yml | 56 ++++++++++-------------------- varnish-cleanup/Dockerfile | 23 ++++++++++++ 3 files changed, 95 insertions(+), 37 deletions(-) create mode 100644 .github/actions/action.yml create mode 100644 varnish-cleanup/Dockerfile diff --git a/.github/actions/action.yml b/.github/actions/action.yml new file mode 100644 index 0000000..d80e72f --- /dev/null +++ b/.github/actions/action.yml @@ -0,0 +1,53 @@ +name: Docker Build & Push +description: Composite GitHub Action to build and push Docker images to the DLCS GitHub Packages repositories. + +inputs: + image-name: + description: "Name of the image to push to the GHCR repository." + required: true + dockerfile: + description: "The Dockerfile to build and push." + required: true + context: + description: "The context to use when building the Dockerfile." + required: true + github-token: + description: "The GitHub token used when interacting with GCHR." + required: true + +runs: + using: "composite" + steps: + - id: checkout + uses: actions/checkout@v2 + - id: docker-setup-buildx + uses: docker/setup-buildx-action@v2 + with: + driver-opts: | + image=moby/buildkit:v0.10.6 + - id: docker-meta + uses: docker/metadata-action@v3 + with: + images: ghcr.io/dlcs/${{ inputs.image-name }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=sha,enable=true,prefix=,format=long + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + - id: docker-login + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ inputs.github-token }} + - id: docker-build-push + uses: docker/build-push-action@v2 + with: + context: ${{ inputs.context }} + file: ${{ inputs.dockerfile }} + builder: ${{ steps.docker-setup-buildx.outputs.name }} + tags: ${{ steps.docker-meta.outputs.tags }} + labels: ${{ steps.docker-meta.outputs.labels }} + push: ${{ github.actor != 'dependabot[bot]' }} diff --git a/.github/workflows/build-deploy.yml b/.github/workflows/build-deploy.yml index 9c3bd76..3f696d2 100644 --- a/.github/workflows/build-deploy.yml +++ b/.github/workflows/build-deploy.yml @@ -11,45 +11,27 @@ on: - master jobs: - build-push: + build-push-varnish: runs-on: ubuntu-latest - + needs: test-dotnet steps: - - name: Check out code - id: checkout - uses: actions/checkout@v2 - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + - uses: actions/checkout@v3 + - uses: ./.github/actions/docker-build-and-push + name: build and push with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + image-name: "dlcs-varnish" + dockerfile: "Dockerfile" + context: "." + github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Docker meta - id: docker_meta - uses: docker/metadata-action@v4 - with: - images: ghcr.io/dlcs/dlcs-varnish - tags: | - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=sha,enable=true,prefix=,format=long - - - name: Build and push - id: docker_build - uses: docker/build-push-action@v4 + build-push-thumbs: + runs-on: ubuntu-latest + needs: test-dotnet + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/docker-build-and-push with: - context: . - builder: ${{ steps.buildx.outputs.name }} - push: true - labels: ${{ steps.docker_meta.outputs.labels }} - tags: ${{ steps.docker_meta.outputs.tags }} - - \ No newline at end of file + image-name: "dlcs-varnish-cleanup" + dockerfile: "Dockerfile" + context: "./varnish-cleanup" + github-token: ${{ secrets.GITHUB_TOKEN } diff --git a/varnish-cleanup/Dockerfile b/varnish-cleanup/Dockerfile new file mode 100644 index 0000000..68055c2 --- /dev/null +++ b/varnish-cleanup/Dockerfile @@ -0,0 +1,23 @@ +#Deriving the latest base image +FROM python:latest + + +#Labels as key value pair +LABEL Maintainer="digirati" + +RUN pip install pipenv +WORKDIR /usr/app/src +COPY requirements.txt ./ +#RUN pipenv lock > requirements.txt +RUN pip install -r requirements.txt + +#to COPY the remote file at working directory in container +COPY cleanup_handler.py ./ +COPY app ./app +# Now the structure looks like this '/usr/app/src/test.py' + + +#CMD instruction should be used to run the software +#contained by your image, along with any arguments. + +CMD [ "python", "./cleanup_handler.py"] \ No newline at end of file From fad58cf62e9e7febbcd128277ddd6c775d5199f9 Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Thu, 10 Aug 2023 09:43:53 +0100 Subject: [PATCH 04/14] closing off GitHub token --- .github/workflows/build-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-deploy.yml b/.github/workflows/build-deploy.yml index 3f696d2..27912c4 100644 --- a/.github/workflows/build-deploy.yml +++ b/.github/workflows/build-deploy.yml @@ -34,4 +34,4 @@ jobs: image-name: "dlcs-varnish-cleanup" dockerfile: "Dockerfile" context: "./varnish-cleanup" - github-token: ${{ secrets.GITHUB_TOKEN } + github-token: ${{ secrets.GITHUB_TOKEN }} From 6c41ca1ecd9b4dca04d8a002292924709bfa9f3e Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Thu, 10 Aug 2023 09:45:02 +0100 Subject: [PATCH 05/14] removing unused needs --- .github/workflows/build-deploy.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build-deploy.yml b/.github/workflows/build-deploy.yml index 27912c4..1d1d9ae 100644 --- a/.github/workflows/build-deploy.yml +++ b/.github/workflows/build-deploy.yml @@ -13,7 +13,6 @@ on: jobs: build-push-varnish: runs-on: ubuntu-latest - needs: test-dotnet steps: - uses: actions/checkout@v3 - uses: ./.github/actions/docker-build-and-push @@ -26,7 +25,6 @@ jobs: build-push-thumbs: runs-on: ubuntu-latest - needs: test-dotnet steps: - uses: actions/checkout@v3 - uses: ./.github/actions/docker-build-and-push From 672217b1dc53fe066f8e86a624699142fa7348f7 Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Thu, 10 Aug 2023 09:46:42 +0100 Subject: [PATCH 06/14] moving file location --- .github/actions/{ => docker-build-and-push}/action.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/actions/{ => docker-build-and-push}/action.yml (100%) diff --git a/.github/actions/action.yml b/.github/actions/docker-build-and-push/action.yml similarity index 100% rename from .github/actions/action.yml rename to .github/actions/docker-build-and-push/action.yml From 1d8a2d0cbb26a384fc5f22a8596de388ef9f1960 Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Thu, 10 Aug 2023 09:52:56 +0100 Subject: [PATCH 07/14] updating to fix dockerfile locaction --- .github/workflows/build-deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-deploy.yml b/.github/workflows/build-deploy.yml index 1d1d9ae..3c5d81d 100644 --- a/.github/workflows/build-deploy.yml +++ b/.github/workflows/build-deploy.yml @@ -23,13 +23,13 @@ jobs: context: "." github-token: ${{ secrets.GITHUB_TOKEN }} - build-push-thumbs: + build-push-dlcs-varnish-cleanup: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - uses: ./.github/actions/docker-build-and-push with: image-name: "dlcs-varnish-cleanup" - dockerfile: "Dockerfile" + dockerfile: "varnish-cleanup/Dockerfile" context: "./varnish-cleanup" github-token: ${{ secrets.GITHUB_TOKEN }} From 5ad2694053b9f35847e9262835b30a82f6a679fd Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Thu, 10 Aug 2023 11:33:30 +0100 Subject: [PATCH 08/14] getting the python code to run in the varnish container --- Dockerfile | 8 ++++++++ etc/default.vcl | 17 ++++++++--------- start.sh | 6 +++++- varnish-cleanup/Dockerfile | 9 ++------- 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index d3cefb4..bd7f593 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,14 @@ RUN pip install awscli COPY start.sh /start.sh RUN chmod +x /start.sh +RUN pip install pipenv +WORKDIR /usr/app/src +COPY varnish-cleanup/requirements.txt ./ +RUN pip install -r requirements.txt + +COPY varnish-cleanup/cleanup_handler.py ./ +COPY varnish-cleanup/app ./app + ENV VARNISH_PORT 80 EXPOSE 80 diff --git a/etc/default.vcl b/etc/default.vcl index 7556fba..8288723 100644 --- a/etc/default.vcl +++ b/etc/default.vcl @@ -29,19 +29,18 @@ sub vcl_hit { return(deliver); } -# sub vcl_fetch { -# # Get the response. Set the cache lifetime of the response to 1 hour. -# set beresp.ttl = 1h; +sub vcl_backend_response { + # Get the response. Set the cache lifetime of the response to 1 hour. + set beresp.ttl = 1h; -# # Indicate that this response is cacheable. This is important. -# set beresp.http.X-Cacheable = "YES"; + # Indicate that this response is cacheable. This is important. + set beresp.http.X-Cacheable = "YES"; -# # Now pass this backend response along to the cache to be stored and served. -# return(deliver); -# } + # Now pass this backend response along to the cache to be stored and served. + return(deliver); +} sub vcl_deliver { - # Add debug header to see if it's a HIT/MISS and the number of hits, disable when not needed if (obj.hits > 0) { set resp.http.X-Cache = "HIT"; diff --git a/start.sh b/start.sh index 717a324..8ad5817 100755 --- a/start.sh +++ b/start.sh @@ -12,4 +12,8 @@ mkdir -p ${VARNISH_CACHE_FOLDER} varnishd -a 0.0.0.0:${VARNISH_PORT} -T 127.0.0.1:6082 -f /etc/varnish/default.vcl -s file,${VARNISH_CACHE_FOLDER}/varnish_cache.bin,${VARNISH_CACHE_SIZE} -varnishlog +varnishlog & + +# Start varnish cleanup + +python3 /usr/app/src/cleanup_handler.py diff --git a/varnish-cleanup/Dockerfile b/varnish-cleanup/Dockerfile index 68055c2..215e6cb 100644 --- a/varnish-cleanup/Dockerfile +++ b/varnish-cleanup/Dockerfile @@ -4,20 +4,15 @@ FROM python:latest #Labels as key value pair LABEL Maintainer="digirati" +LABEL org.opencontainers.image.source=https://github.com/dlcs/dlcs-varnish-cleanup +LABEL org.opencontainers.image.description="Performs bans on varnish" RUN pip install pipenv WORKDIR /usr/app/src COPY requirements.txt ./ -#RUN pipenv lock > requirements.txt RUN pip install -r requirements.txt -#to COPY the remote file at working directory in container COPY cleanup_handler.py ./ COPY app ./app -# Now the structure looks like this '/usr/app/src/test.py' - - -#CMD instruction should be used to run the software -#contained by your image, along with any arguments. CMD [ "python", "./cleanup_handler.py"] \ No newline at end of file From 3b564a6267bb77b106d41e97794e546ca606e627 Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Thu, 10 Aug 2023 12:04:52 +0100 Subject: [PATCH 09/14] updating readme, getting start.sh to use a local file and updating default.vcl --- README.md | 44 ++++++++++++++++++++++++++++----- etc/default.vcl | 2 +- start.sh | 11 ++++++--- varnish-cleanup/app/settings.py | 2 +- 4 files changed, 48 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 94b4dd9..2f1a1be 100644 --- a/README.md +++ b/README.md @@ -2,17 +2,25 @@ Docker image using disk-backed Varnish instance for caching. -On startup it uses the AWS CLI to copy vcl file from location specified by `S3_VCL_FILE` environment variable. +On startup it uses the AWS CLI to copy vcl file from location specified by `S3_VCL_FILE` environment variable. +Optionally, a local file can also be configured for development purposes ## Configuration -The following environment files are expected: - +The following environment settings are expected: * `S3_VCL_FILE` - The location of a vcl file to use. Expected S3Uri as it is used by [aws s3 cp](https://docs.aws.amazon.com/cli/latest/reference/s3/cp.html) command. * `VARNISH_CACHE_FOLDER` - Folder where disk backed cache is stored. * `VARNISH_CACHE_SIZE` - Size of cache. * `VARNISH_PORT` - Which port Varnish should listen on (defaults to 80) +* `AWS_PROFILE` - Required to run locally + +The following configuration is optional: +* `VARNISH_ADDRESS` - The location of varnish used by the cleanup handler. Defaults to localhost +* `AWS_REGION` - The region used by the cleanup handler. Defaults to eu-west-1 +* `USE_LOCAL_CONFIG` - Whether to use a local config file over S3. + +*NOTE:* using `USE_LOCAL_CONFIG` requires a `mount`to be added to the `docker run` containing the VCL ## Running ```bash @@ -21,10 +29,34 @@ docker build -t dlcs-varnish:local . # run docker run -it --rm \ - --env AWS_ACCESS_KEY_ID='xxx' \ - --env AWS_SECRET_ACCESS_KEY='xxx' \ --env S3_VCL_FILE='s3://my-bucket/varnish-config.vcl' \ --env VARNISH_CACHE_FOLDER='/path/to/folder' \ - --env VARNISH_CACHE_SIZE='100M' + --env VARNISH_CACHE_SIZE='100M' \ + --env-file='/path/to/env' \ + {REQUIRED FOR LOCAL RUNNING}--volume $HOME\.aws\credentials:/root/.aws/credentials:ro \ + {OPTIONAL}--mount type=bind,source=.\etc\default.vcl,target=/mnt/varnish/default.vcl \ + dlcs-varnish:local +``` +# varnish-cleanup + +Additionally, there is a standalone docker container for the cleanup handler. + +## Configuration + +Required: +* `AWS_PROFILE` - Required to run locally + +Optional: +* `VARNISH_ADDRESS` - The location of varnish used by the cleanup handler. Defaults to localhost +* `AWS_REGION` - The region used by the cleanup handler. Defaults to eu-west-1 + +```bash +# build +docker build -t dlcs-varnish-cleanup:local ./varnish-cleanup + +# run +docker run -it --rm \ + --env-file='/path/to/env' + {REQUIRED FOR LOCAL RUNNING}--volume=$HOME\.aws\credentials:/root/.aws/credentials:ro dlcs-varnish:local ``` \ No newline at end of file diff --git a/etc/default.vcl b/etc/default.vcl index 8288723..dfbfb20 100644 --- a/etc/default.vcl +++ b/etc/default.vcl @@ -7,7 +7,7 @@ backend default { } sub vcl_recv { - #set req.backend = default; + set req.backend_hint = default; if (req.method == "BAN") { diff --git a/start.sh b/start.sh index 8ad5817..391e6f2 100755 --- a/start.sh +++ b/start.sh @@ -1,8 +1,13 @@ #!/bin/bash -#aws s3 cp ${S3_VCL_FILE} /etc/varnish/new.vcl - -cp /mnt/varnish/default.vcl /etc/varnish/default.vcl +if [ "$USE_LOCAL_CONFIG" = true ] +then + echo 'Using local config!' + cp /mnt/varnish/default.vcl /etc/varnish/default.vcl +else + echo 'Using s3 config!' + aws s3 cp ${S3_VCL_FILE} /etc/varnish/default.vcl +fi RELOAD_VCL=1 diff --git a/varnish-cleanup/app/settings.py b/varnish-cleanup/app/settings.py index cce4e5d..99dbcd4 100644 --- a/varnish-cleanup/app/settings.py +++ b/varnish-cleanup/app/settings.py @@ -15,4 +15,4 @@ def _get_boolean(env_name: str, fallback: str) -> bool: LOCALSTACK_ADDRESS = os.environ.get("LOCALSTACK_ADDRESS", "http://localhost:4566") # varnish -VARNISH_ADDRESS = os.environ.get("VARNISH_ADDRESS", "http://localhost:65345") \ No newline at end of file +VARNISH_ADDRESS = os.environ.get("VARNISH_ADDRESS", "http://localhost") \ No newline at end of file From 535738b0c4114f954d0e4c98cb0743fe0002e463 Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Thu, 10 Aug 2023 13:49:09 +0100 Subject: [PATCH 10/14] changing to return(hash) --- etc/default.vcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/default.vcl b/etc/default.vcl index dfbfb20..ec24baf 100644 --- a/etc/default.vcl +++ b/etc/default.vcl @@ -18,7 +18,7 @@ sub vcl_recv { return(synth(200, "Ban added")); } - return(pass); + return(hash); } sub vcl_miss { From d87f9a422c796b89c1039c55a4a61c5aea83e9dd Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Thu, 10 Aug 2023 13:53:12 +0100 Subject: [PATCH 11/14] removing reference to varnish part --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 2f1a1be..812ae28 100644 --- a/README.md +++ b/README.md @@ -11,13 +11,12 @@ The following environment settings are expected: * `S3_VCL_FILE` - The location of a vcl file to use. Expected S3Uri as it is used by [aws s3 cp](https://docs.aws.amazon.com/cli/latest/reference/s3/cp.html) command. * `VARNISH_CACHE_FOLDER` - Folder where disk backed cache is stored. * `VARNISH_CACHE_SIZE` - Size of cache. -* `VARNISH_PORT` - Which port Varnish should listen on (defaults to 80) * `AWS_PROFILE` - Required to run locally The following configuration is optional: * `VARNISH_ADDRESS` - The location of varnish used by the cleanup handler. Defaults to localhost -* `AWS_REGION` - The region used by the cleanup handler. Defaults to eu-west-1 +* `AWS_REGION` - The AWS region. Defaults to eu-west-1 * `USE_LOCAL_CONFIG` - Whether to use a local config file over S3. *NOTE:* using `USE_LOCAL_CONFIG` requires a `mount`to be added to the `docker run` containing the VCL From 95d847ea489bbb1904238cce1e9599313880ff00 Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Thu, 10 Aug 2023 14:35:12 +0100 Subject: [PATCH 12/14] removing unneeded link to pipenv --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index bd7f593..2fca67d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,6 @@ RUN pip install awscli COPY start.sh /start.sh RUN chmod +x /start.sh -RUN pip install pipenv WORKDIR /usr/app/src COPY varnish-cleanup/requirements.txt ./ RUN pip install -r requirements.txt From 687c690850d57446b5747c3f02cfbb1bf1d36144 Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Thu, 10 Aug 2023 16:00:36 +0100 Subject: [PATCH 13/14] code review changes --- .github/actions/docker-build-and-push/action.yml | 6 +++--- README.md | 1 + varnish-cleanup/Test/test_cleanup_handler.py | 4 ++-- varnish-cleanup/app/aws_factory.py | 16 ---------------- varnish-cleanup/app/settings.py | 1 - varnish-cleanup/cleanup_handler.py | 8 +++----- 6 files changed, 9 insertions(+), 27 deletions(-) diff --git a/.github/actions/docker-build-and-push/action.yml b/.github/actions/docker-build-and-push/action.yml index d80e72f..2e94251 100644 --- a/.github/actions/docker-build-and-push/action.yml +++ b/.github/actions/docker-build-and-push/action.yml @@ -26,7 +26,7 @@ runs: driver-opts: | image=moby/buildkit:v0.10.6 - id: docker-meta - uses: docker/metadata-action@v3 + uses: docker/metadata-action@v4 with: images: ghcr.io/dlcs/${{ inputs.image-name }} tags: | @@ -37,13 +37,13 @@ runs: type=semver,pattern={{major}}.{{minor}} type=semver,pattern={{major}} - id: docker-login - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ inputs.github-token }} - id: docker-build-push - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v4 with: context: ${{ inputs.context }} file: ${{ inputs.dockerfile }} diff --git a/README.md b/README.md index 812ae28..7dfcae4 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ The following environment settings are expected: * `VARNISH_CACHE_FOLDER` - Folder where disk backed cache is stored. * `VARNISH_CACHE_SIZE` - Size of cache. * `AWS_PROFILE` - Required to run locally +* `INCOMING_QUEUE` - the name of the queue that the cleanup handler listens to The following configuration is optional: diff --git a/varnish-cleanup/Test/test_cleanup_handler.py b/varnish-cleanup/Test/test_cleanup_handler.py index e221586..99db25c 100644 --- a/varnish-cleanup/Test/test_cleanup_handler.py +++ b/varnish-cleanup/Test/test_cleanup_handler.py @@ -48,7 +48,7 @@ def test_receive_message_bans(): sqs_mock_message.body = data with requests_mock.Mocker() as mo: - mo.request("BAN", url="http://localhost:65345/26/18/54378677") + mo.request("BAN", url="http://localhost/26/18/54378677") # Act response = cleanup_handler._handle_message(sqs_mock_message) @@ -66,7 +66,7 @@ def test_receive_message_bans_fail(): sqs_mock_message.body = data with requests_mock.Mocker() as mo: - mo.request("BAN", url="http://localhost:65345/26/18/54378677", status_code=400) + mo.request("BAN", url="http://localhost/26/18/54378677", status_code=400) # Act response = cleanup_handler._handle_message(sqs_mock_message) diff --git a/varnish-cleanup/app/aws_factory.py b/varnish-cleanup/app/aws_factory.py index 10ff5b4..f8564ed 100644 --- a/varnish-cleanup/app/aws_factory.py +++ b/varnish-cleanup/app/aws_factory.py @@ -4,22 +4,6 @@ from app.settings import LOCALSTACK, REGION, LOCALSTACK_ADDRESS from logzero import logger - -def get_aws_client(client_type: str): - """Get an aws client configured to use LocalStack if env var is set""" - if LOCALSTACK: - logger.warn(f"Using localstack for {client_type} client") - return boto3.client( - client_type, - region_name=REGION, - endpoint_url=LOCALSTACK_ADDRESS, - aws_access_key_id="foo", - aws_secret_access_key="bar", - ) - else: - return boto3.client(client_type, REGION) - - def get_aws_resource(resource_type: str): """Get an aws resource configured to use LocalStack if env var is set""" if LOCALSTACK: diff --git a/varnish-cleanup/app/settings.py b/varnish-cleanup/app/settings.py index 99dbcd4..034dba7 100644 --- a/varnish-cleanup/app/settings.py +++ b/varnish-cleanup/app/settings.py @@ -8,7 +8,6 @@ def _get_boolean(env_name: str, fallback: str) -> bool: # AWS REGION = os.environ.get("AWS_REGION", "eu-west-1") INCOMING_QUEUE = os.environ.get("INCOMING_QUEUE") -COMPLETED_TOPIC_ARN = os.environ.get("COMPLETED_TOPIC_ARN") # LocalStack LOCALSTACK = _get_boolean("LOCALSTACK", "False") diff --git a/varnish-cleanup/cleanup_handler.py b/varnish-cleanup/cleanup_handler.py index d95ce2a..2c3b617 100644 --- a/varnish-cleanup/cleanup_handler.py +++ b/varnish-cleanup/cleanup_handler.py @@ -5,7 +5,7 @@ from logzero import logger from app.aws_factory import get_aws_resource -from app.settings import INCOMING_QUEUE, MONITOR_SLEEP_SECS, COMPLETED_TOPIC_ARN, VARNISH_ADDRESS +from app.settings import INCOMING_QUEUE, MONITOR_SLEEP_SECS, VARNISH_ADDRESS from app.signal_handler import SignalHandler @@ -51,9 +51,7 @@ def _get_messages_from_queue(queue): def _handle_message(received_message): logger.debug(received_message) message = json.loads(received_message.body) - delivery_channels = message["asset"]["deliveryChannels"] - - id = _convert_asset_id(message["asset"]["id"], message["customerPathElement"]["name"]) + id = _convert_asset_id(message["asset"]["id"]) success = True varnishUrl = f"{VARNISH_ADDRESS}/{id}" @@ -72,7 +70,7 @@ def _handle_message(received_message): return success -def _convert_asset_id(id, name): +def _convert_asset_id(id): customer = id["customer"] space = id["space"] asset = id["asset"] From db2212a47493df0754ee2905d833a60cb42ecdf3 Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Fri, 11 Aug 2023 15:51:40 +0100 Subject: [PATCH 14/14] modifying default vcl to match a working config --- etc/default.vcl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/etc/default.vcl b/etc/default.vcl index ec24baf..24f7616 100644 --- a/etc/default.vcl +++ b/etc/default.vcl @@ -11,7 +11,9 @@ sub vcl_recv { if (req.method == "BAN") { - ban("obj.http.x-cache-tags ~ " + req.url); + set req.url = regsub(req.url, "^(\/)", ""); + + ban("obj.http.x-asset-id == " + req.url); # Throw a synthetic page so the # request won't go to the backend.