From 92753bfddf25572e854c36bff1f556665853fb7e Mon Sep 17 00:00:00 2001 From: Paul Wheatcroft Date: Thu, 13 Jun 2024 14:08:29 +0100 Subject: [PATCH 1/6] copilot preparation --- .copilot/config.yml | 7 +++++++ .copilot/image_build_run.sh | 9 +++++++++ 2 files changed, 16 insertions(+) create mode 100644 .copilot/config.yml create mode 100755 .copilot/image_build_run.sh diff --git a/.copilot/config.yml b/.copilot/config.yml new file mode 100644 index 00000000..a8dbff5b --- /dev/null +++ b/.copilot/config.yml @@ -0,0 +1,7 @@ +repository: fft +builder: + name: paketobuildpacks/builder-jammy-base + version: 0.4.240 +packages: + - libpq-dev + - libsqlite3-dev diff --git a/.copilot/image_build_run.sh b/.copilot/image_build_run.sh new file mode 100755 index 00000000..03fb3211 --- /dev/null +++ b/.copilot/image_build_run.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +# Exit early if something goes wrong +set -e + +# Add commands below to run inside the container after all the other buildpacks have been applied +export $(grep -v '^#' .env.ci | xargs) + +python manage.py collectstatic --noinput \ No newline at end of file From 948ba890283c4a6f41be431fd9fabf2420c00b4e Mon Sep 17 00:00:00 2001 From: Paul Wheatcroft Date: Thu, 13 Jun 2024 14:26:41 +0100 Subject: [PATCH 2/6] version change --- .copilot/config.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.copilot/config.yml b/.copilot/config.yml index a8dbff5b..152156bf 100644 --- a/.copilot/config.yml +++ b/.copilot/config.yml @@ -1,7 +1,4 @@ repository: fft builder: - name: paketobuildpacks/builder-jammy-base - version: 0.4.240 -packages: - - libpq-dev - - libsqlite3-dev + name: paketobuildpacks/builder-jammy-full + version: 0.3.339 From fdacb188d852ed092621993196f6736d5fe7b321 Mon Sep 17 00:00:00 2001 From: Paul Wheatcroft Date: Thu, 13 Jun 2024 15:45:35 +0100 Subject: [PATCH 3/6] if is_copilot database condition added --- config/settings/base.py | 53 +++++++++++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 13 deletions(-) diff --git a/config/settings/base.py b/config/settings/base.py index 1d5b36a8..58893118 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -13,7 +13,9 @@ import os from pathlib import Path +import dj_database_url import environ +from dbt_copilot_python.database import database_url_from_env from dbt_copilot_python.utility import is_copilot from django.urls import reverse_lazy from django_log_formatter_asim import ASIMFormatter @@ -107,12 +109,19 @@ VCAP_SERVICES = env.json("VCAP_SERVICES", default={}) -if "postgres" in VCAP_SERVICES: - DATABASE_URL = VCAP_SERVICES["postgres"][0]["credentials"]["uri"] +if is_copilot(): + DATABASES = { + "default": dj_database_url.config( + default=database_url_from_env("DATABASE_CREDENTIALS") + ) + } else: - DATABASE_URL = os.getenv("DATABASE_URL") + if "postgres" in VCAP_SERVICES: + DATABASE_URL = VCAP_SERVICES["postgres"][0]["credentials"]["uri"] + else: + DATABASE_URL = os.getenv("DATABASE_URL") -DATABASES = {"default": env.db()} + DATABASES = {"default": env.db()} DEFAULT_AUTO_FIELD = "django.db.models.AutoField" @@ -124,8 +133,12 @@ "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator" }, # noqa {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator"}, - {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator"}, - {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"}, + { + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator" + }, + { + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator" + }, ] # Internationalization @@ -186,16 +199,22 @@ def FILTERS_VERBOSE_LOOKUPS(): # If "temp" is in instance name it means it's the temp files bucket if "temp" in bucket["instance_name"]: - TEMP_FILE_AWS_ACCESS_KEY_ID = app_bucket_credentials["aws_access_key_id"] + TEMP_FILE_AWS_ACCESS_KEY_ID = app_bucket_credentials[ + "aws_access_key_id" + ] TEMP_FILE_AWS_SECRET_ACCESS_KEY = app_bucket_credentials[ "aws_secret_access_key" ] TEMP_FILE_AWS_REGION = app_bucket_credentials["aws_region"] TEMP_FILE_AWS_S3_REGION_NAME = app_bucket_credentials["aws_region"] - TEMP_FILE_AWS_STORAGE_BUCKET_NAME = app_bucket_credentials["bucket_name"] + TEMP_FILE_AWS_STORAGE_BUCKET_NAME = app_bucket_credentials[ + "bucket_name" + ] else: AWS_ACCESS_KEY_ID = app_bucket_credentials["aws_access_key_id"] - AWS_SECRET_ACCESS_KEY = app_bucket_credentials["aws_secret_access_key"] + AWS_SECRET_ACCESS_KEY = app_bucket_credentials[ + "aws_secret_access_key" + ] AWS_REGION = app_bucket_credentials["aws_region"] AWS_S3_REGION_NAME = app_bucket_credentials["aws_region"] AWS_STORAGE_BUCKET_NAME = app_bucket_credentials["bucket_name"] @@ -206,8 +225,12 @@ def FILTERS_VERBOSE_LOOKUPS(): AWS_S3_REGION_NAME = env("AWS_REGION", default="") AWS_STORAGE_BUCKET_NAME = env("AWS_STORAGE_BUCKET_NAME", default="") - TEMP_FILE_AWS_ACCESS_KEY_ID = env("TEMP_FILE_AWS_ACCESS_KEY_ID", default="") - TEMP_FILE_AWS_SECRET_ACCESS_KEY = env("TEMP_FILE_AWS_SECRET_ACCESS_KEY", default="") + TEMP_FILE_AWS_ACCESS_KEY_ID = env( + "TEMP_FILE_AWS_ACCESS_KEY_ID", default="" + ) + TEMP_FILE_AWS_SECRET_ACCESS_KEY = env( + "TEMP_FILE_AWS_SECRET_ACCESS_KEY", default="" + ) TEMP_FILE_AWS_REGION = env("TEMP_FILE_AWS_REGION", default="") TEMP_FILE_AWS_S3_REGION_NAME = env("TEMP_FILE_AWS_REGION", default="") TEMP_FILE_AWS_STORAGE_BUCKET_NAME = env( @@ -334,8 +357,12 @@ def FILTERS_VERBOSE_LOOKUPS(): # Vite VITE_DEV = env.bool("VITE_DEV", default=False) -VITE_DEV_SERVER_URL = env.str("VITE_DEV_SERVER_URL", default="http://localhost:5173") -VITE_MANIFEST_PATH = BASE_DIR / "front_end" / "build" / ".vite" / "manifest.json" +VITE_DEV_SERVER_URL = env.str( + "VITE_DEV_SERVER_URL", default="http://localhost:5173" +) +VITE_MANIFEST_PATH = ( + BASE_DIR / "front_end" / "build" / ".vite" / "manifest.json" +) # Selenium (BDD tests) USE_REMOTE_CHROME = env("USE_REMOTE_CHROME", default=True) From c18f947e4728482d7bf95a85fa5347381dd5e467 Mon Sep 17 00:00:00 2001 From: Paul Wheatcroft Date: Thu, 13 Jun 2024 16:19:17 +0100 Subject: [PATCH 4/6] Move redis setting to paas.py --- config/settings/base.py | 36 +++++++++--------------------------- config/settings/paas.py | 2 ++ 2 files changed, 11 insertions(+), 27 deletions(-) diff --git a/config/settings/base.py b/config/settings/base.py index 58893118..d2909245 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -133,12 +133,8 @@ "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator" }, # noqa {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator"}, - { - "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator" - }, - { - "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator" - }, + {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator"}, + {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"}, ] # Internationalization @@ -199,22 +195,16 @@ def FILTERS_VERBOSE_LOOKUPS(): # If "temp" is in instance name it means it's the temp files bucket if "temp" in bucket["instance_name"]: - TEMP_FILE_AWS_ACCESS_KEY_ID = app_bucket_credentials[ - "aws_access_key_id" - ] + TEMP_FILE_AWS_ACCESS_KEY_ID = app_bucket_credentials["aws_access_key_id"] TEMP_FILE_AWS_SECRET_ACCESS_KEY = app_bucket_credentials[ "aws_secret_access_key" ] TEMP_FILE_AWS_REGION = app_bucket_credentials["aws_region"] TEMP_FILE_AWS_S3_REGION_NAME = app_bucket_credentials["aws_region"] - TEMP_FILE_AWS_STORAGE_BUCKET_NAME = app_bucket_credentials[ - "bucket_name" - ] + TEMP_FILE_AWS_STORAGE_BUCKET_NAME = app_bucket_credentials["bucket_name"] else: AWS_ACCESS_KEY_ID = app_bucket_credentials["aws_access_key_id"] - AWS_SECRET_ACCESS_KEY = app_bucket_credentials[ - "aws_secret_access_key" - ] + AWS_SECRET_ACCESS_KEY = app_bucket_credentials["aws_secret_access_key"] AWS_REGION = app_bucket_credentials["aws_region"] AWS_S3_REGION_NAME = app_bucket_credentials["aws_region"] AWS_STORAGE_BUCKET_NAME = app_bucket_credentials["bucket_name"] @@ -225,12 +215,8 @@ def FILTERS_VERBOSE_LOOKUPS(): AWS_S3_REGION_NAME = env("AWS_REGION", default="") AWS_STORAGE_BUCKET_NAME = env("AWS_STORAGE_BUCKET_NAME", default="") - TEMP_FILE_AWS_ACCESS_KEY_ID = env( - "TEMP_FILE_AWS_ACCESS_KEY_ID", default="" - ) - TEMP_FILE_AWS_SECRET_ACCESS_KEY = env( - "TEMP_FILE_AWS_SECRET_ACCESS_KEY", default="" - ) + TEMP_FILE_AWS_ACCESS_KEY_ID = env("TEMP_FILE_AWS_ACCESS_KEY_ID", default="") + TEMP_FILE_AWS_SECRET_ACCESS_KEY = env("TEMP_FILE_AWS_SECRET_ACCESS_KEY", default="") TEMP_FILE_AWS_REGION = env("TEMP_FILE_AWS_REGION", default="") TEMP_FILE_AWS_S3_REGION_NAME = env("TEMP_FILE_AWS_REGION", default="") TEMP_FILE_AWS_STORAGE_BUCKET_NAME = env( @@ -357,12 +343,8 @@ def FILTERS_VERBOSE_LOOKUPS(): # Vite VITE_DEV = env.bool("VITE_DEV", default=False) -VITE_DEV_SERVER_URL = env.str( - "VITE_DEV_SERVER_URL", default="http://localhost:5173" -) -VITE_MANIFEST_PATH = ( - BASE_DIR / "front_end" / "build" / ".vite" / "manifest.json" -) +VITE_DEV_SERVER_URL = env.str("VITE_DEV_SERVER_URL", default="http://localhost:5173") +VITE_MANIFEST_PATH = BASE_DIR / "front_end" / "build" / ".vite" / "manifest.json" # Selenium (BDD tests) USE_REMOTE_CHROME = env("USE_REMOTE_CHROME", default=True) diff --git a/config/settings/paas.py b/config/settings/paas.py index b8aa86cb..e2a4a3d6 100644 --- a/config/settings/paas.py +++ b/config/settings/paas.py @@ -16,4 +16,6 @@ default=database_url_from_env("DATABASE_CREDENTIALS") ) + REDIS_URL = env("REDIS_URL", default=None) + "?ssl_cert_reqs=required" + LOGGING["handlers"]["console"]["formatter"] = "asim" From daaaab025862905bdf935487a58ee6e39828bbc9 Mon Sep 17 00:00:00 2001 From: Paul Wheatcroft Date: Tue, 18 Jun 2024 10:45:08 +0100 Subject: [PATCH 5/6] Updated env removed redis name change CACHE_URL added REDIS_URL = env.str("REDIS_URL") Standardise REDIS_URL env key Testing database settings in paas.py Re-implemented is_copilot for databases and removed from paas.py Ranamed REDI_URL env key to CACHE_ENDPOINT --- .env.ci | 2 +- config/settings/base.py | 2 +- config/settings/paas.py | 6 ------ 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.env.ci b/.env.ci index d51c5be9..35544d61 100644 --- a/.env.ci +++ b/.env.ci @@ -3,7 +3,7 @@ SECRET_KEY=used_for_testing ALLOWED_HOSTS="*" DEBUG=False DATABASE_URL=psql://postgres:postgres@db:5432/fido -REDIS_ENDPOINT=redis://redis:6379 +CACHE_ENDPOINT=redis://redis:6379 AUTHBROKER_CLIENT_ID= AUTHBROKER_CLIENT_SECRET= AUTHBROKER_URL= diff --git a/config/settings/base.py b/config/settings/base.py index d2909245..128b380b 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -238,7 +238,7 @@ def FILTERS_VERBOSE_LOOKUPS(): credentials["port"], ) else: - REDIS_URL = env.str("REDIS_ENDPOINT") + REDIS_URL = env("CACHE_ENDPOINT", default=None) # Celery CELERY_BROKER_URL = REDIS_URL diff --git a/config/settings/paas.py b/config/settings/paas.py index e2a4a3d6..d2783b53 100644 --- a/config/settings/paas.py +++ b/config/settings/paas.py @@ -12,10 +12,4 @@ if is_copilot(): ALLOWED_HOSTS = setup_allowed_hosts(ALLOWED_HOSTS) - DATABASES["default"] = dj_database_url.config( - default=database_url_from_env("DATABASE_CREDENTIALS") - ) - - REDIS_URL = env("REDIS_URL", default=None) + "?ssl_cert_reqs=required" - LOGGING["handlers"]["console"]["formatter"] = "asim" From 2db80608505d09e2841cb42f5deac26036807d16 Mon Sep 17 00:00:00 2001 From: Paul Wheatcroft <72826129+PaulWheatcroft@users.noreply.github.com> Date: Wed, 24 Jul 2024 17:00:34 +0100 Subject: [PATCH 6/6] Update .copilot/image_build_run.sh Co-authored-by: Lawrence <34475808+acodeninja@users.noreply.github.com> --- .copilot/image_build_run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.copilot/image_build_run.sh b/.copilot/image_build_run.sh index 03fb3211..b87c0039 100755 --- a/.copilot/image_build_run.sh +++ b/.copilot/image_build_run.sh @@ -6,4 +6,4 @@ set -e # Add commands below to run inside the container after all the other buildpacks have been applied export $(grep -v '^#' .env.ci | xargs) -python manage.py collectstatic --noinput \ No newline at end of file +python manage.py collectstatic --noinput