From 91a236d6d1a5d75fd18b293bc72497f2996da20f Mon Sep 17 00:00:00 2001 From: Dylan Hillerbrand Date: Fri, 20 Sep 2024 10:19:05 -0400 Subject: [PATCH] build: separate production and development django settings - Separates out development/debugging django settings into a separate debug_settings.py file - Adds ability to set a non-default (non cantusdata.settings) django settings file via environment variables - Creates a compose-dev.yaml compose configuration file for development - Removes unnecessary port mappings from the production docker compose configuration - Updates the README to account for these changes --- .env.sample | 9 +- README.md | 12 +-- app/django-config.sh | 2 +- app/public/cantusdata/debug_settings.py | 22 +++++ app/public/cantusdata/settings.py | 23 ++--- compose-dev.yaml | 108 ++++++++++++++++++++++++ docker-compose.yaml | 26 +----- 7 files changed, 151 insertions(+), 51 deletions(-) create mode 100644 app/public/cantusdata/debug_settings.py create mode 100644 compose-dev.yaml diff --git a/.env.sample b/.env.sample index b44b0125..18b46384 100644 --- a/.env.sample +++ b/.env.sample @@ -1,10 +1,15 @@ APP_VERSION=3.2-0.12.1 COMPOSE_PROJECT_NAME=cantus -#When DEVELOPMENT is False, Django's DEBUG setting -#is False and app is served by gunicorn +# When DEVELOPMENT is False, app is served by gunicorn, +# otherwise the Django development server is used. The +# default django settings module is cantusdata.settings. +# To run with DEBUG=True and other DEBUG settings, uncomment +# the DJANGO_SETTINGS_MODULE variable and change to +# cantusdata.debug_settings DEVELOPMENT=False #DJANGO_SECRET_KEY=**PROVIDE SECRET KEY HERE** +#DJANGO_SETTINGS_MODULE=cantusdata.settings #Postgres authentication variables POSTGRES_DB=cantus_db diff --git a/README.md b/README.md index 7f57d673..ca703b6e 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ The build process relies on environment variables specified in an `.env` file lo Make a copy of `.env.sample` and name it `.env`. You must make two modifications to this file before the docker containers will build. Both `POSTGRES_PASSWORD` and `RABBIT_PASSWORD` should be uncommented and set with secure passwords. -Before launching the site, ensure that the `DEVELOPMENT` variable is set correctly. For development, it should be set to `True`; for deployment, it should be `False`. This variable configures the application's debug settings. Deploying the website with `DEVELOPMENT=True` would leak debugging information on a live server and use Django's development server rather than gunicorn and must be avoided. +Before launching the site, ensure that the `DEVELOPMENT` and `DJANGO_SETTINGS_MODULE` variables are set correctly. For development, `DEVELOPMENT` should be set to `True`; for deployment, it should be `False`. Deploying the website with `DEVELOPMENT=True` would leak debugging information on a live server and use Django's development server rather than gunicorn and must be avoided. `DJANGO_SETTINGS_MODULE` defaults to `cantusdata.settings`, which has the `DEBUG` setting set to `False` and does not include various debugging settings. To take advantage of those debugging applications, like `django_debug_toolbar`, uncomment `DJANGO_SETTINGS_MODULE` in the `.env` file and set to ` #### Handling `postgres` authentication issues @@ -41,7 +41,7 @@ rm -r data ### Launch in development -In the `.env.sample` file, the `DEVELOPMENT` variable is set to `False` by default. For local development, set this to `True` to turn on Django's debug mode, which allows you to access detailed traces when Django encounters errors. For deployment on a server, this should remain set to `False`. +In the `.env.sample` file, the `DEVELOPMENT` variable is set to `False` by default. For local development, set this to `True` in your local `.env` file to install development dependencies and run the development server using Django and django-extensions. For deployment on a server, this should remain set to `False`. You should also uncomment the `DJANGO_SETTINGS_MODULE` variable and set to `cantusdata.debug_settings`. > **Windows Users:** Make sure `/app/django-config.sh` has `LF` line endings before launching. This file gets copied over into an Ubuntu container and will break the process if git automatically checked out the file using Windows (`CRLF`) line endings. @@ -49,14 +49,14 @@ Execute the following commands from the root directory of the repo: ```sh # Build the images and launch the containers (this will take a while) -$ docker compose build -$ docker compose up -d +$ docker compose -f compose-dev.yaml build +$ docker compose -f compose-dev.yaml up -d ``` When testing subsequent changes, you can do this in one step by including the `--build` flag: ``` -docker compose up --build -d +docker compose -f compose-dev.yaml up --build -d ``` After the build process completes, the site should be available on http://localhost:8000/ in your host machine. @@ -88,7 +88,7 @@ Whenever changes are made to database models, they need to be applied to the Pos If, during development, you make a change to any models, build and run the containers, as above. Then, enter the command line in the `app` container: ```sh -$ docker compose exec -it app bash +$ docker compose -f compose-dev.yaml exec -it app bash ``` Then, run the `makemigrations` command: diff --git a/app/django-config.sh b/app/django-config.sh index 49a0750d..c826d9fc 100644 --- a/app/django-config.sh +++ b/app/django-config.sh @@ -4,7 +4,7 @@ python manage.py clear_session_data python manage.py collectstatic --noinput if [[ $DEVELOPMENT == "True" ]]; then - python manage.py runserver_plus 0:8001 + python -Xfrozen_modules=off -m debugpy --listen 0.0.0.0:3000 manage.py runserver_plus 0:8001 else gunicorn -b 0:8001 cantusdata.wsgi --timeout 600 --workers 4 fi \ No newline at end of file diff --git a/app/public/cantusdata/debug_settings.py b/app/public/cantusdata/debug_settings.py new file mode 100644 index 00000000..c3a7c475 --- /dev/null +++ b/app/public/cantusdata/debug_settings.py @@ -0,0 +1,22 @@ +""" +Overwrites some main settings that are used in development. +""" + +from .settings import * + +DEBUG = is_development + +DEBUG_TOOLBAR_CONFIG = { + "SHOW_TOOLBAR_CALLBACK": lambda request: ( + False if request.headers.get("x-requested-with") == "XMLHttpRequest" else True + ), +} +MIDDLEWARE.insert(0, "debug_toolbar.middleware.DebugToolbarMiddleware") +INSTALLED_APPS.extend(["django_extensions", "debug_toolbar"]) + + +SESSION_COOKIE_SECURE = False +CSRF_COOKIE_SECURE = False + +SECURE_HSTS_INCLUDE_SUBDOMAINS = False +SECURE_HSTS_PRELOAD = False diff --git a/app/public/cantusdata/settings.py b/app/public/cantusdata/settings.py index 3182dfd8..0c646b61 100644 --- a/app/public/cantusdata/settings.py +++ b/app/public/cantusdata/settings.py @@ -12,8 +12,6 @@ import os is_development = os.environ.get("DEVELOPMENT") == "True" -is_production = not is_development - # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -21,7 +19,7 @@ SECRET_KEY = os.environ.get("DJANGO_SECRET_KEY") # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = is_development +DEBUG = False ALLOWED_HOSTS = [ "cantus.simssa.ca", @@ -48,9 +46,6 @@ "cantusdata.CantusdataConfig", ] -if DEBUG: - INSTALLED_APPS.extend(["django_extensions", "debug_toolbar"]) - MIDDLEWARE = [ "django.middleware.security.SecurityMiddleware", # Migration: + django 3.1 "django.contrib.sessions.middleware.SessionMiddleware", @@ -61,8 +56,6 @@ "django.middleware.clickjacking.XFrameOptionsMiddleware", ] -if DEBUG: - MIDDLEWARE.insert(0, "debug_toolbar.middleware.DebugToolbarMiddleware") ROOT_URLCONF = "cantusdata.urls" @@ -168,13 +161,13 @@ DEFAULT_AUTO_FIELD = "django.db.models.AutoField" -SESSION_COOKIE_SECURE = is_production -CSRF_COOKIE_SECURE = is_production +SESSION_COOKIE_SECURE = True +CSRF_COOKIE_SECURE = True CSRF_TRUSTED_ORIGINS = ["https://cantus.simssa.ca", "https://cantus.staging.simssa.ca"] SECURE_HSTS_SECONDS = 86400 -SECURE_HSTS_INCLUDE_SUBDOMAINS = is_production -SECURE_HSTS_PRELOAD = is_production +SECURE_HSTS_INCLUDE_SUBDOMAINS = True +SECURE_HSTS_PRELOAD = True CELERY_BROKER_URL = f"amqp://{os.environ.get('RABBIT_USER')}:{os.environ.get('RABBIT_PASSWORD')}@cantus-rabbitmq-1:5672/{os.environ.get('RABBIT_VHOST')}" CELERY_RESULT_BACKEND = "django-db" @@ -182,9 +175,3 @@ CELERY_RESULT_EXTENDED = True CELERY_APP = "cantusdata" CELERY_TASK_TRACK_STARTED = True - -DEBUG_TOOLBAR_CONFIG = { - "SHOW_TOOLBAR_CALLBACK": lambda request: ( - False if request.headers.get("x-requested-with") == "XMLHttpRequest" else True - ), -} diff --git a/compose-dev.yaml b/compose-dev.yaml new file mode 100644 index 00000000..812e472a --- /dev/null +++ b/compose-dev.yaml @@ -0,0 +1,108 @@ +services: + app: + build: + context: . + dockerfile: ./app/Dockerfile + args: + DEVELOPMENT: ${DEVELOPMENT} + container_name: cantus-app-1 + command: /code/django-config.sh + volumes: + - ./app/public:/code/public + - django_static_volume:/code/static + depends_on: + postgres: + condition: service_healthy + solr: + condition: service_started + ports: + - "3000:3000" + environment: + - APP_VERSION=${APP_VERSION} + - DEVELOPMENT=${DEVELOPMENT} + - DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE} + - DJANGO_SECRET_KEY=${DJANGO_SECRET_KEY} + - POSTGRES_DB=${POSTGRES_DB} + - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - RABBIT_USER=${RABBIT_USER} + - RABBIT_PASSWORD=${RABBIT_PASSWORD} + - RABBIT_VHOST=${RABBIT_VHOST} + + postgres: + build: ./postgres + container_name: cantus-postgres-1 + volumes: + - ./data/postgres:/var/lib/postgresql/data + environment: + - POSTGRES_DB=${POSTGRES_DB} + - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + ports: + - "5432:5432" + healthcheck: + test: "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}" + interval: 30s + timeout: 30s + retries: 5 + start_period: 30s + + nginx: + container_name: cantus-nginx-1 + build: ./nginx + ports: + - "8000:8000" + depends_on: + - app + volumes: + - ./cantaloupe/interim_files/manifests:/code/manifests + - django_static_volume:/code/static/django + + solr: + build: ./solr + container_name: cantus-solr-1 + ports: + - "8983:8983" + environment: + - SOLR_SECURITY_MANAGER_ENABLED=false + volumes: + - solr_volume:/var/solr + + rabbitmq: + build: ./rabbitmq + container_name: cantus-rabbitmq-1 + environment: + - RABBITMQ_DEFAULT_USER=${RABBIT_USER} + - RABBITMQ_DEFAULT_PASS=${RABBIT_PASSWORD} + - RABBITMQ_DEFAULT_VHOST=${RABBIT_VHOST} + ports: + - "5672:5672" + restart: always + + celery: + image: cantus-app + container_name: cantus-celery-1 + volumes: + - ./app/public:/code/public + command: [ "/code/.venv/bin/python", "/code/.venv/bin/celery", "-A", "cantusdata", "worker", "-l", "INFO", "-c", "1" ] + environment: + - APP_VERSION=${APP_VERSION} + - DEVELOPMENT=${DEVELOPMENT} + - POSTGRES_DB=${POSTGRES_DB} + - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - RABBIT_USER=${RABBIT_USER} + - RABBIT_PASSWORD=${RABBIT_PASSWORD} + - RABBIT_VHOST=${RABBIT_VHOST} + restart: always + + cantaloupe: + build: ./cantaloupe + container_name: cantus-cantaloupe-1 + volumes: + - ./cantaloupe/interim_files/images:/srv/images + command: [ "java", "-Dcantaloupe.config=/etc/cantaloupe/cantaloupe.properties", "-Xmx2g", "-jar", "/opt/cantaloupe/cantaloupe-5.0.5/cantaloupe-5.0.5.jar" ] + +volumes: + django_static_volume: + solr_volume: diff --git a/docker-compose.yaml b/docker-compose.yaml index 22a775ef..c3ebaa64 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -56,8 +56,6 @@ services: solr: build: ./solr container_name: cantus-solr-1 - ports: - - "8983:8983" environment: - SOLR_SECURITY_MANAGER_ENABLED=false volumes: @@ -70,8 +68,6 @@ services: - RABBITMQ_DEFAULT_USER=${RABBIT_USER} - RABBITMQ_DEFAULT_PASS=${RABBIT_PASSWORD} - RABBITMQ_DEFAULT_VHOST=${RABBIT_VHOST} - ports: - - "5672:5672" restart: always celery: @@ -79,18 +75,7 @@ services: container_name: cantus-celery-1 volumes: - ./app/public:/code/public - command: - [ - "/code/.venv/bin/python", - "/code/.venv/bin/celery", - "-A", - "cantusdata", - "worker", - "-l", - "INFO", - "-c", - "1" - ] + command: [ "/code/.venv/bin/python", "/code/.venv/bin/celery", "-A", "cantusdata", "worker", "-l", "INFO", "-c", "1" ] environment: - APP_VERSION=${APP_VERSION} - DEVELOPMENT=${DEVELOPMENT} @@ -107,14 +92,7 @@ services: container_name: cantus-cantaloupe-1 volumes: - ./cantaloupe/interim_files/images:/srv/images - command: - [ - "java", - "-Dcantaloupe.config=/etc/cantaloupe/cantaloupe.properties", - "-Xmx2g", - "-jar", - "/opt/cantaloupe/cantaloupe-5.0.5/cantaloupe-5.0.5.jar" - ] + command: [ "java", "-Dcantaloupe.config=/etc/cantaloupe/cantaloupe.properties", "-Xmx2g", "-jar", "/opt/cantaloupe/cantaloupe-5.0.5/cantaloupe-5.0.5.jar" ] volumes: django_static_volume: