Populate DB URI from .env vars to allow ? in passwords and add health…

…checks (#285)

* Use environment variables to populate redis/postgres connection URLs
* Add Docker health checks to redis/postgres and delay other containers until they are ready

README.md

@@ -274,14 +274,14 @@ Reverse any changes you made (from the previous upgrades) to your `config.yml`,
 ### Note for multi servers beyond 3 servers
 
 You can copy the the last server section in the docker-compose.yml file and paste it, while replacing all the \_3 by \_4, also add the required variable in your .env (copy a whole section and replace the \_3 to \_4 suffix)
-Also note that you must add this extra keys in your docker-compose.yml after REDIS_URL. And mind the DB number that should change with each server
+Also note that you must add this extra keys in your docker-compose.yml after HLL_REDIS_URL. And mind the DB number that should change with each server
 
 ```
       ....
-      REDIS_URL: redis://redis:6379/1
-      REDIS_HOST: redis
-      REDIS_PORT: 6379
-      REDIS_DB: 1
+      HLL_REDIS_URL: redis://redis:6379/1
+      HLL_REDIS_HOST: redis
+      HLL_REDIS_PORT: 6379
+      HLL_REDIS_DB: 1
       ....
 ```
 
@@ -382,8 +382,8 @@ This will make redis and postgres available on you localhost with their default
     export DJANGO_DEBUG=True
     export SERVER_NUMBER=1
 
-    export DB_URL=postgres://rcon:developmentpassword@localhost:5432
-    export REDIS_URL=redis://localhost:6379/0
+    export HLL_DB_URL=postgres://rcon:developmentpassword@localhost:5432
+    export HLL_REDIS_URL=redis://localhost:6379/0
 
 #### Prepare the DB
 

alembic/env.py

@@ -20,8 +20,7 @@
 # target_metadata = mymodel.Base.metadata
 target_metadata = Base.metadata
 
-HLL_DB = os.getenv("DB_URL")
-print(HLL_DB)
+HLL_DB = os.getenv("HLL_DB_URL")
 config.set_main_option("sqlalchemy.url", HLL_DB)
 # other values from the config, defined by the needs of env.py,
 # can be acquired:

config/supervisord.conf

@@ -48,7 +48,7 @@ startsecs=1
 autostart=true
 
 [program:workers]
-command=rq worker --with-scheduler -u %(ENV_REDIS_URL)s
+command=rq worker --with-scheduler -u %(ENV_HLL_REDIS_URL)s
 environment=LOGGING_FILENAME=workers_%(ENV_SERVER_NUMBER)s.log
 startretries=100
 startsecs=1
@@ -81,7 +81,7 @@ command=/bin/bash -c "/usr/bin/crontab /config/crontab && /usr/sbin/cron -f"
 
 [program:scheduler]
 environment=LOGGING_FILENAME=scheduler_%(ENV_SERVER_NUMBER)s.log
-command=rqscheduler -H %(ENV_REDIS_HOST)s -p %(ENV_REDIS_PORT)s -d %(ENV_REDIS_DB)s
+command=rqscheduler -H %(ENV_HLL_REDIS_HOST)s -p %(ENV_HLL_REDIS_PORT)s -d %(ENV_HLL_REDIS_DB)s
 
 [unix_http_server]
 file=/tmp/supervisor.sock  

default.env

@@ -29,14 +29,32 @@ TAGGED_VERSION=latest
 # -----------------------------
 # This is a database password you have to choose, It does not have to be memorable 
 # as you will probably never access it directly but make it strong.
-# Prohibited characters for HLL_DB_PASSWORD: %#
-HLL_DB_PASSWORD=
+# Prohibited characters for HLL_DB_PASSWORD: %
+# You should wrap the password in single quotes: 'likethis' to avoid
+# parameter expansion: https://docs.docker.com/compose/environment-variables/env-file/
+HLL_DB_PASSWORD=''
+
+# Don't touch this unless you know what you're doing.
+# This is the postgres database name
+HLL_DB_NAME='rcon'
+
+# Don't touch this unless you know what you're doing.
+# This is the postgres user name
+HLL_DB_USER='rcon'
+
+# Don't touch this unless you know what you're doing. 
+# This is the postgres host name
+HLL_DB_HOST='postgres'
 
 # Don't touch this unless you know what you're doing. 
 # This is the DB port as exposed on the host machine
 HLL_DB_HOST_PORT=5432
 
-# Don't touch that unless you know what you're doing. 
+# Don't touch this unless you know what you're doing. 
+# This is the Redis host name
+HLL_REDIS_HOST='redis'
+
+# Don't touch this unless you know what you're doing. 
 # This is the Redis port as exposed on the host machine
 HLL_REDIS_HOST_PORT=6379
 

docker-compose-common-components.yml

@@ -9,11 +9,16 @@ services:
             HLL_PASSWORD: ${HLL_PASSWORD}
             LOGGING_LEVEL: 'INFO'
             LOGGING_PATH: /logs/
-            REDIS_URL: redis://redis:6379/0
-            REDIS_HOST: redis 
-            REDIS_PORT: 6379
-            REDIS_DB: 0
-            DB_URL: 'postgres://rcon:${HLL_DB_PASSWORD}@postgres:5432'
+            HLL_REDIS_HOST: ${HLL_REDIS_HOST} 
+            HLL_REDIS_PORT: ${HLL_REDIS_HOST_PORT}
+            HLL_REDIS_DB: 0
+            HLL_REDIS_URL: redis://${HLL_REDIS_HOST}:${HLL_REDIS_HOST_PORT}/0
+            HLL_DB_USER: ${HLL_DB_USER}
+            HLL_DB_PASSWORD: ${HLL_DB_PASSWORD}
+            HLL_DB_NAME: ${HLL_DB_NAME}
+            HLL_DB_HOST: ${HLL_DB_HOST}
+            HLL_DB_HOST_PORT: ${HLL_DB_HOST_PORT}
+            HLL_DB_URL: 'postgres://${HLL_DB_USER}:${HLL_DB_PASSWORD}@${HLL_DB_HOST}:${HLL_DB_HOST_PORT}/${HLL_DB_NAME}'
             DISCORD_WEBHOOK_AUDIT_LOG: ${DISCORD_WEBHOOK_AUDIT_LOG}
             RCONWEB_API_SECRET: ${RCONWEB_API_SECRET}
             SERVER_SHORT_NAME: ${SERVER_SHORT_NAME}

docker-compose.yml

@@ -14,8 +14,10 @@ services:
       service: python
     command: supervisor
     depends_on:
-      - postgres
-      - redis
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
     networks:
       common:
       server1:
@@ -60,6 +62,11 @@ services:
     image: redis:alpine
     restart: always
     command: redis-server /usr/local/etc/redis/redis.conf
+    healthcheck:
+      test: ["CMD-SHELL", "redis-cli ping | grep PONG"]
+      interval: 15s
+      timeout: 30s
+      retries: 5
     volumes:
       - ./redis_data:/data
       - ./config:/usr/local/etc/redis
@@ -71,15 +78,23 @@ services:
     environment:
       # If a password is not defined this container will fail to create
       POSTGRES_PASSWORD: ${HLL_DB_PASSWORD}
-      POSTGRES_USER: rcon
-      POSTGRES_DB: rcon
+      POSTGRES_USER: ${HLL_DB_USER}
+      POSTGRES_DB: ${HLL_DB_NAME}
       PGDATA: /data
+      HLL_DB_NAME: ${HLL_DB_NAME}
+      HLL_DB_USER: ${HLL_DB_USER}
+    healthcheck:
+      test: ["CMD", "pg_isready", "-U", "${HLL_DB_USER}", "-d", "${HLL_DB_NAME}"]
+      interval: 15s
+      timeout: 30s
+      retries: 5
+      start_period: 80s
     volumes:
       - ./db_data:/data
     networks:
       - common
 
-  ############ SERVER 2  #############
+  ########### SERVER 2  #############
   supervisor_2:
     <<: *supervisor
     environment: &env2
@@ -88,11 +103,16 @@ services:
       HLL_PASSWORD: ${HLL_PASSWORD_2}
       LOGGING_LEVEL: 'INFO'
       LOGGING_PATH: /logs/
-      REDIS_URL: redis://redis:6379/1
-      REDIS_HOST: redis 
-      REDIS_PORT: 6379
-      REDIS_DB: 1
-      DB_URL: 'postgres://rcon:${HLL_DB_PASSWORD}@postgres:5432'
+      HLL_REDIS_HOST: ${HLL_REDIS_HOST} 
+      HLL_REDIS_PORT: ${HLL_REDIS_HOST_PORT}
+      HLL_REDIS_DB: 1
+      HLL_REDIS_URL: redis://${HLL_REDIS_HOST}:${HLL_REDIS_HOST_PORT}/1
+      HLL_DB_USER: ${HLL_DB_USER}
+      HLL_DB_PASSWORD: ${HLL_DB_PASSWORD}
+      HLL_DB_NAME: ${HLL_DB_NAME}
+      HLL_DB_HOST: ${HLL_DB_HOST}
+      HLL_DB_HOST_PORT: ${HLL_DB_HOST_PORT}
+      HLL_DB_URL: 'postgres://${HLL_DB_USER}:${HLL_DB_PASSWORD}@${HLL_DB_HOST}:${HLL_DB_HOST_PORT}/${HLL_DB_NAME}'
       DISCORD_WEBHOOK_AUDIT_LOG: ${DISCORD_WEBHOOK_AUDIT_LOG_2}
       RCONWEB_API_SECRET: ${RCONWEB_API_SECRET}
       SERVER_SHORT_NAME: ${SERVER_SHORT_NAME_2}
@@ -158,11 +178,16 @@ services:
       HLL_PASSWORD: ${HLL_PASSWORD_3}
       LOGGING_LEVEL: 'INFO'
       LOGGING_PATH: /logs/
-      REDIS_URL: redis://redis:6379/2
-      REDIS_HOST: redis 
-      REDIS_PORT: 6379
-      REDIS_DB: 2
-      DB_URL: 'postgres://rcon:${HLL_DB_PASSWORD}@postgres:5432'
+      HLL_REDIS_HOST: ${HLL_REDIS_HOST} 
+      HLL_REDIS_PORT: ${HLL_REDIS_HOST_PORT}
+      HLL_REDIS_DB: 2
+      HLL_REDIS_URL: redis://${HLL_REDIS_HOST}:${HLL_REDIS_HOST_PORT}/2
+      HLL_DB_USER: ${HLL_DB_USER}
+      HLL_DB_PASSWORD: ${HLL_DB_PASSWORD}
+      HLL_DB_NAME: ${HLL_DB_NAME}
+      HLL_DB_HOST: ${HLL_DB_HOST}
+      HLL_DB_HOST_PORT: ${HLL_DB_HOST_PORT}
+      HLL_DB_URL: 'postgres://${HLL_DB_USER}:${HLL_DB_PASSWORD}@${HLL_DB_HOST}:${HLL_DB_HOST_PORT}/${HLL_DB_NAME}'
       DISCORD_WEBHOOK_AUDIT_LOG: ${DISCORD_WEBHOOK_AUDIT_LOG_3}
       RCONWEB_API_SECRET: ${RCONWEB_API_SECRET}
       SERVER_SHORT_NAME: ${SERVER_SHORT_NAME_3}

manage.py

@@ -20,8 +20,8 @@ def pre_flight_checks(env):
         "HLL_HOST",
         "HLL_PORT",
         "HLL_PASSWORD",
-        "REDIS_URL",
-        "DB_URL",
+        "HLL_REDIS_URL",
+        "HLL_DB_URL",
     ]
     optionnal = ["DISCORD_WEBHOOK_AUDIT_LOG", "LOGGING_PATH", "LOGGING_LEVEL"]
 

rcon/cache_utils.py

@@ -119,7 +119,7 @@ def clear_all(self):
 
 def get_redis_pool(decode_responses=True):
     global _REDIS_POOL
-    redis_url = os.getenv("REDIS_URL")
+    redis_url = os.getenv("HLL_REDIS_URL")
     if not redis_url:
         return None
 
@@ -144,7 +144,7 @@ def get_redis_client(decode_responses=True):
 def ttl_cache(ttl, *args, is_method=True, cache_falsy=True, **kwargs):
     pool = get_redis_pool(decode_responses=False)
     if not pool:
-        logger.debug("REDIS_URL is not set falling back to memory cache")
+        logger.debug("HLL_REDIS_URL is not set falling back to memory cache")
         return cachetools_ttl_cache(*args, ttl=ttl, **kwargs)
 
     def decorator(func):

rcon/models.py

@@ -53,9 +53,9 @@ def get_engine():
 
     if _ENGINE:
         return _ENGINE
-    url = os.getenv("DB_URL")
+    url = os.getenv("HLL_DB_URL")
     if not url:
-        msg = "No $DB_URL specified. Can't use database features"
+        msg = "No $HLL_DB_URL specified. Can't use database features"
         logger.error(msg)
         raise ValueError(msg)
 

rcon/utils.py

@@ -4,7 +4,6 @@
 import secrets
 from datetime import datetime
 from typing import Generic, TypeVar
-from urllib.parse import urlparse
 
 import redis
 
@@ -455,8 +454,18 @@ def __init__(self):
         if not num:
             raise ValueError("SERVER_NUMBER variable is not set, can't start")
 
-        parts = urlparse(os.getenv("REDIS_URL"))
-        self.red = redis.StrictRedis(host=parts.hostname, port=parts.port, db=0)
+        REDIS_PARTS = {
+            'HOST': os.getenv('HLL_REDIS_HOST'),
+            'PORT': os.getenv('HLL_REDIS_PORT'),
+        }
+
+        if not REDIS_PARTS['HOST']:
+            raise ValueError("HLL_REDIS_HOST must be set")
+
+        if not REDIS_PARTS['PORT']:
+            raise ValueError("HLL_REDIS_PORT must be set")
+
+        self.red = redis.StrictRedis(host=REDIS_PARTS['HOST'], port=REDIS_PARTS['PORT'], db=0)
         self.key_prefix = "frontend_"
         self.key = f"{self.key_prefix}{num}"
 

rconweb/rconweb/settings.py

@@ -189,23 +189,25 @@
 # Database
 # https://docs.djangoproject.com/en/3.0/ref/settings/#databases
 
-from urllib.parse import urlparse
-
-db_info = urlparse(os.getenv("DB_URL"))
-
+db_info = {
+    'USER': os.getenv("HLL_DB_USER"),
+    'PASSWORD': os.getenv("HLL_DB_PASSWORD"),
+    'HOST': os.getenv("HLL_DB_HOST"),
+    'PORT': os.getenv("HLL_DB_HOST_PORT"),
+    'NAME': os.getenv('HLL_DB_NAME')
+}
 
 DATABASES = {
     "default": {
         "ENGINE": "django.db.backends.postgresql",
-        "USER": db_info.username,
-        "PASSWORD": db_info.password,
-        "HOST": db_info.hostname,
-        "PORT": db_info.port,
-        "NAME": "rcon",
+        "USER": db_info['USER'],
+        "PASSWORD": db_info['PASSWORD'],
+        "HOST": db_info['HOST'],
+        "PORT": db_info['PORT'],
+        "NAME": db_info['NAME'],
     }
 }
 
-
 # Password validation
 # https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators