Update docker-compose

.env

@@ -1,4 +1,7 @@
-DOCKER_PYTHONPATH=/app
+# DEV
+# Override AIOD_MOUNT to any value to automatically mount your
+# source code to the containers when using `scripts/up.sh`.
+AIOD_MOUNT=
 
 # REST API
 AIOD_REST_PORT=8000

Dockerfile

@@ -10,6 +10,7 @@ WORKDIR /app
 # warnings.
 ENV PATH="${PATH}:/home/apprunner/.local/bin"
 
+ENV PYTHONPATH="/app"
 
 # Install python packages globally, so that it can also be used from cron dockers (running as root)
 COPY ./pyproject.toml /app/pyproject.toml

README.md

@@ -106,6 +106,7 @@ mysql> SHOW DATABASES;
 
 Now, you can visit the server from your browser at `localhost:8000/docs`.
 
+
 ### Changing the configuration
 You may need to change the configuration locally, for example if you want different ports to be used.
 Do not change files, instead add overrides.
@@ -119,6 +120,7 @@ AIOD_LOGSTASH_PORT=5001
 Then also specify this when you invoke docker compose, e.g.:
 `docker compose --env-file=.env --env-file=override.env up`
 Note that **order is important**, later environment files will override earlier ones.
+You may also use the `./scripts/up.sh` script to achieve this (see ["Shorthands"](#shorthands) below).
 
 #### Config.toml
 The main application supports configuration options through a `toml` file.
@@ -134,8 +136,19 @@ docker compose --profile examples --profile huggingface-datasets --profile openm
 docker compose --profile examples --profile huggingface-datasets --profile openml --profile zenodo-datasets down
 ```
 
-Make sure you use the same profile for `up` and `down`, otherwise some containers might keep 
-running.
+Make sure you use the same profile for `up` and `down`, or use `./scripts/down.sh` (see below),
+otherwise some containers might keep running.
+
+### Shorthands
+We provide two auxiliary scripts for launching docker containers and bringing them down.
+The first, `./scripts/up.sh` invokes `docker compose up -d` and takes any number of profiles to launch as parameters.
+It will also ensure that the changes of the configurations (see above) are observed.
+If `AIOD_MOUNT` is set (e.g., in `override.env`) then your local source code will be mounted on the containers,
+this is useful for local development but should not be used in production.
+E.g., with `AIOD_MOUNT` set `./scripts/up.sh` resolves to:
+`docker compose --env-file=.env --env-file=override.env -f docker-compose.yaml -f docker-compose.dev.yaml --profile examples  up -d`
+
+The second script is a convenience for bringing down all services, including all profiles: `./scripts/down.sh`
 
 #### Local Installation
 

docker-compose.dev.yaml

@@ -0,0 +1,32 @@
+services:
+  app:
+    volumes:
+      - ./src:/app:ro
+
+  fill-db-with-examples:
+    volumes:
+      - ./src:/app:ro
+      - ./connectors:/opt/connectors/script:ro
+
+  deletion:
+    volumes:
+      - ./src:/app:ro
+
+  huggingface-dataset-connector:
+    volumes:
+      - ./src:/app:ro
+      - ./connectors/huggingface/:/opt/connectors/script:ro
+
+  openml-connector:
+    volumes:
+      - ./src:/app:ro
+      - ./connectors/openml/:/opt/connectors/script:ro
+
+  zenodo-dataset-connector:
+    volumes:
+      - ./src:/app:ro
+      - ./connectors/zenodo/:/opt/connectors/script:ro
+
+  es_logstash_setup:
+    volumes:
+      - ./src:/app:ro

docker-compose.yaml

@@ -1,21 +1,18 @@
-
-version: '3.9'
-
 services:
   app:
     build:
       context: ./
       dockerfile: Dockerfile
     image: aiod_metadata_catalogue
     container_name: apiserver
+    volumes:
+      - ./src/config.override.toml:/app/config.override.toml:ro
     environment:
       - KEYCLOAK_CLIENT_SECRET=$KEYCLOAK_CLIENT_SECRET
       - ES_USER=$ES_USER
       - ES_PASSWORD=$ES_PASSWORD
     ports:
       - ${AIOD_REST_PORT}:8000
-    volumes:
-      - ./src:/app:ro
     command: >
       python main.py
       --build-db if-absent
@@ -38,11 +35,9 @@ services:
     environment:
       - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD}
       - KEYCLOAK_CLIENT_SECRET=$KEYCLOAK_CLIENT_SECRET
-      - PYTHONPATH=$DOCKER_PYTHONPATH
     volumes:
-      - ./src:/app:ro
       - ${DATA_PATH}/connectors:/opt/connectors/data
-      - ./connectors:/opt/connectors/script:ro
+      - ./src/config.override.toml:/app/config.override.toml:ro
     command: >
       /bin/bash -c "/opt/connectors/script/fill-examples.sh"
     depends_on:
@@ -56,8 +51,8 @@ services:
     image: aiod_deletion
     container_name: deletion
     volumes:
-      - ./src:/app
       - ${DATA_PATH}/deletion:/opt/deletion/data
+      - ./src/config.override.toml:/app/config.override.toml:ro
     command: >
       /bin/bash -c "/opt/deletion/script/entry.sh"
     depends_on:
@@ -70,11 +65,9 @@ services:
     container_name: huggingface-dataset-connector
     environment:
       - KEYCLOAK_CLIENT_SECRET=$KEYCLOAK_CLIENT_SECRET
-      - PYTHONPATH=$DOCKER_PYTHONPATH
     volumes:
-      - ./src:/app:ro
       - ${DATA_PATH}/connectors:/opt/connectors/data
-      - ./connectors/huggingface/:/opt/connectors/script:ro
+      - ./src/config.override.toml:/app/config.override.toml:ro
     command: >
       /bin/bash -c "/opt/connectors/script/datasets.sh"
     depends_on:
@@ -89,12 +82,10 @@ services:
     image: aiod_openml_connector
     container_name: openml-connector
     environment:
-      - PYTHONPATH=$DOCKER_PYTHONPATH
       - KEYCLOAK_CLIENT_SECRET=$KEYCLOAK_CLIENT_SECRET
     volumes:
-      - ./src:/app:ro
       - ${DATA_PATH}/connectors:/opt/connectors/data
-      - ./connectors/openml/:/opt/connectors/script:ro
+      - ./src/config.override.toml:/app/config.override.toml:ro
     command: >
       /bin/bash -c "/opt/connectors/script/entry.sh"
     depends_on:
@@ -109,12 +100,10 @@ services:
     image: aiod_zenodo_connector
     container_name: zenodo-dataset-connector
     environment:
-      - PYTHONPATH=$DOCKER_PYTHONPATH
       - KEYCLOAK_CLIENT_SECRET=$KEYCLOAK_CLIENT_SECRET
     volumes:
-      - ./src:/app
       - ${DATA_PATH}/connectors:/opt/connectors/data
-      - ./connectors/zenodo/:/opt/connectors/script
+      - ./src/config.override.toml:/app/config.override.toml:ro
     command: >
       /bin/bash -c "/opt/connectors/script/entry.sh"
     depends_on:
@@ -198,9 +187,8 @@ services:
       - MYSQL_ROOT_PASSWORD=$MYSQL_ROOT_PASSWORD
       - ES_USER=$ES_USER
       - ES_PASSWORD=$ES_PASSWORD
-      - PYTHONPATH=$DOCKER_PYTHONPATH
     volumes:
-      - ./src:/app
+      - ./src/config.override.toml:/app/config.override.toml:ro
       - ./logstash:/logstash
     command: >
       /bin/bash -c "python setup/logstash_setup/generate_logstash_config_files.py &&

scripts/README.md

@@ -5,6 +5,13 @@ Unless noted otherwise, all scripts should be run without arguments while the AI
 
 ## Script Descriptions
 
+## `up.sh`
+Shorthand for `docker compose up -d` which takes as arguments any profile names you also want to up.
+Will automatically specify to use both `.env` and `override.env` and also mount local source code if `AIOD_MOUNT` is set.
+
+## `down.sh`
+Shorthand to bring down all docker containers regardless of profile.
+
 ### `clean.sh`
 
 > [!WARNING]

scripts/down.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+command='docker compose --profile "*" down'
+echo "${command}"
+eval "${command}"

scripts/up.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+profiles=""
+for arg in "$@"; do
+  profiles+="--profile $arg "
+done
+
+NC='\033[0m' # No Color
+CYAN='\033[1;36m'
+GREEN='\033[0;32m'
+
+source .env
+source override.env
+
+if [[ -n "${AIOD_MOUNT}" ]]; then
+  compose_with_dev="-f docker-compose.dev.yaml"
+  echo -e "Launching ${CYAN}with${NC} local changes."
+else
+  compose_with_dev=""
+  echo -e "Launching ${GREEN}without${NC} local changes."
+fi
+
+command="docker compose --env-file=.env --env-file=override.env -f docker-compose.yaml ${compose_with_dev} ${profiles} up -d"
+echo "${command}"
+eval "${command}"
+

src/config.py

@@ -7,7 +7,7 @@
     DEFAULT_CONFIG = tomllib.load(fh)
 
 OVERRIDE_CONFIG_PATH = pathlib.Path(__file__).parent / "config.override.toml"
-if OVERRIDE_CONFIG_PATH.exists():
+if OVERRIDE_CONFIG_PATH.exists() and OVERRIDE_CONFIG_PATH.is_file():
     with open(OVERRIDE_CONFIG_PATH, "rb") as fh:
         OVERRIDE_CONFIG = tomllib.load(fh)
 else: