From 53029ee29378f838eedf5746e1f6ee599b1d47fc Mon Sep 17 00:00:00 2001 From: Noelle Wang <73260931+No767@users.noreply.github.com> Date: Wed, 7 Aug 2024 17:02:54 -0700 Subject: [PATCH] Add full Docker support (#170) --- .github/workflows/docker.yml | 48 +++++++++++++++++ .github/workflows/release.yml | 37 +++++++++++++ docker-compose-dev.yml | 18 ------- docker/docker-compose.dev.yml | 20 +++++++ docker/docker-compose.prod.yml | 64 ++++++++++++++++++++++ docker/docker-compose.yml | 43 +++++++++++++++ docker/example.env | 11 ++++ docker/pg/init.sh | 3 +- docker/prometheus.yml | 8 +++ docs/deployment/docker.rst | 97 ++++++++++++++++++++++++++++++++++ docs/deployment/index.rst | 10 ++++ docs/dev-guide/intro.rst | 6 +-- docs/index.rst | 1 + 13 files changed, 343 insertions(+), 23 deletions(-) create mode 100644 .github/workflows/docker.yml delete mode 100644 docker-compose-dev.yml create mode 100644 docker/docker-compose.dev.yml create mode 100644 docker/docker-compose.prod.yml create mode 100644 docker/docker-compose.yml create mode 100644 docker/example.env create mode 100644 docker/prometheus.yml create mode 100644 docs/deployment/docker.rst create mode 100644 docs/deployment/index.rst diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..7457d9a --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,48 @@ +name: Docker + +on: + push: + branches: [main] + +jobs: + Build-and-Push: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Prepare Docker Meta + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ghcr.io/transprogrammer/rodhaj + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=edge,branch=main + + - name: Setup Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + with: + version: latest + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push image + uses: docker/build-push-action@v6 + with: + context: . + file: ./docker/Dockerfile + push: true + cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/rodhaj-build-cache:bot + cache-to: cache-to=type=registry,mode=max,ref=ghcr.io/${{ github.repository_owner }}/rodhaj-build-cache:bot + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ec54411..0c99bcb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,7 +4,37 @@ on: branches: - main jobs: + Bundle: + runs-on: ubuntu-latest + if: contains(github.event.head_commit.message, '#major') || contains(github.event.head_commit.message, '#minor') || contains(github.event.head_commit.message, '#patch') + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Prepare for bundling + run: | + mkdir -p rodhaj-docker + mkdir -p releases + cp docker/docker-compose.yml rodhaj-docker/ + cp docker/example.env rodhaj-docker/ + cp -r docker/pg/ rodhaj-docker/ + + - name: Bundle docker-related files + run: | + zip releases/rodhaj-docker.zip rodhaj-docker/** + tar -czf releases/rodhaj-docker.tar.gz rodhaj-docker/** + + - name: Upload bundle + uses: actions/upload-artifact@v4 + with: + path: releases + + Release: + permissions: + contents: write + needs: Bundle + runs-on: ubuntu-latest if: contains(github.event.head_commit.message, '#major') || contains(github.event.head_commit.message, '#minor') || contains(github.event.head_commit.message, '#patch') steps: @@ -12,6 +42,12 @@ jobs: with: fetch-depth: '0' + - name: Download Artifacts + uses: actions/download-artifact@v4 + with: + name: artifact + path: releases + - name: Bump version and push tag uses: anothrNick/github-tag-action@1.67.0 id: tag_version @@ -27,3 +63,4 @@ jobs: token: ${{ secrets.PAT_TOKEN }} tag: ${{ steps.tag_version.outputs.new_tag }} name: ${{ steps.tag_version.outputs.new_tag }} + artifacts: "releases/rodhaj-docker.zip,releases/rodhaj-docker.tar.gz" diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml deleted file mode 100644 index c6e74f0..0000000 --- a/docker-compose-dev.yml +++ /dev/null @@ -1,18 +0,0 @@ -version: "3.9" -name: "rodhaj-dev" -services: - postgres: - container_name: Rodhaj-Postgres - image: rodhaj-pg:dev-latest - build: - context: ./docker/pg - dockerfile: Dockerfile - env_file: - - .env - volumes: - - postgres_volume:/var/lib/postgresql/data - ports: - - 5432:5432 - -volumes: - postgres_volume: diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml new file mode 100644 index 0000000..3640df0 --- /dev/null +++ b/docker/docker-compose.dev.yml @@ -0,0 +1,20 @@ +name: rodhaj_dev + +# For development purposes, it is recommended in order to start the bot normally and using the Dev Reloader system +services: + database: + container_name: rodhaj_postgres + build: + context: ./pg + dockerfile: Dockerfile + environment: + POSTGRES_PASSWORD: ${DB_PASSWORD} + POSTGRES_DB: ${DB_DATABASE_NAME} + POSTGRES_USER: ${DB_USERNAME} + volumes: + - database:/var/lib/postgresql/data + ports: + - 5432:5432 + +volumes: + database: diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml new file mode 100644 index 0000000..2d3d345 --- /dev/null +++ b/docker/docker-compose.prod.yml @@ -0,0 +1,64 @@ +name: rodhaj_prod + +services: + rodhaj: + container_name: rodhaj + image: ghcr.io/transprogrammer/rodhaj:latest + volumes: + # Do not edit the next line. If you want to change the path of the configuration file, please edit the CONFIG_LOCATION variable + - ${CONFIG_LOCATION}:/rodhaj/bot/config.yml + env_file: + - .env + ports: + - 8555:8555 + depends_on: + - database + command: sh -c '/rodhaj/wait-for database:5432 -- echo "[Wait-for] PostgreSQL is fully up. Starting Rodhaj." && /rodhaj/start.sh' + restart: always + + database: + container_name: rodhaj_postgres + build: + context: ./pg + dockerfile: Dockerfile + environment: + POSTGRES_PASSWORD: ${DB_PASSWORD} + POSTGRES_DB: ${DB_DATABASE_NAME} + POSTGRES_USER: ${DB_USERNAME} + POSTGRES_INITDB_ARGS: '--data-checksums' + ports: + - 5432:5432 + volumes: + # Do not edit the next line. If you want to change the database storage location on your system, edit the value of DB_DATA_LOCATION in the .env file + - database:/var/lib/postgresql/data + healthcheck: + test: pg_isready --dbname='${DB_DATABASE_NAME}' --username='${DB_USERNAME}' || exit 1; Chksum="$$(psql --dbname='${DB_DATABASE_NAME}' --username='${DB_USERNAME}' --tuples-only --no-align --command='SELECT COALESCE(SUM(checksum_failures), 0) FROM pg_stat_database')"; echo "checksum failure count is $$Chksum"; [ "$$Chksum" = '0' ] || exit 1 + interval: 5m + start_interval: 30s + start_period: 5m + restart: always + + rodhaj-prometheus: + container_name: rodhaj_prometheus + ports: + - 9090:9090 + image: prom/prometheus:latest + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml + - prometheus-data:/prometheus + + # first login uses admin/admin + # add data source for http://rodhaj-prometheus:9090 to get started + rodhaj-grafana: + container_name: rodhaj_grafana + command: ['./run.sh', '-disable-reporting'] + ports: + - 3000:3000 + image: grafana/grafana-enterprise:11.1.3-ubuntu + volumes: + - grafana-data:/var/lib/grafana + +volumes: + database: + prometheus-data: + grafana-data: \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..e8edb3f --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,43 @@ +name: rodhaj + +services: + rodhaj: + container_name: rodhaj + image: ghcr.io/transprogrammer/rodhaj:latest + volumes: + # Do not edit the next line. If you want to change the path of the configuration file, please edit the CONFIG_LOCATION variable + - ${CONFIG_LOCATION}:/rodhaj/bot/config.yml + env_file: + - .env + ports: + - 8555:8555 + depends_on: + - database + # Oftentimes if Rodhaj started too early (aka starting without this script), then it would entirely not run the migrations and error out + command: sh -c '/rodhaj/wait-for database:5432 -- echo "[Wait-for] PostgreSQL is fully up. Starting Rodhaj." && /rodhaj/start.sh' + restart: always + + database: + container_name: rodhaj_postgres + build: + context: ./pg + dockerfile: Dockerfile + environment: + POSTGRES_PASSWORD: ${DB_PASSWORD} + POSTGRES_DB: ${DB_DATABASE_NAME} + POSTGRES_USER: ${DB_USERNAME} + POSTGRES_INITDB_ARGS: '--data-checksums' + ports: + - 5432:5432 + volumes: + # Do not edit the next line. If you want to change the database storage location on your system, edit the value of DB_DATA_LOCATION in the .env file + - database:/var/lib/postgresql/data + healthcheck: + test: pg_isready --dbname='${DB_DATABASE_NAME}' --username='${DB_USERNAME}' || exit 1; Chksum="$$(psql --dbname='${DB_DATABASE_NAME}' --username='${DB_USERNAME}' --tuples-only --no-align --command='SELECT COALESCE(SUM(checksum_failures), 0) FROM pg_stat_database')"; echo "checksum failure count is $$Chksum"; [ "$$Chksum" = '0' ] || exit 1 + interval: 5m + start_interval: 30s + start_period: 5m + restart: always + +volumes: + database: \ No newline at end of file diff --git a/docker/example.env b/docker/example.env new file mode 100644 index 0000000..178228c --- /dev/null +++ b/docker/example.env @@ -0,0 +1,11 @@ +# The location of where Rodhaj's configuration is stored. +# The configuration can be found under the config-example.yml +CONFIG_LOCATION=./config.yml + +# Connection secret for the postgres. You should change it to a random password +POSTGRES_PASSWORD=postgres + +# The values below this line do not need to be changed +################################################################################### +POSTGRES_USER=postgres +POSTGRES_DB=rodhaj \ No newline at end of file diff --git a/docker/pg/init.sh b/docker/pg/init.sh index 94f6a61..65dc79d 100644 --- a/docker/pg/init.sh +++ b/docker/pg/init.sh @@ -2,6 +2,5 @@ set -e psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL - CREATE ROLE rodhaj WITH LOGIN PASSWORD '$RODHAJ_PASSWORD'; - CREATE DATABASE rodhaj OWNER rodhaj; + CREATE EXTENSION IF NOT EXISTS pg_trgm; EOSQL diff --git a/docker/prometheus.yml b/docker/prometheus.yml new file mode 100644 index 0000000..776aeec --- /dev/null +++ b/docker/prometheus.yml @@ -0,0 +1,8 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: rodhaj + static_configs: + - targets: ['rodhaj:8555'] \ No newline at end of file diff --git a/docs/deployment/docker.rst b/docs/deployment/docker.rst new file mode 100644 index 0000000..2a784be --- /dev/null +++ b/docs/deployment/docker.rst @@ -0,0 +1,97 @@ +====== +Docker +====== + +.. warning:: + + This method of deployment is only for internal use within the transprogrammer community. + In addition, this deployment method is fairly advanced. This guide is only intended for internal + documentation for possible deployment options. + +Docker Compose can be used to run an Rodhaj instance in production. This will only work if you have access to the +Rodhaj Docker images. + +Step 1 - Download required files +================================ + +Download the necessary archive for getting started. This archive contains all +of the files needed to get started. These are provided either in ``.zip`` or ``.tar.gz`` +formats. + +.. code-block:: bash + + wget https://github.com/transprogrammer/rodhaj/releases/latest/download/rodhaj-docker.tar.gz + + # .zip version download + wget https://github.com/transprogrammer/rodhaj/releases/latest/download/rodhaj-docker.zip + +We need to unpack the archive in order to access the files. The following commands should do that. + +.. code-block:: bash + + tar -xvzf rodhaj-docker.tar.gz + + # .zip version unpacking + unzip rodhaj-docker.zip + +Once we have the files, we can now ``cd`` into the new extracted archive. + +.. code-block:: bash + + cd rodhaj-docker + +.. important:: + + Throughout the rest of the guide, the next steps assume that + you are in the ``rodhaj-docker`` directory. + +Step 2 - Populate ``.env`` and ``config.yml`` file with values +============================================================== + +- Change ``DB_PASSWORD`` to a randomly generated password. +- Provide Rodhaj's bot token in ``config.yml`` +- Change ``rodhaj.guild_id`` in ``config.yml`` to the server ID that Rodhaj is running on +- Modify the PostgreSQL URI used in ``config.yml`` to redirect to the database container and appropriate password + +.. note:: + + In order for Rodhaj to work container-wise, the IP aliases that is provided by the compose file + must be used instead. For example, the URI would look like this (of course replace the password): + + .. code-block:: + + postgresql://postgres:somepwd@database:5432/rodhaj + +.. important:: + + If you are running the full production version, please enable the Prometheus metrics + found in Rodhaj's configuration + +Step 3 - Start all containers +============================= + +Assume that you are in the directory created in Step 1, run the following command to bring up Rodhaj entirely. + +.. code-block:: + + docker compose up -d + +.. tip:: + + If you are having issues downloading container images, you will need to authenticate to the Github Container + Registry. Steps can be found `here `_. + +Step 4 - Upgrading +================== + +.. danger:: + + Although Rodhaj doesn't often update version-wise, there + may be breaking changes between versions. Be careful and be + up-to-date with changes. + +Upgrading Rodhaj is very simple. All you need to do is run the following commands below: + +.. code-block:: bash + + docker compose pull && docker compose up -d \ No newline at end of file diff --git a/docs/deployment/index.rst b/docs/deployment/index.rst new file mode 100644 index 0000000..f2f2fd7 --- /dev/null +++ b/docs/deployment/index.rst @@ -0,0 +1,10 @@ +================ +Deployment Guide +================ + +This document represents the deployment guide for Rodhaj. Currently, these are the officially supported methods as shown below. + +.. toctree:: + :maxdepth: 1 + + docker \ No newline at end of file diff --git a/docs/dev-guide/intro.rst b/docs/dev-guide/intro.rst index 5c18cc5..35f9a70 100644 --- a/docs/dev-guide/intro.rst +++ b/docs/dev-guide/intro.rst @@ -109,17 +109,17 @@ Using Docker If you decide to use Docker to run the local PostgreSQL server, then a pre-built Docker Compose file is provided. Setup instructions are as follows: -1. Copy ``envs/docker.env`` to ``.env`` within the root of the repo. Modify as appropriate. +1. Copy ``docker/example.env`` to ``.env`` within the ``docker`` folder. Modify as appropriate. .. code-block:: bash - cp envs/docker.env .env + cp docker/example.env docker/.env 2. Run the following command to start the PostgreSQL server .. code-block:: bash - docker compose -f docker-compose-dev.yml up -d + docker compose -f docker/docker-compose.dev.yml up -d Extensions ========== diff --git a/docs/index.rst b/docs/index.rst index 9482c09..bcaaa4f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,6 +11,7 @@ Rodhaj :hidden: :caption: Guides + deployment/index user-guide/index dev-guide/index