ci: use self-start test database containers

Signed-off-by: Chenxiong Qi <[email protected]>
tkdchen · Jul 18, 2024 · f610fb6 · f610fb6
1 parent c1ea86c
commit f610fb6
Show file tree

Hide file tree

Showing 3 changed files with 85 additions and 69 deletions.
diff --git a/.github/workflows/unittests.yaml b/.github/workflows/unittests.yaml
@@ -14,46 +14,6 @@ jobs:
     name: "py${{ matrix.python-version }}-${{ matrix.django-version }}-${{ matrix.db }}"
     runs-on: ubuntu-24.04
 
-    services:
-      mariadb:
-        image: mariadb:10.11.8@sha256:75f6e61397758489d1dccf95db33b6b49ebfc7ec1253d40060fdf8ceb7f938a3
-        ports:
-          - 33061:3306
-        env:
-          MYSQL_DATABASE: nitrate
-          MYSQL_ROOT_PASSWORD: pass
-        options: >-
-          --name=testdb-mariadb
-          --health-cmd="mysqladmin ping -uroot -ppass"
-          --health-interval=5s
-          --health-timeout=2s
-          --health-retries=3
-      postgres:
-        image: postgres:16.3@sha256:0aafd2ae7e6c391f39fb6b7621632d79f54068faebc726caf469e87bd1d301c0
-        ports:
-          - 54321:5432
-        env:
-          POSTGRES_PASSWORD: pass
-        options: >-
-          --name=testdb-pgsql
-          --health-cmd="echo pass | psql -U postgres -W --command 'SELECT 1'"
-          --health-interval=5s
-          --health-timeout=2s
-          --health-retries=3
-      mysql:
-        image: mysql:8.0.22@sha256:0fd2898dc1c946b34dceaccc3b80d38b1049285c1dab70df7480de62265d6213
-        ports:
-          - 33062:3306
-        env:
-          MYSQL_DATABASE: nitrate
-          MYSQL_ROOT_PASSWORD: pass
-        options: >-
-          --name=testdb-mysql
-          --health-cmd="mysqladmin ping -uroot -ppass"
-          --health-interval=5s
-          --health-timeout=2s
-          --health-retries=3
-
     strategy:
       max-parallel: 7
       matrix:
@@ -75,7 +35,7 @@ jobs:
       - name: Install packages used to setup testenv
         run: |
           sudo apt update
-          sudo apt-get install gcc python3-dev libkrb5-dev libmariadb-dev libcrypt-dev
+          sudo apt-get install -y jq make gcc python3-dev libkrb5-dev libmariadb-dev libcrypt-dev
 
       - name: Checkout code
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
@@ -85,19 +45,26 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install dependencies
+      - name: Run the tests
+        env:
+          DB: ${{ matrix.db }}
+          PYTHON_VERSION: ${{ matrix.python-version }}
         run: |
-          python -m pip install --upgrade pip
-          python -m pip install tox coveralls
+          set -eu
+          set -o pipefail
 
-      - run: |
-          echo "tox_pyver=${pyver}" | sed "s/\.//" >> $GITHUB_ENV
-        env:
-          pyver: ${{ matrix.python-version }}
+          python3 -m pip install --upgrade pip
+          python3 -m pip install tox coveralls
 
-      - name: Tox
-        run: |
-          tox -e py$tox_pyver-${{ matrix.django-version }}-${{ matrix.db }}
+          python3 -m pip --version
+          tox -l
+
+          if [ "x$DB" != "xsqlite" ]; then
+            make start-testdb-${DB} detach=yes
+            trap 'make stop-testdb-${DB}' EXIT ERR
+            make check-testdb-health DB=${DB/postgres/pgsql}
+          fi
+          tox -e py${PYTHON_VERSION/./}-${PYTHON_VERSION}-${DB}
 
       - name: Parallel coverage report
         uses: coverallsapp/github-action@3dfc5567390f6fa9267c0ee9c251e4c8c3f18949 # v2

diff --git a/Makefile b/Makefile
@@ -38,17 +38,29 @@ db_envs:		# Print environment variables for a specific database engine set by DB
 format-code:		# Format Python code with black.
 	@black --line-length $(shell grep "^max_line_length" tox.ini | cut -d' '  -f3) src/tcms tests
 
-detach_testdb_container ?= false
-gh_wf_unittests = .github/workflows/unittests.yaml
-gh_wf_services = .jobs.unittests.services
+ifeq ($(strip $(detach)),yes)
+detach_opt=-d
+else
+detach_opt=
+endif
+
+common_health_options = --health-interval=5s --health-timeout=2s --health-retries=3
+
+mariadb_image = mariadb:10.11.8@sha256:75f6e61397758489d1dccf95db33b6b49ebfc7ec1253d40060fdf8ceb7f938a3
+mysql_image = mysql:8.0.22@sha256:0fd2898dc1c946b34dceaccc3b80d38b1049285c1dab70df7480de62265d6213
+postgres_image = postgres:16.3@sha256:0aafd2ae7e6c391f39fb6b7621632d79f54068faebc726caf469e87bd1d301c0
 
 # MariaDB setup for testenv
 
 .PHONY: start-testdb-mariadb
 start-testdb-mariadb:
-	podman run --rm -p 33061:3306 -e MYSQL_ROOT_PASSWORD=pass \
-		$(shell yq '$(gh_wf_services).mariadb.options' $(gh_wf_unittests)) \
-		$(shell yq '$(gh_wf_services).mariadb.image' $(gh_wf_unittests))
+	podman run --rm $(detach_opt) \
+		--name=testdb-mariadb \
+		-p 33061:3306 \
+		-e MYSQL_ROOT_PASSWORD=pass \
+		--health-cmd="mysqladmin ping -uroot -ppass" \
+		$(common_health_options) \
+		$(mariadb_image)
 
 .PHONY: stop-testdb-mariadb
 stop-testdb-mariadb:
@@ -58,23 +70,48 @@ stop-testdb-mariadb:
 
 .PHONY: start-testdb-mysql
 start-testdb-mysql:
-	podman run --rm -p 33062:3306 -e MYSQL_ROOT_PASSWORD=pass \
-		$(shell yq '$(gh_wf_services).mysql.options' $(gh_wf_unittests)) \
-		$(shell yq '$(gh_wf_services).mysql.image' $(gh_wf_unittests))
+	podman run --rm $(detach_opt) \
+		--name=testdb-mysql \
+		-p 33062:3306 \
+		-e MYSQL_ROOT_PASSWORD=pass \
+		--health-cmd="mysqladmin ping -uroot -ppass" \
+		$(common_health_options) \
+		$(mysql_image)
 
 .PHONY: stop-testdb-mysql
 stop-testdb-mysql:
 	podman stop testdb-mysql || :
 
 # PostgreSQL setup for testenv
 
-.PHONY: start-testdb-pgsql
-start-testdb-pgsql:
-	podman run --name testdb-pgsql --rm -p 54321:5432 \
+.PHONY: start-testdb-postgres
+start-testdb-postgres:
+	podman run --rm $(detach_opt) \
+		--name=testdb-postgres \
+		-p 54321:5432 \
 		-e POSTGRES_PASSWORD=pass \
-		$(shell yq '$(gh_wf_services).postgres.options' $(gh_wf_unittests)) \
-		$(shell yq '$(gh_wf_services).postgres.image' $(gh_wf_unittests))
-
-.PHONY: stop-testdb-pgsql
-stop-testdb-pgsql:
-	podman stop testdb-pgsql || :
+		--health-cmd="PGPASSWORD=pass psql -h 127.0.0.1 -U postgres -c 'SELECT 1'" \
+		$(common_health_options) \
+		$(postgres_image)
+
+.PHONY: stop-testdb-postgres
+stop-testdb-postgres:
+	podman stop testdb-postgres || :
+
+.PHONY: check-testdb-health
+check-testdb-health:
+	@test_db_engine=$(DB); \
+	for i in {1..5}; do \
+	  health_status=$$(podman inspect testdb-$${DB/pgsql/postgres} | jq -r '.[].State.Health.Status'); \
+	  [ "x$$health_status" == "xhealthy" ] && break; \
+	  if [ $$i -eq 5 ]; then \
+		echo "testdb $${DB/pgsql/postgres} container is not healthy. Seems failed to start." >&2; \
+		echo "container inspect:" >&2; \
+		podman inspect testdb-$${DB/pgsql/postgres} | jq '.[].State' >&2; \
+		exit 1; \
+	  else \
+		echo "Sleep 2s then have another health check" >&2; \
+		sleep 2s; \
+	  fi; \
+	  done; \
+	  echo "Test database $${DB/pgsql/postgres} is healthy."
diff --git a/renovate.json b/renovate.json
@@ -3,9 +3,21 @@
   "extends": [
     "config:best-practices"
   ],
+
   "docker-compose": {
     "fileMatch": ["^container-compose\\.ya?ml$"]
   },
+  "customManagers": [
+    {
+      "customType": "regex",
+      "fileMatch": ["^Makefile$"],
+      "matchStrings": [
+        "(mariadb|mysql|postgres)_image = (?<depName>mariadb|mysql|postgres?):(?<currentValue>[.0-9]+?)@sha256:[0-9a-f]{64}\\n"
+      ],
+      "datasourceTemplate": "docker"
+    }
+  ],
+
   "packageRules": [
     {
       "matchPackageNames": ["mariadb"],