From 0e568e352b26f28eb3079c014cf91b74383aee6b Mon Sep 17 00:00:00 2001 From: Frank Sachsenheim Date: Tue, 18 Jan 2022 23:14:02 +0100 Subject: [PATCH] Adds GH workflows for maintenance releases The problem for now is that the commit after dependency updates isn't pulled by futher steps. It's on the main branch though. Closes #46 by using buildah & qemu. Closes #61 by also pushing to ghcr.io --- .github/dependabot.yml | 10 ++ .github/workflows/build-and-publish.yml | 141 +++++++++++++++++++++++ .github/workflows/maintenance-update.yml | 65 +++++++++++ .github/workflows/quality-checks.yml | 31 ++++- Dockerfile | 14 +-- Dockerfile-dev | 2 +- Makefile | 30 +---- README.rst | 7 +- docs/usage.rst | 6 +- hooks/build | 16 --- hooks/release-arm | 20 ---- hooks/release-multiimage | 30 ----- maintenance-updates.sh | 13 --- pyproject.toml | 1 + 14 files changed, 257 insertions(+), 129 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/build-and-publish.yml create mode 100644 .github/workflows/maintenance-update.yml delete mode 100755 hooks/build delete mode 100755 hooks/release-arm delete mode 100755 hooks/release-multiimage delete mode 100644 maintenance-updates.sh diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..7fed498 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +--- + +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + +... diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml new file mode 100644 index 0000000..098dbdc --- /dev/null +++ b/.github/workflows/build-and-publish.yml @@ -0,0 +1,141 @@ +name: Build & publish deck-chores +on: + push: + tags: ["*"] + workflow_call: + inputs: + ref: + required: false + type: string + secrets: + DOCKER_AUTH_TOKEN: + required: true + PYPI_AUTH_TOKEN: + required: true + +jobs: + version: + outputs: + version: ${{ steps.echo-version.outputs.version }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ inputs.ref || github.ref }} + - uses: actions/setup-python@v2 + with: + python-version: 3.9 + - uses: abatilo/actions-poetry@v2.1.4 + + - id: echo-version + run: echo "::set-ouput name=version::$(poetry version --short)" + + - if: ${{ github.event_name == 'push' }} + run: "[ ${{ steps.echo-version.outputs.version }} == ${{ github.ref }} ]" + + + cheeseshop: + name: Build & publish to the cheeseshop + needs: [version] + runs-on: ubuntu-latest + steps: + + - uses: actions/checkout@v2 + with: + ref: ${{ inputs.ref || github.ref }} + - uses: actions/setup-python@v2 + with: + python-version: 3.9 + - uses: abatilo/actions-poetry@v2.1.4 + + - run: poetry build + - run: poetry publish --username __token__ --password ${{ secrets.PYPI_AUTH_TOKEN }} + + container-image: + name: Build & push multi-architecture image + needs: [version] + env: + IMAGE_NAME: deck-chores + DOCKER_IO_USER: funkyfuture + VERSION: ${{ needs.version.outputs.version }} + runs-on: ubuntu-latest + steps: + + - uses: redhat-actions/podman-login@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + - uses: redhat-actions/podman-login@v1 + with: + registry: docker.io + username: ${{ env.DOCKER_IO_USER }} + password: ${{ secrets.DOCKER_AUTH_TOKEN }} + + - uses: actions/checkout@v2 + with: + ref: ${{ inputs.ref || github.ref }} + + - run: > + echo "PRERELEASE=${{ ( + contains('a', env.VERSION) || contains('b', env.VERSION) + || contains('rc', env.VERSION) || contains('pre', env.VERSION) + ) }}" >> $GITHUB_ENV + - name: echo version related variables + run: | + echo 'VERSION: ${{ env.VERSION }}' + echo 'PRERELEASE: ${{ env.PRERELEASE }}' + - id: docker-metadata + uses: docker/metadata-action@v3 + with: + images: ${{ env.IMAGE_NAME }} + flavor: latest=false + labels: | + org.opencontainers.image.documentation=https://deck-chores.readthedocs.org/ + org.opencontainers.image.url=https://deck-chores.readthedocs.org/ + tags: | + type=sha,prefix=src-commit- + type=pep440,pattern={{version}},value=${{ env.VERSION }} + type=pep440,pattern={{major}},value=${{ env.VERSION }},enable=${{ env.PRERELEASE == 'false' }} + type=pep440,pattern={{major}}.{{minor}},value=${{ env.VERSION }},enable=${{ env.PRERELEASE == 'false' }} + - name: prepare push tag value + id: push-tags-value + run: echo "::set-output name=tags::${{ steps.docker-metadata.outputs.tags }}" | tr "\n" " " | sed "s/${{ env.IMAGE_NAME }}://g" + + - name: install dependency for multi-platform builds + run: | + sudo apt update + sudo apt install -y qemu-user-static + - id: build-image + uses: redhat-actions/buildah-build@v2 + with: + containerfiles: | + ./Dockerfile + image: ${{ env.IMAGE_NAME }} + labels: ${{ steps.docker-metadata.outputs.labels }} + platforms: linux/amd64,linux/arm,linux/arm64 + tags: ${{ steps.docker-metadata.outputs.tags }} + + - name: echo build outputs + run: | + echo "Image: ${{ steps.build-image.outputs.image }}" + echo "Tags: ${{ steps.build-image.outputs.tags }}" + echo "Tagged Image: ${{ steps.build-image.outputs.image-with-tag }}" + - name: echo created images + run: buildah images | grep '${{ env.IMAGE_NAME }}' + - name: echo image metadata + run: buildah inspect ${{ steps.build-image.outputs.image-with-tag }} + + - name: push to ghcr.io + uses: redhat-actions/push-to-registry@v2 + with: + registry: ghcr.io/${{ github.actor }} + image: ${{ steps.build-image.outputs.image }} + tags: ${{ steps.push-tags-value.outputs.tags }} + + - name: push to docker.io + uses: redhat-actions/push-to-registry@v2 + with: + registry: docker.io/${{ env.DOCKER_IO_USER }} + image: ${{ steps.build-image.outputs.image }} + tags: ${{ steps.push-tags-value.outputs.tags }} diff --git a/.github/workflows/maintenance-update.yml b/.github/workflows/maintenance-update.yml new file mode 100644 index 0000000..c6dfd01 --- /dev/null +++ b/.github/workflows/maintenance-update.yml @@ -0,0 +1,65 @@ +name: Update dependencies & create a maintence release +on: + schedule: + - cron: "35 10 12 * *" + workflow_dispatch: + +jobs: + update: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: 3.9 + - uses: abatilo/actions-poetry@v2.1.4 + + - run: poetry update --lock + + - id: commit-and-push + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: Updates dependencies + + - if: ${{ steps.commit-and-push.outputs.changes_detected == false }} + run: gh run cancel ${{ github.run_id }} && tail -f /dev/null + env: + GITHUB_TOKEN: ${{ github.token }} + + run-tests: + needs: [update] + uses: funkyfuture/deck-chores/.github/workflows/quality-checks.yml@main + with: + ref: main + + bump-version: + needs: [run-tests] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: main + - uses: actions/setup-python@v2 + with: + python-version: 3.9 + - uses: abatilo/actions-poetry@v2.1.4 + + - id: bump + run: | + poetry version patch + echo "set-output:: name=version::$(poetry version --short)" + + - uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: > + Bumps version to ${{ steps.bump.outputs.version }} (maintenance release) + tagging_message: ${{ steps.bump.outputs.version }} + + build-and-publish: + needs: [bump-version] + uses: funkyfuture/deck-chores/.github/workflows/build-and-publish.yml@main + with: + ref: main + secrets: + DOCKER_AUTH_TOKEN: ${{ secrets.DOCKER_AUTH_TOKEN }} + PYPI_AUTH_TOKEN: ${{ secrets.PYPI_AUTH_TOKEN }} diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml index 3b562a2..3f021fc 100644 --- a/.github/workflows/quality-checks.yml +++ b/.github/workflows/quality-checks.yml @@ -1,11 +1,22 @@ --- name: quality-checks + on: pull_request: push: - branches: - - main + branches: + - main + workflow_call: + inputs: + ref: + required: false + type: string + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.sha }} + cancel-in-progress: true + jobs: python-tests: runs-on: ubuntu-latest @@ -18,6 +29,8 @@ jobs: - target: doclinks steps: - uses: actions/checkout@v2 + with: + ref: ${{ inputs.ref || github.sha }} - uses: actions/setup-python@v2 with: python-version: 3.9 @@ -28,13 +41,21 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - run: python -m pip install --upgrade pip setuptools wheel - - uses: abatilo/actions-poetry@v2.1.0 + - uses: abatilo/actions-poetry@v2.1.4 - run: poetry install -v - run: poetry run make ${{ matrix.target }} - docker-build-test: + image-build-test: runs-on: ubuntu-latest steps: - - uses: docker/build-push-action@v2 + - uses: actions/checkout@v2 + with: + ref: ${{ inputs.ref || github.sha }} + - uses: redhat-actions/buildah-build@v2 + with: + containerfiles: | + ./Dockerfile + image: ${{ github.repository }} + tags: test-build-${{ github.sha }} ... diff --git a/Dockerfile b/Dockerfile index e7c6508..e0de10e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,7 @@ -FROM python:3.9-alpine +FROM docker.io/python:3.9-alpine MAINTAINER Frank Sachsenheim -ARG VERSION -ARG SOURCE_COMMIT -ARG BUILD_DATE - -LABEL org.opencontainers.image.created=$BUILD_DATE \ - org.opencontainers.image.description="Job scheduler for Docker containers, configured via labels." \ - org.opencontainers.image.documentation="https://deck-chores.readthedocs.org/" \ - org.opencontainers.image.revision=$SOURCE_COMMIT \ - org.opencontainers.image.source="https://github.com/funkyfuture/deck-chores" \ - org.opencontainers.image.title="deck-chores" \ - org.opencontainers.image.version=$VERSION - CMD ["deck-chores"] ENV PYTHONOPTIMIZE=1 # could be 2 with Cerberus 2 diff --git a/Dockerfile-dev b/Dockerfile-dev index b95db97..25724b3 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -1,4 +1,4 @@ -FROM python:3.9-alpine +FROM docker.io/python:3.9-alpine CMD ["deck-chores"] LABEL org.label-schema.name="deck-chores" diff --git a/Makefile b/Makefile index 8d37782..12b530f 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,8 @@ .DEFAULT_GOAL := build-dev REPO_NAME = funkyfuture/deck-chores -VERSION = $(shell grep -oP "^version = \K.+" pyproject.toml | tr -d '"') +VERSION = $(shell poetry version --short) IMAGE_NAME = $(REPO_NAME):$(VERSION) -GIT_SHA1 = $(shell git rev-parse HEAD) - -export IMAGE_NAME -export GIT_SHA1 define PRINT_HELP_PYSCRIPT import re, sys @@ -21,11 +17,11 @@ export PRINT_HELP_PYSCRIPT .PHONY: black black: ## code-formatting with black - poetry run black deck_chores tests + poetry run black deck_chores tests .PHONY: build build: ## builds the Docker image - hooks/build + docker build --tag $(IMAGE_NAME) . .PHONY: build-dev build-dev: ## builds the Docker image for debugging @@ -76,11 +72,6 @@ help: ## print make targets help lint: black ## check style with flake8 poetry run flake8 --max-complexity=10 --max-line-length=89 deck_chores tests -.PHONY: maintenance-release -maintenance-release: ## publish a maintenance with updated dependencies - bash ./maintenance-updates.sh - $(MAKE) release - .PHONY: mypy mypy: ## check types with mypy poetry run mypy --ignore-missing-imports deck_chores @@ -94,19 +85,8 @@ test: lint mypy pytest ## run all tests .PHONY: release release: test doclinks build ## release the current version on github, the PyPI and the Docker hub - git tag -f $(VERSION) - git push origin main - git push -f origin $(VERSION) - poetry publish --build - $(MAKE) release-multiimage - -.PHONY: release-arm -release-arm: ## release the arm build on the Docker hub - hooks/release-arm $(IMAGE_NAME) $(GIT_SHA1) - -.PHONY: release-multiimage -release-multiimage: release-arm ## release the multi-arch manifest on the Docker hub - hooks/release-multiimage $(REPO_NAME) $(VERSION) + git tag $(VERSION) + git push origin refs/tags/$(VERSION) .PHONY: run run: build ## runs deck-chores in a temporary container diff --git a/README.rst b/README.rst index fa5a1b6..3ccdc8f 100644 --- a/README.rst +++ b/README.rst @@ -13,7 +13,9 @@ deck-chores **A job scheduler for Docker containers, configured via container labels.** * Documentation: https://deck-chores.readthedocs.io -* Image repository: https://hub.docker.com/r/funkyfuture/deck-chores +* Image repositories: + * https://github.com/funkyfuture/deck-chores/pkgs/container/deck-chores + * https://hub.docker.com/r/funkyfuture/deck-chores * Code repository: https://github.com/funkyfuture/deck-chores * Issue tracker: https://github.com/funkyfuture/deck-chores/issues * Free software: ISC license @@ -27,8 +29,7 @@ Features - use date, interval and cron-like triggers - set a maximum of simultaneously running instances per job - restrict job scheduling to one container per service -- multi-architecture image supports ``amd64``, ``arm64`` and ``armv7l`` platforms, no emulator - involved +- multi-architecture image supports ``amd64``, ``arm64`` and ``arm`` platforms Example diff --git a/docs/usage.rst b/docs/usage.rst index 6667b65..82cb6e2 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -9,7 +9,7 @@ On a single host Usually you would run ``deck-chores`` in a container:: - $ docker run --rm -v /var/run/docker.sock:/var/run/docker.sock funkyfuture/deck-chores:1 + $ docker run --rm -v /var/run/docker.sock:/var/run/docker.sock ghcr.io/funkyfuture/deck-chores:1 .. note:: @@ -26,7 +26,7 @@ Likewise, docker-compose_ can be used with such configuration: services: officer: - image: funkyfuture/deck-chores:1 + image: ghcr.io/funkyfuture/deck-chores:1 restart: unless-stopped environment: TIMEZONE: Asia/Tel Aviv @@ -60,7 +60,7 @@ a suitable stack definition: services: officer: - image: funkyfuture/deck-chores:1 + image: ghcr.io/funkyfuture/deck-chores:1 deploy: mode: global environment: diff --git a/hooks/build b/hooks/build deleted file mode 100755 index a6b8066..0000000 --- a/hooks/build +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env sh - -echo "------ HOOK START - BUILD -------" - -BASE_IMAGE=$(head -n1 Dockerfile | cut -d" " -f2) -VERSION=$(grep -oP "^version = \K.+" pyproject.toml | tr -d '"') -docker pull "${BASE_IMAGE}" -printenv -docker build \ - --build-arg BUILD_DATE="$(date --rfc-3339 seconds)" \ - --build-arg SOURCE_COMMIT="${GIT_SHA1}" \ - --build-arg VERSION="${VERSION}" \ - --tag "${IMAGE_NAME}" \ - . - -echo "------ HOOK END - BUILD -------" diff --git a/hooks/release-arm b/hooks/release-arm deleted file mode 100755 index 529e36d..0000000 --- a/hooks/release-arm +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/sh - -set -ex - -for ARCH in arm64 armv7l ; do - -BUILD_HOST="${ARCH}-host" -REMOTE_PATH="docker/images/deck-chores" -IMAGE_NAME="$1-${ARCH}" -SOURCE_COMMIT=$2 - -rsync -e ssh -vca --filter=':- .gitignore' --exclude='.git/' --del . $BUILD_HOST:~/$REMOTE_PATH - -ssh $BUILD_HOST <<-SCRIPT - cd ${REMOTE_PATH} - IMAGE_NAME="${IMAGE_NAME}" GIT_SHA1="${SOURCE_COMMIT}" hooks/build - docker push "${IMAGE_NAME}" -SCRIPT - -done diff --git a/hooks/release-multiimage b/hooks/release-multiimage deleted file mode 100755 index 6321ae4..0000000 --- a/hooks/release-multiimage +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/sh - -set -ex - -IMAGE=$1 -VERSION=$2 - -AMD64_IMAGE="${IMAGE}:${VERSION}-amd64" -ARM64_IMAGE="${IMAGE}:${VERSION}-arm64" -ARMV7L_IMAGE="${IMAGE}:${VERSION}-armv7l" - -while ! docker pull ${AMD64_IMAGE} ; do sleep 20 ; done - -MAJOR=$(echo $VERSION | cut -d . -f 1) -MINOR=$(echo $VERSION | cut -d . -f 2) -PATCH=$(echo $VERSION | cut -d . -f 3) - -for TAG in ${MAJOR}.${MINOR}.${PATCH} ${MAJOR}.${MINOR} ${MAJOR} ; do - - MANIFEST="${IMAGE}:${TAG}" - - docker manifest create "${MANIFEST}" "${AMD64_IMAGE}" "${ARM64_IMAGE}" "${ARMV7L_IMAGE}" - docker manifest annotate --os linux --arch "amd64" "${MANIFEST}" "${AMD64_IMAGE}" - docker manifest annotate --os linux --arch "arm64" "${MANIFEST}" "${ARM64_IMAGE}" - docker manifest annotate --os linux --arch "arm" --variant "v7" "${MANIFEST}" "${ARMV7L_IMAGE}" - - docker manifest inspect "${MANIFEST}" - docker manifest push --purge "${MANIFEST}" - -done diff --git a/maintenance-updates.sh b/maintenance-updates.sh deleted file mode 100644 index f699d9a..0000000 --- a/maintenance-updates.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/this/is/not/supposed/to/be/called/directly - -set -ex - -poetry update -if git status --porcelain | grep "M poetry.lock" ; then - git add poetry.lock - git commit -m "Updates dependencies" -fi - -poetry version patch -git add pyproject.toml -git commit -m "Bumps version to $(poetry version | cut -d ' ' -f 2) (maintenance build)" diff --git a/pyproject.toml b/pyproject.toml index 7d16f3f..8e60946 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ deck-chores = "deck_chores.main:main" [tool.poetry.dependencies] # the Dockerfile-dev has these values too # .readthedocs.yml must be in parity with the Python version +# it's also all over the Github workflows python = "^3.9" apscheduler = "^3.6" cerberus = "^1.3.4"