From eadf08261f5ca29cd3481d7ccf53526c5409613b Mon Sep 17 00:00:00 2001 From: jlestel Date: Tue, 21 Nov 2023 09:41:08 +0100 Subject: [PATCH] [CI] Build PR images on GHCR (#3445) * Upgrade CI * Tmp Test * Test tmp * Test master * Distinct cache by branch * Correct version name * Dynamic dockerhub push * correct cache name --------- Co-authored-by: Julien --- .github/actions/build/action.yaml | 78 +++++++++++++++++++++ .github/actions/grafana/action.yml | 37 ++++++++++ .github/actions/merge/action.yml | 40 +++++++++++ .github/workflows/buildx.yml | 108 ++++++----------------------- .github/workflows/elixir.yml | 10 ++- .github/workflows/ghcr_build.yml | 93 +++++++++++++++++++++++++ .github/workflows/ghcr_purge.yml | 35 ++++++++++ 7 files changed, 313 insertions(+), 88 deletions(-) create mode 100644 .github/actions/build/action.yaml create mode 100644 .github/actions/grafana/action.yml create mode 100644 .github/actions/merge/action.yml create mode 100644 .github/workflows/ghcr_build.yml create mode 100644 .github/workflows/ghcr_purge.yml diff --git a/.github/actions/build/action.yaml b/.github/actions/build/action.yaml new file mode 100644 index 0000000000..b41f229453 --- /dev/null +++ b/.github/actions/build/action.yaml @@ -0,0 +1,78 @@ +name: "Build" +description: "Build images" +inputs: + is_dockerhub_pushed: + description: "Need docker hub login?" + required: true + default: '' + docker_password: + description: "Docker password" + required: true + repository_owner: + description: "Repository owner" + required: true + default: "teslamate-org" + repository: + description: "Repo owner and name (repo_owner/repo_name)" + required: true + default: "teslamate" + github_token: + description: "GitHub Token" + required: true + labels: + description: "Labels added on metadata" + required: false + default: "" + version: + description: "Branch deployed" + required: false + default: "" +runs: + using: "composite" + steps: + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY_IMAGE }} + labels: | + {{ inputs.labels }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to Docker Hub + if: inputs.is_dockerhub_pushed != '' + uses: docker/login-action@v3.0.0 + with: + username: teslamate + password: ${{ inputs.docker_password }} + - name: Login to GitHub Container Registry + uses: docker/login-action@v3.0.0 + with: + registry: ghcr.io + username: ${{ inputs.repository_owner }} + password: ${{ inputs.github_token }} + - name: Build and push by digest + id: build + uses: docker/build-push-action@v5.0.0 + with: + context: . + platforms: ${{ matrix.platform }} + labels: ${{ steps.meta.outputs.labels }} + tags: ${{ steps.docker_meta.outputs.tags }} + cache-from: type=registry,ref=ghcr.io/${{ inputs.repository }}:buildcache-${{ matrix.cache_id }}-${{ inputs.version }} + cache-to: type=registry,ref=ghcr.io/${{ inputs.repository }}:buildcache-${{ matrix.cache_id }}-${{ inputs.version }},mode=max + outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true + - name: Export digest + shell: bash + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + ls -l /tmp/digests/ + - name: Upload digest + uses: actions/upload-artifact@v3 + with: + name: digests + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 diff --git a/.github/actions/grafana/action.yml b/.github/actions/grafana/action.yml new file mode 100644 index 0000000000..151503c1e5 --- /dev/null +++ b/.github/actions/grafana/action.yml @@ -0,0 +1,37 @@ +name: "Grafana" +description: "Grafana images" +inputs: + tags: + description: "Tags" + required: false + default: "" + labels: + description: "Labels" + required: true + image: + description: "Image target" + required: true + default: "teslamate/grafana" +runs: + using: "composite" + steps: + - name: Docker meta + id: docker_meta + uses: docker/metadata-action@v5 + with: + images: ${{ inputs.image }} + tags: ${{ inputs.tags }} + labels: ${{ inputs.labels }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push + uses: docker/build-push-action@v5.0.0 + with: + context: grafana + push: true + platforms: linux/amd64,linux/arm/v7,linux/arm64 + tags: ${{ steps.docker_meta.outputs.tags }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/actions/merge/action.yml b/.github/actions/merge/action.yml new file mode 100644 index 0000000000..aa2c00e867 --- /dev/null +++ b/.github/actions/merge/action.yml @@ -0,0 +1,40 @@ +name: "Merge" +description: "Merge images" +inputs: + tags: + description: "Tags" + required: false + default: "" + image: + description: "Image target" + required: true + +runs: + using: "composite" + steps: + - name: Download digests + uses: actions/download-artifact@v3 + with: + name: digests + path: /tmp/digests + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ inputs.image }} + tags: ${{ inputs.tags }} + + - name: Create manifest list and push + working-directory: /tmp/digests + shell: bash + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *) + - name: Inspect image + shell: bash + run: | + docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }} diff --git a/.github/workflows/buildx.yml b/.github/workflows/buildx.yml index 8e6999c77a..de152edbd5 100644 --- a/.github/workflows/buildx.yml +++ b/.github/workflows/buildx.yml @@ -31,118 +31,54 @@ jobs: cache_id: arm64 runs-on: ${{ matrix.runs_on }} + timeout-minutes: 10 steps: - name: Checkout uses: actions/checkout@v4 - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 + - name: Buildx + uses: ./.github/actions/build with: - images: ${{ env.REGISTRY_IMAGE }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Login to Docker Hub - uses: docker/login-action@v3.0.0 - with: - username: teslamate - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v3.0.0 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push by digest - id: build - uses: docker/build-push-action@v5.0.0 - with: - context: . - platforms: ${{ matrix.platform }} - labels: ${{ steps.meta.outputs.labels }} - tags: ${{ steps.docker_meta.outputs.tags }} - cache-from: type=registry,ref=ghcr.io/teslamate-org/teslamate:buildcache-${{ matrix.cache_id }} - cache-to: type=registry,ref=ghcr.io/teslamate-org/teslamate:buildcache-${{ matrix.cache_id }},mode=max - outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true - - name: Export digest - run: | - mkdir -p /tmp/digests - digest="${{ steps.build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" - - name: Upload digest - uses: actions/upload-artifact@v3 - with: - name: digests - path: /tmp/digests/* - if-no-files-found: error - retention-days: 1 + is_dockerhub_pushed: '1' + docker_password: ${{ secrets.DOCKER_PASSWORD }} + repository_owner: ${{ github.repository_owner }} + repository: ${{ github.repository }} + github_token: ${{ secrets.GITHUB_TOKEN }} + version: ${{ github.ref_name }} teslamate_merge: runs-on: ubuntu-latest needs: - teslamate_build steps: - - name: Download digests - uses: actions/download-artifact@v3 + - name: Checkout + uses: actions/checkout@v4 + + - name: Login to Docker Hub + uses: docker/login-action@v3.0.0 with: - name: digests - path: /tmp/digests - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 + username: teslamate + password: ${{ secrets.DOCKER_PASSWORD }} + + - uses: ./.github/actions/merge with: - images: ${{ env.REGISTRY_IMAGE }} + image: ${{ env.REGISTRY_IMAGE }} tags: | type=schedule,pattern=edge type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} type=edge - - name: Login to Docker Hub - uses: docker/login-action@v3.0.0 - with: - username: teslamate - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Create manifest list and push - working-directory: /tmp/digests - run: | - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *) - - name: Inspect image - run: | - docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }} - grafana: runs-on: ubuntu-latest + timeout-minutes: 10 steps: - name: Checkout uses: actions/checkout@v4 - - name: Docker meta - id: docker_meta - uses: docker/metadata-action@v5 + + - uses: ./.github/actions/grafana with: - images: teslamate/grafana tags: | type=edge type=schedule,pattern=edge type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Login to DockerHub - uses: docker/login-action@v3.0.0 - with: - username: teslamate - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and push - uses: docker/build-push-action@v5.0.0 - with: - context: grafana - push: true - platforms: linux/amd64,linux/arm/v7,linux/arm64 - tags: ${{ steps.docker_meta.outputs.tags }} - cache-from: type=gha - cache-to: type=gha,mode=max diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml index 0eb62d86c2..9b000c6047 100644 --- a/.github/workflows/elixir.yml +++ b/.github/workflows/elixir.yml @@ -2,8 +2,14 @@ name: Elixir CI on: push: + paths: + - "**/*" + - "!.github/**" # Important: Exclude PRs related to .github from auto-run pull_request: - branches: [ "master" ] + branches: ["master"] + paths: + - "**/*" + - "!.github/**" # Important: Exclude PRs related to .github from auto-run jobs: lint: @@ -124,7 +130,7 @@ jobs: services: db: image: postgres:15 - ports: [ "5432:5432" ] + ports: ["5432:5432"] env: POSTGRES_PASSWORD: postgres options: >- diff --git a/.github/workflows/ghcr_build.yml b/.github/workflows/ghcr_build.yml new file mode 100644 index 0000000000..a92b9d40c1 --- /dev/null +++ b/.github/workflows/ghcr_build.yml @@ -0,0 +1,93 @@ +name: Build GHCR images + +on: + workflow_dispatch: + push: + paths: + - "**/*" + - "!.github/**" + pull_request_target: + branches: ["master"] + paths: + - "**/*" + - "!.github/**" # Important: Exclude PRs related to .github from auto-run + +env: + REGISTRY_IMAGE: ghcr.io/${{ github.repository }} + +permissions: + contents: read + packages: write + +jobs: + teslamate_build: + name: Build images + strategy: + fail-fast: false + matrix: + include: + - platform: "linux/amd64" + runs_on: "ubuntu-latest" + cache_id: amd64 + - platform: "linux/arm/v7" + runs_on: "buildjet-2vcpu-ubuntu-2204-arm" + cache_id: arm + - platform: "linux/arm64" + runs_on: "buildjet-2vcpu-ubuntu-2204-arm" + cache_id: arm64 + + runs-on: ${{ matrix.runs_on }} + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Buildx + uses: ./.github/actions/build + with: + docker_password: ${{ secrets.DOCKER_PASSWORD }} + repository_owner: ${{ github.repository_owner }} + repository: ${{ github.repository }} + github_token: ${{ secrets.GITHUB_TOKEN }} + version: ${{ github.ref_name }} + labels: | + org.opencontainers.image.version=${{ github.ref || github.ref_name }} + + teslamate_merge: + name: Merge GHCR images + runs-on: ubuntu-latest + needs: + - teslamate_build + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Login to GitHub Container Registry + uses: docker/login-action@v3.0.0 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Merge + id: merge + uses: ./.github/actions/merge + with: + image: ${{ env.REGISTRY_IMAGE }} + + grafana: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3.0.0 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - uses: ./.github/actions/grafana + with: + image: ${{ env.REGISTRY_IMAGE }}/grafana + labels: | + org.opencontainers.image.version=${{ github.ref || github.ref_name }} diff --git a/.github/workflows/ghcr_purge.yml b/.github/workflows/ghcr_purge.yml new file mode 100644 index 0000000000..a6c9d3bc23 --- /dev/null +++ b/.github/workflows/ghcr_purge.yml @@ -0,0 +1,35 @@ +name: Purge PR images + +on: + pull_request_target: + types: + - closed + +permissions: + contents: read + packages: write + +jobs: + purge-pr-untagged: + name: Delete untagged images from ghcr.io + runs-on: ubuntu-latest + + steps: + - uses: actions/delete-package-versions@v4 + with: + package-name: "teslamate" + package-type: "container" + min-versions-to-keep: 0 + delete-only-untagged-versions: "true" + token: ${{ secrets.GITHUB_TOKEN }} + + purge-pr-package: + name: Delete image from ghcr.io + runs-on: ubuntu-latest + + steps: + - uses: chipkent/action-cleanup-package@v1.0.1 + with: + package-name: ${{ github.event.repository.name }} + tag: pr-${{ github.event.pull_request.number }} + github-token: ${{ secrets.GITHUB_TOKEN }}