From 84dd1cf437c4f34faf1655b9699caedb1d2804db Mon Sep 17 00:00:00 2001 From: Earlopain <14981592+Earlopain@users.noreply.github.com> Date: Tue, 8 Oct 2024 11:18:10 +0200 Subject: [PATCH] Give fully parallelised build a try --- .github/workflows/nightly.yml | 109 ++++++++++++++++++++++++++++++---- architectures.rb | 16 ----- images.rb | 35 +++++++++++ nightly-versions.rb | 6 +- 4 files changed, 137 insertions(+), 29 deletions(-) delete mode 100755 architectures.rb create mode 100755 images.rb diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 99493576a..1b968f063 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -9,11 +9,16 @@ permissions: contents: read packages: write +env: + REGISTRY_IMAGE: ghcr.io/docker-ruby-nightly/ruby + jobs: set-up-matrix: name: Set up matrix runs-on: ubuntu-latest outputs: + date: ${{ steps.fetch-date.outputs.date }} + images: ${{ steps.fetch-matrix.outputs.images }} variants: ${{ steps.fetch-variants.outputs.variants }} json: ${{ steps.fetch-json.outputs.json }} steps: @@ -22,6 +27,12 @@ jobs: run: echo "variants=$(./variants.rb)" >> "$GITHUB_OUTPUT" - id: fetch-json run: echo "json=$(./nightly-versions.rb)" >> "$GITHUB_OUTPUT" + - id: fetch-date + run: echo "date=$(jq -r .master.date versions.json)" >> $GITHUB_OUTPUT + - run: ./apply-templates.sh + + - id: fetch-matrix + run: echo "images=$(./images.rb)" >> "$GITHUB_OUTPUT" build: needs: set-up-matrix @@ -29,15 +40,28 @@ jobs: strategy: fail-fast: false matrix: - variant: ${{ fromJson(needs.set-up-matrix.outputs.variants) }} - name: ${{ matrix.variant }} + image: ${{ fromJson(needs.set-up-matrix.outputs.images) }} + name: ${{ matrix.image.variant }} - ${{ matrix.image.platform }} steps: - uses: actions/checkout@v4 - run: echo '${{ needs.set-up-matrix.outputs.json }}' > versions.json - - run: echo date=$(jq -r .master.date versions.json) >> $GITHUB_ENV - run: ./apply-templates.sh - - run: echo base=$(ruby -e 's = File.read("master/${{ matrix.variant }}/Dockerfile"); puts s[/FROM (.*)/, 1]') >> $GITHUB_ENV - - run: echo platforms=$(./architectures.rb $base) >> $GITHUB_ENV + + # linux/amd64 => linux-amd64 + - run: | + platform=${{ matrix.image.platform }} + echo "platform_safe=${platform//\//-}" >> $GITHUB_ENV + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + env: + date: ${{ needs.set-up-matrix.outputs.date }} + with: + images: ${{ env.REGISTRY_IMAGE }} + tags: | + ${{ matrix.image.variant }}-nightly-${{ env.date }} + ${{ matrix.image.variant }}-nightly - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -52,13 +76,74 @@ jobs: username: ${{ github.repository_owner }} password: ${{ github.token }} - - name: Build and push + - name: Build and push by digest + id: build uses: docker/build-push-action@v6 + with: + context: ./master/${{ matrix.image.variant }} + platforms: ${{ matrix.image.platform }} + outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ matrix.image.variant }}-${{ env.platform_safe }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + merge: + runs-on: ubuntu-latest + needs: + - build + - set-up-matrix + strategy: + fail-fast: false + matrix: + variant: ${{ fromJson(needs.set-up-matrix.outputs.variants) }} + name: Merge - ${{ matrix.variant }} + + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digests-${{ matrix.variant }}-* + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 env: - tag1: ghcr.io/docker-ruby-nightly/ruby:${{ matrix.variant }}-nightly-${{ env.date }} - tag2: ghcr.io/docker-ruby-nightly/ruby:${{ matrix.variant }}-nightly + date: ${{ needs.set-up-matrix.outputs.date }} + with: + images: ${{ env.REGISTRY_IMAGE }} + tags: | + ${{ matrix.variant }}-nightly-${{ env.date }} + ${{ matrix.variant }}-nightly + + - name: Login to Github Container Registry + uses: docker/login-action@v3 with: - context: ./master/${{ matrix.variant }} - push: true - platforms: ${{ env.platforms }} - tags: ${{ env.tag1 }},${{ env.tag2 }} + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ github.token }} + + - name: Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *) + + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }} diff --git a/architectures.rb b/architectures.rb deleted file mode 100755 index 5a854471f..000000000 --- a/architectures.rb +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env ruby - -require "json" - -image = ARGV[0] -json = JSON.parse(`docker manifest inspect #{image}`) - -identifiers = json["manifests"].filter_map do |manifest| - platform = manifest["platform"] - next unless platform["os"] == "linux" - - optional_variant = "/#{platform['variant']}" if platform['variant'] - "#{platform['os']}/#{platform['architecture']}#{optional_variant}" -end - -puts identifiers.join(",") diff --git a/images.rb b/images.rb new file mode 100755 index 000000000..1437c1e52 --- /dev/null +++ b/images.rb @@ -0,0 +1,35 @@ +#!/usr/bin/env ruby + +require "json" + +json = JSON.parse(File.read("versions.json")) +latest_release = json.compact.keys.max_by(&:to_f) +variants = json[latest_release]["variants"] + +# Result: +# [ +# { "variant": "slim-bullseye", "platform": "linux/amd64" }, +# { "variant": "slim-bullseye", "platform": "linux/arm64/v8" }, +# { "variant": "slim-bullseye", "platform": "linux/386" }, +# { "variant": "alpine3.20", "platform": "linux/amd64" }, +# { "variant": "alpine3.20", "platform": "linux/arm64/v8" }, +# { "variant": "alpine3.20", "platform": "linux/386" } +# ] +images = variants.map do |variant| + # debian:bullseye-slim + base_image = File.read("master/#{variant}/Dockerfile")[/FROM (.*)/, 1] + manifest_inspect = JSON.parse(`docker manifest inspect #{base_image}`) + + # ["linux/amd64", "linux/arm64/v8", "linux/386"] + platforms = manifest_inspect["manifests"].filter_map do |manifest| + platform = manifest["platform"] + next unless platform["os"] == "linux" + + optional_variant = "/#{platform['variant']}" if platform['variant'] + "#{platform['os']}/#{platform['architecture']}#{optional_variant}" + end + + platforms.map { |platform| { variant: variant, platform: platform } } +end.flatten + +puts images.to_json diff --git a/nightly-versions.rb b/nightly-versions.rb index 2d047b287..694266e10 100755 --- a/nightly-versions.rb +++ b/nightly-versions.rb @@ -20,4 +20,8 @@ "rust" => donor["rust"], "rustup" => donor["rustup"], }) -puts JSON.generate({ "master" => master }) +result = { "master" => master }.to_json +# For step output +puts result +# For apply-templates.sh +File.write("versions.json", result)