ci.yaml

name: CI
on:
  push:
    tags:
      - "v*.*.*"
    branches:
      - main
  pull_request:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

permissions:
  checks: write
  pull-requests: write

env:
  BUILD_TARGET: "x86_64-unknown-linux-gnu"

jobs:
  docker-targets-build:
    runs-on: ubuntu-latest
    if: github.event_name == 'push' && github.ref_type == 'tag'
    strategy:
      fail-fast: false
      matrix:
        platform:
          - platform: linux/amd64
            name: linux/amd64
            target: x86_64-unknown-linux-gnu
          - platform: linux/arm64
            name: linux/arm64
            target: aarch64-unknown-linux-gnu
    name: docker-targets-build (${{ matrix.platform.platform }})
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
          submodules: recursive
      - name: Fetch tag
        run: git fetch origin --deepen=1
      - name: Prepare
        run: |
          platform=${{ matrix.platform.platform }}
          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
      - name: Setup Rust
        run: rustup update stable --no-self-update && rustup default stable && rustup target add ${{ matrix.platform.target }}
      - uses: davidB/rust-cargo-make@v1
      - name: Install Rust Target
        run: rustup target add ${{ matrix.platform.target }}
      - name: Install Protoc
        uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Install gcc-aarch64-linux-gnu
        if: matrix.platform.platform == 'linux/arm64'
        run: |
          sudo apt-get update
          sudo apt-get install gcc-aarch64-linux-gnu
      - uses: Swatinem/rust-cache@v2
        with:
          shared-key: release-gcc-aarch64-linux-gnu
          cache-all-crates: true
      - name: Build Executables
        env:
          PLATFORM_OVERRIDE: ${{ matrix.platform.name }}
        run: cargo make --profile ci build-release
      - name: Prepare Targets
        env:
          PLATFORM_OVERRIDE: ${{ matrix.platform.name }}
        run: cargo make --profile ci package-release
      - uses: actions/upload-artifact@v4
        name: Upload Targets
        with:
          name: docker-targets-build-${{ env.PLATFORM_PAIR }}
          path: target/${{ matrix.platform.target }}.tar
  docker-publish:
    runs-on: ubuntu-latest
    needs: [ docker-targets-build ]
    if: github.event_name == 'push' && github.ref_type == 'tag'
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
          submodules: recursive
      - name: Fetch tag
        run: git fetch origin --deepen=1
      - name: Prepare
        run: |
          echo "PLATFORMS=linux/amd64,linux/arm64" >> $GITHUB_ENV
      - uses: actions/download-artifact@v4
        name: Download Targets
        with:
          pattern: docker-targets-build-*
          path: target
          merge-multiple: true
      - name: Extract Targets
        run: |
          ls -R target
          cd target
          for f in *.tar; do tar xvf "$f"; done
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Set Docker version
        id: set-version
        run: |
          if [ "${{ github.event_name }}" == 'push' ] && [ "${{ github.ref_type }}" == 'tag' ]; then
            DOCKER_VERSION=$(echo "${{ github.ref }}" | sed 's|^refs/tags/v||')
            echo "DOCKER_VERSION=${DOCKER_VERSION}" >> $GITHUB_ENV
          else
            COMMIT_SHORT_HASH=$(git rev-parse --short=7 HEAD)
            echo "DOCKER_VERSION=${COMMIT_SHORT_HASH}" >> $GITHUB_ENV
          fi
      - uses: jpribyl/action-docker-layer-caching@v0.1.1
        continue-on-error: true
      - name: Log in to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}
      - name: Extract metadata (tags, labels) for worker executor
        id: meta-worker-executor
        uses: docker/metadata-action@v5
        with:
          images: golemservices/golem-worker-executor
      - name: Build and push worker executor image
        uses: docker/build-push-action@v5
        with:
          context: .
          file: ./golem-worker-executor/docker/Dockerfile
          push: true
          platforms: ${{ env.PLATFORMS }}
          tags: ${{ steps.meta-worker-executor.outputs.tags }}
          labels: ${{ steps.meta-worker-executor.outputs.labels }}
      - name: Extract metadata (tags, labels) for shard manager
        id: meta-shard-manager
        uses: docker/metadata-action@v5
        with:
          images: golemservices/golem-shard-manager
      - name: Build and push shard manager image
        uses: docker/build-push-action@v5
        with:
          context: .
          file: ./golem-shard-manager/docker/Dockerfile
          push: true
          platforms: ${{ env.PLATFORMS }}
          tags: ${{ steps.meta-shard-manager.outputs.tags }}
          labels: ${{ steps.meta-shard-manager.outputs.labels }}
      - name: Extract metadata (tags, labels) for golem component service
        id: meta-golem-component-service
        uses: docker/metadata-action@v5
        with:
          images: golemservices/golem-component-service
      - name: Build and push golem component service image
        uses: docker/build-push-action@v5
        with:
          context: .
          file: ./golem-component-service/docker/Dockerfile
          push: true
          platforms: ${{ env.PLATFORMS }}
          tags: ${{ steps.meta-golem-component-service.outputs.tags }}
          labels: ${{ steps.meta-golem-component-service.outputs.labels }}
      - name: Extract metadata (tags, labels) for golem worker service
        id: meta-golem-worker-service
        uses: docker/metadata-action@v5
        with:
          images: golemservices/golem-worker-service
      - name: Build and push golem worker service image
        uses: docker/build-push-action@v5
        with:
          context: .
          file: ./golem-worker-service/docker/Dockerfile
          push: true
          platforms: ${{ env.PLATFORMS }}
          tags: ${{ steps.meta-golem-worker-service.outputs.tags }}
          labels: ${{ steps.meta-golem-worker-service.outputs.labels }}
      - name: Extract metadata (tags, labels) for golem component compilation service
        id: meta-golem-component-compilation-service
        uses: docker/metadata-action@v5
        with:
          images: golemservices/golem-component-compilation-service
      - name: Build and push golem component compilation service image
        uses: docker/build-push-action@v5
        with:
          context: .
          file: ./golem-component-compilation-service/docker/Dockerfile
          push: true
          platforms: ${{ env.PLATFORMS }}
          tags: ${{ steps.meta-golem-component-compilation-service.outputs.tags }}
          labels: ${{ steps.meta-golem-component-compilation-service.outputs.labels }}
      - name: Extract metadata (tags, labels) for golem router
        id: meta-golem-router
        uses: docker/metadata-action@v5
        with:
          images: golemservices/golem-router
      - name: Build and push golem router
        uses: docker/build-push-action@v5
        with:
          context: .
          file: ./golem-router/docker/Dockerfile
          push: true
          platforms: ${{ env.PLATFORMS }}
          tags: ${{ steps.meta-golem-router.outputs.tags }}
          labels: ${{ steps.meta-golem-router.outputs.labels }}

  build-and-test:
    runs-on: ubuntu-latest-large
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
          submodules: recursive
      - name: Fetch tag
        run: git fetch origin --deepen=1
      - name: Setup Rust
        run: rustup update stable --no-self-update && rustup default stable
      - uses: Swatinem/rust-cache@v2
        with:
          shared-key: debug
          cache-all-crates: true
          save-if: true
      - uses: davidB/rust-cargo-make@v1
      - name: Install Protoc
        uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Build all targets
        run: cargo make --profile ci build
      - name: Check that OpenAPI specs are up-to-date
        run: cargo make --profile ci check-openapi
      - name: Check that configs are up-to-date
        run: cargo make --profile ci check-configs
      - name: Unit tests
        run: cargo make --profile ci unit-tests
      - name: Check formatting and clippy rules
        run: cargo make --profile ci check

  worker-tests-group1:
    runs-on: ubuntu-latest-large
    name: worker-tests-group1
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
          submodules: recursive
      - name: Fetch tag
        run: git fetch origin --deepen=1
      - name: Setup Rust
        run: rustup update stable --no-self-update && rustup default stable
      - uses: Swatinem/rust-cache@v2
        with:
          shared-key: debug-workertests
          cache-all-crates: true
      - uses: davidB/rust-cargo-make@v1
      - name: Install Protoc
        uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Setup Redis
        uses: shogo82148/actions-setup-redis@v1.35.1
        with:
          redis-version: latest
          auto-start: false
      - name: Worker Executor integration tests
        run: cargo make --profile ci worker-executor-tests-group1
        timeout-minutes: 20
  worker-tests:
    needs: [ worker-tests-group1 ]
    runs-on: ubuntu-latest-large
    strategy:
      fail-fast: true
      matrix:
        group:
          - name: group2
          - name: group3
          - name: group4
          - name: group5
          - name: group6
          - name: group7
          - name: group8
    name: worker-tests-${{ matrix.group.name }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
          submodules: recursive
      - name: Fetch tag
        run: git fetch origin --deepen=1
      - name: Setup Rust
        run: rustup update stable --no-self-update && rustup default stable
      - uses: Swatinem/rust-cache@v2
        with:
          shared-key: debug-workertests
          cache-all-crates: true
          save-if: false
      - uses: davidB/rust-cargo-make@v1
      - name: Install Protoc
        uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Setup Redis
        uses: shogo82148/actions-setup-redis@v1.35.1
        with:
          redis-version: latest
          auto-start: false
      - name: Worker Executor integration tests
        run: cargo make --profile ci worker-executor-tests-${{ matrix.group.name }}
        timeout-minutes: 40
  integration-tests:
    runs-on: ubuntu-latest-large
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
          submodules: recursive
      - name: Fetch tag
        run: git fetch origin --deepen=1
      - name: Setup Rust
        run: rustup update stable --no-self-update && rustup default stable
      - uses: Swatinem/rust-cache@v2
        with:
          shared-key: debug
          cache-all-crates: true
          save-if: false
      - uses: davidB/rust-cargo-make@v1
      - name: Install Protoc
        uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Setup Redis
        uses: shogo82148/actions-setup-redis@v1.35.1
        with:
          redis-version: latest
          auto-start: false
      - name: Integration tests
        env:
          QUIET: 1
        run: cargo make --profile ci integration-tests
        timeout-minutes: 30
  cli-tests:
    runs-on: ubuntu-latest-large
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
          submodules: recursive
      - name: Fetch tag
        run: git fetch origin --deepen=1
      - name: Setup Rust
        run: rustup update stable --no-self-update && rustup default stable
      - uses: Swatinem/rust-cache@v2
        with:
          shared-key: debug
          cache-all-crates: true
          save-if: false
      - uses: davidB/rust-cargo-make@v1
      - name: Install Protoc
        uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Setup Redis
        uses: shogo82148/actions-setup-redis@v1.35.1
        with:
          redis-version: latest
          auto-start: false
      - name: CLI tests
        env:
          QUIET: 1
        run: cargo make --profile ci cli-tests
        timeout-minutes: 35
  sharding-tests:
    runs-on: ubuntu-latest-large
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
          submodules: recursive
      - name: Fetch tag
        run: git fetch origin --deepen=1
      - name: Setup Rust
        run: rustup update stable --no-self-update && rustup default stable
      - uses: Swatinem/rust-cache@v2
        with:
          shared-key: debug
          cache-all-crates: true
          save-if: false
      - uses: davidB/rust-cargo-make@v1
      - name: Install Protoc
        uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Setup Redis
        uses: shogo82148/actions-setup-redis@v1.35.1
        with:
          redis-version: latest
          auto-start: false
      - name: Sharding tests
        env:
          QUIET: 1
        run: cargo make --profile ci sharding-tests
        timeout-minutes: 30
  publish:
    needs:
      [
        build-and-test,
        worker-tests,
        integration-tests,
        cli-tests,
        sharding-tests,
      ]
    if: "startsWith(github.ref, 'refs/tags/v')"
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
          submodules: recursive
      - name: Fetch tag
        run: git fetch origin --deepen=1
      - name: Setup Rust
        run: rustup update stable --no-self-update && rustup default stable
      - uses: Swatinem/rust-cache@v2
        with:
          shared-key: release
          cache-all-crates: true
      - uses: davidB/rust-cargo-make@v1
      - name: Install Protoc
        uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - id: get_version
        uses: battila7/get-version-action@v2
      - name: Publish all packages
        env:
          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
          VERSION: ${{ steps.get_version.outputs.version-without-v }}
        run: cargo make --profile ci publish

  publish_cli_binaries:
    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'golemcloud/golem'
    needs:
      [
        build-and-test,
        worker-tests,
        integration-tests,
        cli-tests,
        sharding-tests
      ]
    name: Publish binaries of golem-cli
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - rust-target: x86_64-unknown-linux-gnu
            os: ubuntu-latest
          - rust-target: aarch64-unknown-linux-gnu
            os: ubuntu-latest
            cross: true
          - rust-target: x86_64-apple-darwin
            os: macos-latest
          - rust-target: aarch64-apple-darwin
            os: macos-latest
          - rust-target: x86_64-pc-windows-gnu
            os: windows-latest
    permissions:
      contents: write
    steps:
      - uses: actions/checkout@v4
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
          submodules: recursive
      - name: Fetch tag
        run: git fetch origin --deepen=1
      - name: Setup Rust
        run: rustup update stable --no-self-update && rustup default stable && rustup target add ${{ matrix.rust-target }}
      - uses: davidB/rust-cargo-make@v1
      - name: Install Protoc
        uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - id: get_version
        uses: battila7/get-version-action@v2
      - name: Set version for all packages
        env:
          VERSION: ${{ steps.get_version.outputs.version-without-v }}
        run: cargo make --profile ci set-version
      - run: cargo build -p golem-cli --release --target ${{ matrix.rust-target }}
        if: ${{ ! matrix.cross }}
      - run: cargo install cross
        if: ${{ matrix.cross }}
      - run: cross build -p golem-cli --release --target ${{ matrix.rust-target }}
        if: ${{ matrix.cross }}
      - run: mv ./target/${{ matrix.rust-target }}/release/golem-cli.exe ./target/${{ matrix.rust-target }}/release/golem-cli-${{ matrix.rust-target }}.exe
        if: matrix.os == 'windows-latest'
      - run: mv ./target/${{ matrix.rust-target }}/release/golem-cli ./target/${{ matrix.rust-target }}/release/golem-cli-${{ matrix.rust-target }}
        if: matrix.os != 'windows-latest'
      - name: Login GH CLI
        shell: bash
        run: gh auth login --with-token < <(echo ${{ secrets.GITHUB_TOKEN }})
      - run: gh release upload -R golemcloud/golem --clobber ${{ github.ref_name }} target/${{ matrix.rust-target }}/release/golem-cli-${{ matrix.rust-target }}.exe
        if: matrix.os == 'windows-latest'
      - run: gh release upload -R golemcloud/golem --clobber ${{ github.ref_name }} target/${{ matrix.rust-target }}/release/golem-cli-${{ matrix.rust-target }}
        if: matrix.os != 'windows-latest'

  publish-slack-notification:
    if: ${{ always() && startsWith(github.ref, 'refs/tags/v') }}
    needs: [ publish, docker-publish, publish_cli_binaries ]
    runs-on: ubuntu-latest
    steps:
      - name: Publish Slack Notification
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_COLOR: ${{ needs.publish.result }}
          SLACK_ICON: https://uploads-ssl.webflow.com/64721eeec7cd7ef4f6f1683e/64831138b73a996d0e831773_32.png
          SLACK_TITLE: "Release Notification"
          SLACK_MESSAGE: "Publish - cargo result: `${{ needs.publish.result }}`, cli bin result: `${{ needs.publish_cli_binaries.result }}`, docker result: `${{ needs.docker-publish.result }}`"
          SLACK_USERNAME: CI
          SLACK_WEBHOOK: ${{ secrets.SLACK_ALERT_URL }}