ci.yaml

name: CI
on:
  push:
    tags:
      - "v*.*.*"
    branches:
      - main
  pull_request:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  BUILD_TARGET: "x86_64-unknown-linux-gnu"

jobs:
  docker-targets-build:
    runs-on: ubuntu-latest
    if: github.event_name == 'push' && github.ref_type == 'tag'
    strategy:
      fail-fast: false
      matrix:
        platform:
          - platform: linux/amd64
            target: x86_64-unknown-linux-gnu
            build-config: ""
          - platform: linux/arm64
            target: aarch64-unknown-linux-gnu
            build-config: '--config "target.aarch64-unknown-linux-gnu.linker = ''aarch64-linux-gnu-gcc''"'
    name: docker-targets-build (${{ matrix.platform.platform }})
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          submodules: recursive
          fetch-depth: "0"
      - name: Prepare
        run: |
          platform=${{ matrix.platform.platform }}
          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
      - name: Setup Rust
        uses: actions-rs/toolchain@v1
        with:
          toolchain: stable
          target: ${{ matrix.platform.target }}
          override: true
      - name: Install Rust Target
        run: rustup target add ${{ matrix.platform.target }}
      - name: Install Protoc
        uses: arduino/setup-protoc@v1
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Install gcc-aarch64-linux-gnu
        if: matrix.platform.platform == 'linux/arm64'
        run: |
          sudo apt-get update
          sudo apt-get install gcc-aarch64-linux-gnu
      - uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/.crates2.json
            ~/.cargo/.crates.toml
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: ${{ runner.os }}-cargo-release-${{ hashFiles('**/Cargo.lock') }}
      - name: Build Executables
        run: cargo build --release --target ${{ matrix.platform.target }} ${{ matrix.platform.build-config }}
      - name: Prepare Targets
        run: |
          cd target
          tar -cvf ${{ matrix.platform.target }}.tar \
            ${{ matrix.platform.target }}/release/golem-shard-manager \
            ${{ matrix.platform.target }}/release/worker-executor \
            ${{ matrix.platform.target }}/release/golem-template-service  \
            ${{ matrix.platform.target }}/release/golem-worker-service  \
            ${{ matrix.platform.target }}/release/golem-template-compilation-service 
      - uses: actions/upload-artifact@v4
        name: Upload Targets
        with:
          name: docker-targets-build-${{ env.PLATFORM_PAIR }}
          path: target/${{ matrix.platform.target }}.tar
  docker-publish:
    runs-on: ubuntu-latest
    needs: [docker-targets-build]
    if: github.event_name == 'push' && github.ref_type == 'tag'
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          submodules: recursive
          fetch-depth: "0"
      - name: Prepare
        run: |
          echo "PLATFORMS=linux/amd64,linux/arm64" >> $GITHUB_ENV
      - uses: actions/download-artifact@v4
        name: Download Targets
        with:
          pattern: docker-targets-build-*
          path: target
          merge-multiple: true
      - name: Extract Targets
        run: |
          ls -R target
          cd target
          for f in *.tar; do tar xvf "$f"; done
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Set Docker version
        id: set-version
        run: |
          if [ "${{ github.event_name }}" == 'push' ] && [ "${{ github.ref_type }}" == 'tag' ]; then
            DOCKER_VERSION=$(echo "${{ github.ref }}" | sed 's|^refs/tags/v||')
            echo "DOCKER_VERSION=${DOCKER_VERSION}" >> $GITHUB_ENV
          else
            COMMIT_SHORT_HASH=$(git rev-parse --short=7 HEAD)
            echo "DOCKER_VERSION=${COMMIT_SHORT_HASH}" >> $GITHUB_ENV
          fi
      - uses: jpribyl/action-docker-layer-caching@v0.1.1
        continue-on-error: true
      - name: Log in to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}
      - name: Extract metadata (tags, labels) for worker executor
        id: meta-worker-executor
        uses: docker/metadata-action@v5
        with:
          images: golemservices/golem-worker-executor
      - name: Build and push worker executor image
        uses: docker/build-push-action@v5
        with:
          context: .
          file: ./golem-worker-executor/docker/Dockerfile
          push: true
          platforms: ${{ env.PLATFORMS }}
          tags: ${{ steps.meta-worker-executor.outputs.tags }}
          labels: ${{ steps.meta-worker-executor.outputs.labels }}
      - name: Extract metadata (tags, labels) for shard manager
        id: meta-shard-manager
        uses: docker/metadata-action@v5
        with:
          images: golemservices/golem-shard-manager
      - name: Build and push shard manager image
        uses: docker/build-push-action@v5
        with:
          context: .
          file: ./golem-shard-manager/docker/Dockerfile
          push: true
          platforms: ${{ env.PLATFORMS }}
          tags: ${{ steps.meta-shard-manager.outputs.tags }}
          labels: ${{ steps.meta-shard-manager.outputs.labels }}
      - name: Extract metadata (tags, labels) for golem template service
        id: meta-golem-template-service
        uses: docker/metadata-action@v5
        with:
          images: golemservices/golem-template-service
      - name: Build and push golem template service image
        uses: docker/build-push-action@v5
        with:
          context: .
          file: ./golem-template-service/docker/Dockerfile
          push: true
          platforms: ${{ env.PLATFORMS }}
          tags: ${{ steps.meta-golem-template-service.outputs.tags }}
          labels: ${{ steps.meta-golem-template-service.outputs.labels }}
      - name: Extract metadata (tags, labels) for golem worker service
        id: meta-golem-worker-service
        uses: docker/metadata-action@v5
        with:
          images: golemservices/golem-worker-service
      - name: Build and push golem worker service image
        uses: docker/build-push-action@v5
        with:
          context: .
          file: ./golem-worker-service/docker/Dockerfile
          push: true
          platforms: ${{ env.PLATFORMS }}
          tags: ${{ steps.meta-golem-worker-service.outputs.tags }}
          labels: ${{ steps.meta-golem-worker-service.outputs.labels }}
      - name: Extract metadata (tags, labels) for golem template compilation service
        id: meta-golem-template-compilation-service
        uses: docker/metadata-action@v5
        with:
          images: golemservices/golem-template-compilation-service
      - name: Build and push golem template compilation service image
        uses: docker/build-push-action@v5
        with:
          context: .
          file: ./golem-template-compilation-service/docker/Dockerfile
          push: true
          platforms: ${{ env.PLATFORMS }}
          tags: ${{ steps.meta-golem-template-compilation-service.outputs.tags }}
          labels: ${{ steps.meta-golem-template-compilation-service.outputs.labels }}
      - name: Extract metadata (tags, labels) for golem router
        id: meta-golem-router
        uses: docker/metadata-action@v5
        with:
          images: golemservices/golem-router
      - name: Build and push golem router
        uses: docker/build-push-action@v5
        with:
          context: .
          file: ./golem-router/docker/Dockerfile
          push: true
          platforms: ${{ env.PLATFORMS }}
          tags: ${{ steps.meta-golem-router.outputs.tags }}
          labels: ${{ steps.meta-golem-router.outputs.labels }}
  unit-tests:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          submodules: recursive
      - uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/.crates2.json
            ~/.cargo/.crates.toml
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: ${{ runner.os }}-cargo-debug-${{ hashFiles('**/Cargo.lock') }}-is
      - name: Setup Rust
        uses: actions-rs/toolchain@v1
        with:
          toolchain: stable
          override: true
      - name: Install Protoc
        uses: arduino/setup-protoc@v1
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Check formatting
        run: cargo fmt -- --check
      - name: Clippy
        run: cargo clippy -- -Dwarnings
      - name: Unit tests
        run: cargo test --lib --bins --all-features
  worker-tests:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          submodules: recursive
      - uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/.crates2.json
            ~/.cargo/.crates.toml
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: ${{ runner.os }}-cargo-debug-${{ hashFiles('**/Cargo.lock') }}-is
      - name: Setup Rust
        uses: actions-rs/toolchain@v1
        with:
          toolchain: stable
          override: true
      - name: Install Protoc
        uses: arduino/setup-protoc@v1
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Setup Redis
        uses: shogo82148/actions-setup-redis@v1.29.0
        with:
          redis-version: latest
          auto-start: false
      - name: "Worker Executor integration tests"
        run: WASMTIME_BACKTRACE_DETAILS=1 cargo test --package golem-worker-executor-base --test '*' -- --nocapture
  build:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          submodules: recursive
      - uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/.crates2.json
            ~/.cargo/.crates.toml
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: ${{ runner.os }}-cargo-debug-${{ hashFiles('**/Cargo.lock') }}-is
      - name: Setup Rust
        uses: actions-rs/toolchain@v1
        with:
          toolchain: stable
          override: true
      - name: Install Protoc
        uses: arduino/setup-protoc@v1
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Build all
        # --release to minimize artifact size
        run: cargo build --release
      - name: Upload binaries
        uses: actions/upload-artifact@v4
        with:
          name: golem-binaries
          retention-days: 1
          path: |
            target/release/golem-template-service
            target/release/golem-worker-service
            target/release/golem-template-compilation-service
            target/release/golem-worker-service-yaml
            target/release/golem-template-service-yaml
            target/release/golem-shard-manager
            target/release/worker-executor
  integration-tests:
    needs: [build]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          submodules: recursive
      - uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/.crates2.json
            ~/.cargo/.crates.toml
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            golem-cli/target/
          key: ${{ runner.os }}-cargo-debug-${{ hashFiles('**/Cargo.lock') }}-is
      - name: Setup Rust
        uses: actions-rs/toolchain@v1
        with:
          toolchain: stable
          override: true
      - name: Install Protoc
        uses: arduino/setup-protoc@v1
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Setup Redis
        uses: shogo82148/actions-setup-redis@v1.29.0
        with:
          redis-version: latest
          auto-start: false
      - name: Download binaries
        uses: actions/download-artifact@v4
        with:
          name: golem-binaries
          path: ./target/debug/
      - name: Restore executable flag
        run: chmod a+x ./target/debug/*
      - name: Generate golem-template-service OpenAPI yaml
        run: ./target/debug/golem-template-service-yaml > ./target/golem-template-service.yaml
      - name: Generate golem-worker-service OpenAPI yaml
        run: ./target/debug/golem-worker-service-yaml > ./target/golem-worker-service.yaml
      - name: Merge openapi specs
        run: |
          cargo install golem-openapi-client-generator@0.0.1
          golem-openapi-client-generator merge --spec-yaml target/golem-template-service.yaml target/golem-worker-service.yaml --output-yaml target/golem-service.yaml
      - name: Check Golem OpenAPI yaml latest
        shell: bash
        run: |
          if diff openapi/golem-service.yaml target/golem-service.yaml  >/dev/null 2>&1
            then
              echo "Latest Golem OpenAPI spec version detected."
            else
              echo "openapi/golem-service.yaml is not the same as produced by golem-service-yaml." 1>&2
              echo "Run ./scripts/generate-openapi-yaml.sh to generate new spec." 1>&2
              exit 1
          fi
      - name: Generate Golem client
        run: |
          expected_version=$( scripts/expected-client-version.sh )
          cargo install golem-openapi-client-generator@0.0.1
          golem-openapi-client-generator generate --spec-yaml openapi/golem-service.yaml --output-directory golem-client --name "golem-client" --client-version "${expected_version}"
      - name: Integration tests
        working-directory: golem-cli
        env:
          CI: true
        run: |
          mkdir -p ./.cargo
          echo 'paths = ["../golem-client"]' > ./.cargo/config.toml
          cargo build
          RUST_LOG=info cargo test --test integration
      - name: Sharding tests
        working-directory: golem-cli
        env:
          CI: true
        run: |
          mkdir -p ./.cargo
          echo 'paths = ["../golem-client"]' > ./.cargo/config.toml
          cargo build
          RUST_LOG=info cargo test --test sharding
        timeout-minutes: 10
  publish:
    needs: [unit-tests, worker-tests, integration-tests]
    if: "startsWith(github.ref, 'refs/tags/v')"
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          submodules: recursive
      - uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/.crates2.json
            ~/.cargo/.crates.toml
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: ${{ runner.os }}-cargo-debug-${{ hashFiles('**/Cargo.lock') }}-is

      - name: Setup Rust
        uses: actions-rs/toolchain@v1
        with:
          toolchain: stable
          override: true
      - name: Install Protoc
        uses: arduino/setup-protoc@v1
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - id: get_version
        uses: battila7/get-version-action@v2
      - name: Generate Golem client
        run: |
          export VERSION="${{ steps.get_version.outputs.version-without-v }}"
          cargo install golem-openapi-client-generator@0.0.1
          golem-openapi-client-generator generate --spec-yaml openapi/golem-service.yaml --output-directory golem-client --name "golem-client" --client-version "${VERSION}"
      - name: Publish Golem client
        env:
          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
        working-directory: golem-client
        run: cargo publish --all-features --allow-dirty
      - name: Publish Golem CLI
        env:
          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
        run: |
          export VERSION="${{ steps.get_version.outputs.version-without-v }}"
          sed -i "s/0.0.0/$VERSION/g" golem-cli/Cargo.toml
          cargo publish -p golem-cli --all-features --allow-dirty