Reusable Worker Service #907
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: CI | |
on: | |
push: | |
tags: | |
- "v*.*.*" | |
branches: | |
- main | |
pull_request: | |
concurrency: | |
group: ${{ github.workflow }}-${{ github.ref }} | |
cancel-in-progress: true | |
env: | |
BUILD_TARGET: "x86_64-unknown-linux-gnu" | |
jobs: | |
docker-targets-build: | |
runs-on: ubuntu-latest | |
if: github.event_name == 'push' && github.ref_type == 'tag' | |
strategy: | |
fail-fast: false | |
matrix: | |
platform: | |
- platform: linux/amd64 | |
target: x86_64-unknown-linux-gnu | |
build-config: "" | |
- platform: linux/arm64 | |
target: aarch64-unknown-linux-gnu | |
build-config: '--config "target.aarch64-unknown-linux-gnu.linker = ''aarch64-linux-gnu-gcc''"' | |
name: docker-targets-build (${{ matrix.platform.platform }}) | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v3 | |
with: | |
submodules: recursive | |
fetch-depth: "0" | |
- name: Prepare | |
run: | | |
platform=${{ matrix.platform.platform }} | |
echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV | |
- name: Setup Rust | |
uses: actions-rs/toolchain@v1 | |
with: | |
toolchain: stable | |
target: ${{ matrix.platform.target }} | |
override: true | |
- name: Install Rust Target | |
run: rustup target add ${{ matrix.platform.target }} | |
- name: Install Protoc | |
uses: arduino/setup-protoc@v1 | |
with: | |
repo-token: ${{ secrets.GITHUB_TOKEN }} | |
- name: Install gcc-aarch64-linux-gnu | |
if: matrix.platform.platform == 'linux/arm64' | |
run: | | |
sudo apt-get update | |
sudo apt-get install gcc-aarch64-linux-gnu | |
- uses: actions/cache@v3 | |
with: | |
path: | | |
~/.cargo/bin/ | |
~/.cargo/.crates2.json | |
~/.cargo/.crates.toml | |
~/.cargo/registry/index/ | |
~/.cargo/registry/cache/ | |
~/.cargo/git/db/ | |
target/ | |
key: ${{ runner.os }}-cargo-release-${{ hashFiles('**/Cargo.lock') }} | |
- name: Build Executables | |
run: cargo build --release --target ${{ matrix.platform.target }} ${{ matrix.platform.build-config }} | |
- name: Prepare Targets | |
run: | | |
cd target | |
tar -cvf ${{ matrix.platform.target }}.tar \ | |
${{ matrix.platform.target }}/release/golem-shard-manager \ | |
${{ matrix.platform.target }}/release/worker-executor \ | |
${{ matrix.platform.target }}/release/golem-template-service \ | |
${{ matrix.platform.target }}/release/golem-worker-service \ | |
${{ matrix.platform.target }}/release/golem-template-compilation-service | |
- uses: actions/upload-artifact@v4 | |
name: Upload Targets | |
with: | |
name: docker-targets-build-${{ env.PLATFORM_PAIR }} | |
path: target/${{ matrix.platform.target }}.tar | |
docker-publish: | |
runs-on: ubuntu-latest | |
needs: [docker-targets-build] | |
if: github.event_name == 'push' && github.ref_type == 'tag' | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v3 | |
with: | |
submodules: recursive | |
fetch-depth: "0" | |
- name: Prepare | |
run: | | |
echo "PLATFORMS=linux/amd64,linux/arm64" >> $GITHUB_ENV | |
- uses: actions/download-artifact@v4 | |
name: Download Targets | |
with: | |
pattern: docker-targets-build-* | |
path: target | |
merge-multiple: true | |
- name: Extract Targets | |
run: | | |
ls -R target | |
cd target | |
for f in *.tar; do tar xvf "$f"; done | |
- name: Set up QEMU | |
uses: docker/setup-qemu-action@v3 | |
- name: Set up Docker Buildx | |
uses: docker/setup-buildx-action@v3 | |
- name: Set Docker version | |
id: set-version | |
run: | | |
if [ "${{ github.event_name }}" == 'push' ] && [ "${{ github.ref_type }}" == 'tag' ]; then | |
DOCKER_VERSION=$(echo "${{ github.ref }}" | sed 's|^refs/tags/v||') | |
echo "DOCKER_VERSION=${DOCKER_VERSION}" >> $GITHUB_ENV | |
else | |
COMMIT_SHORT_HASH=$(git rev-parse --short=7 HEAD) | |
echo "DOCKER_VERSION=${COMMIT_SHORT_HASH}" >> $GITHUB_ENV | |
fi | |
- uses: jpribyl/[email protected] | |
continue-on-error: true | |
- name: Log in to Docker Hub | |
uses: docker/login-action@v3 | |
with: | |
username: ${{ secrets.DOCKERHUB_USERNAME }} | |
password: ${{ secrets.DOCKERHUB_PASSWORD }} | |
- name: Extract metadata (tags, labels) for worker executor | |
id: meta-worker-executor | |
uses: docker/metadata-action@v5 | |
with: | |
images: golemservices/golem-worker-executor | |
- name: Build and push worker executor image | |
uses: docker/build-push-action@v5 | |
with: | |
context: . | |
file: ./golem-worker-executor/docker/Dockerfile | |
push: true | |
platforms: ${{ env.PLATFORMS }} | |
tags: ${{ steps.meta-worker-executor.outputs.tags }} | |
labels: ${{ steps.meta-worker-executor.outputs.labels }} | |
- name: Extract metadata (tags, labels) for shard manager | |
id: meta-shard-manager | |
uses: docker/metadata-action@v5 | |
with: | |
images: golemservices/golem-shard-manager | |
- name: Build and push shard manager image | |
uses: docker/build-push-action@v5 | |
with: | |
context: . | |
file: ./golem-shard-manager/docker/Dockerfile | |
push: true | |
platforms: ${{ env.PLATFORMS }} | |
tags: ${{ steps.meta-shard-manager.outputs.tags }} | |
labels: ${{ steps.meta-shard-manager.outputs.labels }} | |
- name: Extract metadata (tags, labels) for golem template service | |
id: meta-golem-template-service | |
uses: docker/metadata-action@v5 | |
with: | |
images: golemservices/golem-template-service | |
- name: Build and push golem template service image | |
uses: docker/build-push-action@v5 | |
with: | |
context: . | |
file: ./golem-template-service/docker/Dockerfile | |
push: true | |
platforms: ${{ env.PLATFORMS }} | |
tags: ${{ steps.meta-golem-template-service.outputs.tags }} | |
labels: ${{ steps.meta-golem-template-service.outputs.labels }} | |
- name: Extract metadata (tags, labels) for golem worker service | |
id: meta-golem-worker-service | |
uses: docker/metadata-action@v5 | |
with: | |
images: golemservices/golem-worker-service | |
- name: Build and push golem worker service image | |
uses: docker/build-push-action@v5 | |
with: | |
context: . | |
file: ./golem-worker-service/docker/Dockerfile | |
push: true | |
platforms: ${{ env.PLATFORMS }} | |
tags: ${{ steps.meta-golem-worker-service.outputs.tags }} | |
labels: ${{ steps.meta-golem-worker-service.outputs.labels }} | |
- name: Extract metadata (tags, labels) for golem template compilation service | |
id: meta-golem-template-compilation-service | |
uses: docker/metadata-action@v5 | |
with: | |
images: golemservices/golem-template-compilation-service | |
- name: Build and push golem template compilation service image | |
uses: docker/build-push-action@v5 | |
with: | |
context: . | |
file: ./golem-template-compilation-service/docker/Dockerfile | |
push: true | |
platforms: ${{ env.PLATFORMS }} | |
tags: ${{ steps.meta-golem-template-compilation-service.outputs.tags }} | |
labels: ${{ steps.meta-golem-template-compilation-service.outputs.labels }} | |
- name: Extract metadata (tags, labels) for golem router | |
id: meta-golem-router | |
uses: docker/metadata-action@v5 | |
with: | |
images: golemservices/golem-router | |
- name: Build and push golem router | |
uses: docker/build-push-action@v5 | |
with: | |
context: . | |
file: ./golem-router/docker/Dockerfile | |
push: true | |
platforms: ${{ env.PLATFORMS }} | |
tags: ${{ steps.meta-golem-router.outputs.tags }} | |
labels: ${{ steps.meta-golem-router.outputs.labels }} | |
unit-tests: | |
runs-on: ubuntu-latest | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v3 | |
with: | |
submodules: recursive | |
- uses: actions/cache@v3 | |
with: | |
path: | | |
~/.cargo/bin/ | |
~/.cargo/.crates2.json | |
~/.cargo/.crates.toml | |
~/.cargo/registry/index/ | |
~/.cargo/registry/cache/ | |
~/.cargo/git/db/ | |
target/ | |
key: ${{ runner.os }}-cargo-debug-${{ hashFiles('**/Cargo.lock') }}-is | |
- name: Setup Rust | |
uses: actions-rs/toolchain@v1 | |
with: | |
toolchain: stable | |
override: true | |
- name: Install Protoc | |
uses: arduino/setup-protoc@v1 | |
with: | |
repo-token: ${{ secrets.GITHUB_TOKEN }} | |
- name: Check formatting | |
run: cargo fmt -- --check | |
- name: Clippy | |
run: cargo clippy -- -Dwarnings | |
- name: Unit tests | |
run: cargo test --lib --bins --all-features | |
worker-tests: | |
runs-on: ubuntu-latest | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v3 | |
with: | |
submodules: recursive | |
- uses: actions/cache@v3 | |
with: | |
path: | | |
~/.cargo/bin/ | |
~/.cargo/.crates2.json | |
~/.cargo/.crates.toml | |
~/.cargo/registry/index/ | |
~/.cargo/registry/cache/ | |
~/.cargo/git/db/ | |
target/ | |
key: ${{ runner.os }}-cargo-debug-${{ hashFiles('**/Cargo.lock') }}-is | |
- name: Setup Rust | |
uses: actions-rs/toolchain@v1 | |
with: | |
toolchain: stable | |
override: true | |
- name: Install Protoc | |
uses: arduino/setup-protoc@v1 | |
with: | |
repo-token: ${{ secrets.GITHUB_TOKEN }} | |
- name: Setup Redis | |
uses: shogo82148/[email protected] | |
with: | |
redis-version: latest | |
auto-start: false | |
- name: "Worker Executor integration tests" | |
run: WASMTIME_BACKTRACE_DETAILS=1 cargo test --package golem-worker-executor-base --test '*' -- --nocapture | |
build: | |
runs-on: ubuntu-latest | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v3 | |
with: | |
submodules: recursive | |
- uses: actions/cache@v3 | |
with: | |
path: | | |
~/.cargo/bin/ | |
~/.cargo/.crates2.json | |
~/.cargo/.crates.toml | |
~/.cargo/registry/index/ | |
~/.cargo/registry/cache/ | |
~/.cargo/git/db/ | |
target/ | |
key: ${{ runner.os }}-cargo-debug-${{ hashFiles('**/Cargo.lock') }}-is | |
- name: Setup Rust | |
uses: actions-rs/toolchain@v1 | |
with: | |
toolchain: stable | |
override: true | |
- name: Install Protoc | |
uses: arduino/setup-protoc@v1 | |
with: | |
repo-token: ${{ secrets.GITHUB_TOKEN }} | |
- name: Build all | |
# --release to minimize artifact size | |
run: cargo build --release | |
- name: Upload binaries | |
uses: actions/upload-artifact@v4 | |
with: | |
name: golem-binaries | |
retention-days: 1 | |
path: | | |
target/release/golem-template-service | |
target/release/golem-worker-service | |
target/release/golem-template-compilation-service | |
target/release/golem-worker-service-yaml | |
target/release/golem-template-service-yaml | |
target/release/golem-shard-manager | |
target/release/worker-executor | |
integration-tests: | |
needs: [build] | |
runs-on: ubuntu-latest | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v3 | |
with: | |
submodules: recursive | |
- uses: actions/cache@v3 | |
with: | |
path: | | |
~/.cargo/bin/ | |
~/.cargo/.crates2.json | |
~/.cargo/.crates.toml | |
~/.cargo/registry/index/ | |
~/.cargo/registry/cache/ | |
~/.cargo/git/db/ | |
golem-cli/target/ | |
key: ${{ runner.os }}-cargo-debug-${{ hashFiles('**/Cargo.lock') }}-is | |
- name: Setup Rust | |
uses: actions-rs/toolchain@v1 | |
with: | |
toolchain: stable | |
override: true | |
- name: Install Protoc | |
uses: arduino/setup-protoc@v1 | |
with: | |
repo-token: ${{ secrets.GITHUB_TOKEN }} | |
- name: Setup Redis | |
uses: shogo82148/[email protected] | |
with: | |
redis-version: latest | |
auto-start: false | |
- name: Download binaries | |
uses: actions/download-artifact@v4 | |
with: | |
name: golem-binaries | |
path: ./target/debug/ | |
- name: Restore executable flag | |
run: chmod a+x ./target/debug/* | |
- name: Generate golem-template-service OpenAPI yaml | |
run: ./target/debug/golem-template-service-yaml > ./target/golem-template-service.yaml | |
- name: Generate golem-worker-service OpenAPI yaml | |
run: ./target/debug/golem-worker-service-yaml > ./target/golem-worker-service.yaml | |
- name: Merge openapi specs | |
run: | | |
cargo install [email protected] | |
golem-openapi-client-generator merge --spec-yaml target/golem-template-service.yaml target/golem-worker-service.yaml --output-yaml target/golem-service.yaml | |
- name: Check Golem OpenAPI yaml latest | |
shell: bash | |
run: | | |
if diff openapi/golem-service.yaml target/golem-service.yaml >/dev/null 2>&1 | |
then | |
echo "Latest Golem OpenAPI spec version detected." | |
else | |
echo "openapi/golem-service.yaml is not the same as produced by golem-service-yaml." 1>&2 | |
echo "Run ./scripts/generate-openapi-yaml.sh to generate new spec." 1>&2 | |
exit 1 | |
fi | |
- name: Integration tests | |
working-directory: golem-cli | |
env: | |
CI: true | |
run: | | |
cargo build | |
RUST_LOG=info cargo test --test integration | |
- name: Sharding tests | |
working-directory: golem-cli | |
env: | |
CI: true | |
run: | | |
cargo build | |
RUST_LOG=info cargo test --test sharding | |
timeout-minutes: 10 | |
publish: | |
needs: [unit-tests, worker-tests, integration-tests] | |
if: "startsWith(github.ref, 'refs/tags/v')" | |
runs-on: ubuntu-latest | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v3 | |
with: | |
submodules: recursive | |
- uses: actions/cache@v3 | |
with: | |
path: | | |
~/.cargo/bin/ | |
~/.cargo/.crates2.json | |
~/.cargo/.crates.toml | |
~/.cargo/registry/index/ | |
~/.cargo/registry/cache/ | |
~/.cargo/git/db/ | |
target/ | |
key: ${{ runner.os }}-cargo-debug-${{ hashFiles('**/Cargo.lock') }}-is | |
- name: Setup Rust | |
uses: actions-rs/toolchain@v1 | |
with: | |
toolchain: stable | |
override: true | |
- name: Install Protoc | |
uses: arduino/setup-protoc@v1 | |
with: | |
repo-token: ${{ secrets.GITHUB_TOKEN }} | |
- id: get_version | |
uses: battila7/get-version-action@v2 | |
- name: Publish Golem client | |
env: | |
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} | |
run: | | |
export VERSION="${{ steps.get_version.outputs.version-without-v }}" | |
sed -i "s/0.0.0/$VERSION/g" golem-client/Cargo.toml | |
cargo publish -p golem-client --all-features --allow-dirty | |
- name: Publish Golem CLI | |
env: | |
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} | |
run: | | |
export VERSION="${{ steps.get_version.outputs.version-without-v }}" | |
sed -i "s/0.0.0/$VERSION/g" golem-cli/Cargo.toml | |
cargo publish -p golem-cli --all-features --allow-dirty |