Skip to content
Run Tests

check-tests CI workflow benchmark tests for manta-dev (#1197) #4068

Sign in to view logs

check-tests CI workflow benchmark tests for manta-dev (#1197)

check-tests CI workflow benchmark tests for manta-dev (#1197) #4068

Workflow file for this run

.github/workflows/check_tests.yml at 9b85510
name: Run Tests
on:
pull_request:
branches: [manta]
push:
branches: [manta]
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
env:
AWS_REGION: us-east-1
AWS_INSTANCE_TYPE: c5.9xlarge
AWS_INSTANCE_ROOT_VOLUME_SIZE: 1024
AWS_IMAGE_SEARCH_PATTERN: ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*
AWS_IMAGE_SEARCH_OWNERS: '["099720109477"]'
jobs:
start-integration-test-checks:
timeout-minutes: 480
runs-on: ubuntu-20.04
outputs:
runner-label: ${{ steps.start-self-hosted-runner.outputs.runner-label }}
aws-region: ${{ steps.start-self-hosted-runner.outputs.aws-region }}
aws-instance-id: ${{ steps.start-self-hosted-runner.outputs.aws-instance-id }}
steps:
- id: start-self-hosted-runner
uses: audacious-network/[email protected]
with:
mode: start
github-token: ${{ secrets.SELF_HOSTED_RUNNER_TOKEN }}
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ env.AWS_REGION }}
aws-instance-type: ${{ env.AWS_INSTANCE_TYPE }}
aws-instance-root-volume-size: ${{ env.AWS_INSTANCE_ROOT_VOLUME_SIZE }}
aws-image-search-pattern: ${{ env.AWS_IMAGE_SEARCH_PATTERN }}
aws-image-search-owners: ${{ env.AWS_IMAGE_SEARCH_OWNERS }}
- uses: actions/checkout@v2
- name: install sccache
env:
SCCACHE_RELEASE_URL: https://github.com/mozilla/sccache/releases/download
SCCACHE_VERSION: v0.5.3
run: |
SCCACHE_FILE=sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl
mkdir -p $HOME/.local/bin
curl -L "$SCCACHE_RELEASE_URL/$SCCACHE_VERSION/$SCCACHE_FILE.tar.gz" | tar xz
mv -f $SCCACHE_FILE/sccache $HOME/.local/bin/sccache
chmod +x $HOME/.local/bin/sccache
echo "$HOME/.local/bin" >> $GITHUB_PATH
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
- name: cache cargo registry
uses: actions/cache@v2
with:
path: |
~/.cargo/registry
~/.cargo/git
key: cargo-${{ hashFiles('**/Cargo.lock') }}
restore-keys: |
cargo-
- name: cache sccache
uses: actions/cache@v2
continue-on-error: false
with:
path: /home/runner/.cache/sccache
key: sccache-${{ hashFiles('**/Cargo.lock') }}
restore-keys: |
sccache-
- name: start sccache server
run: sccache --start-server
- name: init
run: |
sudo apt update
sudo apt install -y pkg-config libssl-dev protobuf-compiler
protoc --version
curl -s https://sh.rustup.rs -sSf | sh -s -- -y
source ${HOME}/.cargo/env
rustup update
rustup toolchain install nightly-2023-03-03
rustup default nightly-2023-03-03
rustup target add wasm32-unknown-unknown
- name: Run Integration Tests
env:
RUST_BACKTRACE: full
RUSTC_WRAPPER: sccache
SCCACHE_CACHE_SIZE: 120G
SCCACHE_DIR: /home/runner/.cache/sccache
run: |
source ${HOME}/.cargo/env
RUSTC_BOOTSTRAP=1 cargo test -p integration-tests --release --features=calamari --no-default-features
RUSTC_BOOTSTRAP=1 cargo test -p integration-tests --release --features=manta --no-default-features
- name: stop sccache server
run: sccache --stop-server || true
stop-integration-test-checks:
timeout-minutes: 15
needs: start-integration-test-checks
runs-on: ubuntu-20.04
if: ${{ always() }}
steps:
- continue-on-error: true
uses: audacious-network/[email protected]
with:
mode: stop
github-token: ${{ secrets.SELF_HOSTED_RUNNER_TOKEN }}
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ needs.start-integration-test-checks.outputs.aws-region }}
runner-label: ${{ needs.start-integration-test-checks.outputs.runner-label }}
aws-instance-id: ${{ needs.start-integration-test-checks.outputs.aws-instance-id }}
- name: discard stopper success/failure
run: true
start-benchmark-checks:
timeout-minutes: 180
runs-on: ubuntu-20.04
outputs:
runner-label: ${{ steps.start-self-hosted-runner.outputs.runner-label }}
aws-region: ${{ steps.start-self-hosted-runner.outputs.aws-region }}
aws-instance-id: ${{ steps.start-self-hosted-runner.outputs.aws-instance-id }}
steps:
- id: start-self-hosted-runner
uses: audacious-network/[email protected]
with:
mode: start
github-token: ${{ secrets.SELF_HOSTED_RUNNER_TOKEN }}
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ env.AWS_REGION }}
aws-instance-type: ${{ env.AWS_INSTANCE_TYPE }}
aws-instance-root-volume-size: ${{ env.AWS_INSTANCE_ROOT_VOLUME_SIZE }}
aws-image-search-pattern: ${{ env.AWS_IMAGE_SEARCH_PATTERN }}
aws-image-search-owners: ${{ env.AWS_IMAGE_SEARCH_OWNERS }}
- uses: actions/checkout@v2
- name: install sccache
env:
SCCACHE_RELEASE_URL: https://github.com/mozilla/sccache/releases/download
SCCACHE_VERSION: v0.5.3
run: |
SCCACHE_FILE=sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl
mkdir -p $HOME/.local/bin
curl -L "$SCCACHE_RELEASE_URL/$SCCACHE_VERSION/$SCCACHE_FILE.tar.gz" | tar xz
mv -f $SCCACHE_FILE/sccache $HOME/.local/bin/sccache
chmod +x $HOME/.local/bin/sccache
echo "$HOME/.local/bin" >> $GITHUB_PATH
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
- name: cache cargo registry
uses: actions/cache@v2
with:
path: |
~/.cargo/registry
~/.cargo/git
key: cargo-${{ hashFiles('**/Cargo.lock') }}
restore-keys: |
cargo-
- name: cache sccache
uses: actions/cache@v2
continue-on-error: false
with:
path: /home/runner/.cache/sccache
key: sccache-${{ hashFiles('**/Cargo.lock') }}
restore-keys: |
sccache-
- name: start sccache server
run: sccache --start-server
- name: build benchmarking binary
env:
RUST_BACKTRACE: full
RUSTC_WRAPPER: sccache
SCCACHE_CACHE_SIZE: 120G
SCCACHE_DIR: /home/runner/.cache/sccache
CARGO_TERM_COLOR: always
run: |
sudo apt update
sudo apt install -y pkg-config libssl-dev protobuf-compiler
protoc --version
curl -s https://sh.rustup.rs -sSf | sh -s -- -y
source ${HOME}/.cargo/env
rustup toolchain install nightly-2023-03-03
rustup default nightly-2023-03-03
rustup target add wasm32-unknown-unknown
df -h
sudo apt remove -y '^dotnet-.*'
sudo apt remove -y '^mongodb-.*'
sudo apt remove -y '^mysql-.*'
sudo apt remove -y '^postgresql-*'
sudo apt remove -y google-cloud-sdk google-chrome-stable firefox powershell mono-devel libgl1-mesa-dri
sudo apt autoremove -y
sudo apt clean
sudo rm -rf /usr/local/lib/android
sudo rm -rf /usr/share/dotnet
df -h
- name: build benchmarking binary
run: |
RUSTC_BOOTSTRAP=1 cargo build --release --features runtime-benchmarks
- name: create-calamari-chainspec
run: |
${{ github.workspace }}/target/release/manta build-spec --chain calamari-dev --disable-default-bootnode --raw > ${{ github.workspace }}/tests/data/fork.json
- name: append manta-pay storage for Calamari
run: |
wget -P ${{ github.workspace }}/tests/data https://manta-ops.s3.amazonaws.com/integration-tests-data/storage.json
cd ${{ github.workspace }}/tests
yarn install
yarn
node append_storage.js
- name: Run live benchmarks test
env:
RUST_BACKTRACE: full
RUSTC_WRAPPER: sccache
SCCACHE_CACHE_SIZE: 120G
SCCACHE_DIR: /home/runner/.cache/sccache
run: |
${{ github.workspace }}/target/release/manta \
benchmark \
pallet \
--chain=${{ github.workspace }}/tests/data/fork.json \
--pallet=* \
--extrinsic=* \
--repeat=1 \
--steps=2
- name: create-manta-dev-chainspec
run: |
${{ github.workspace }}/target/release/manta build-spec --chain manta-dev --disable-default-bootnode --raw > ${{ github.workspace }}/tests/data/fork.json
- name: append manta-pay storage for Manta
run: |
wget -P ${{ github.workspace }}/tests/data https://manta-ops.s3.amazonaws.com/integration-tests-data/storage.json
cd ${{ github.workspace }}/tests
yarn install
yarn
node append_storage.js
- name: Run live benchmarks test
env:
RUST_BACKTRACE: full
RUSTC_WRAPPER: sccache
SCCACHE_CACHE_SIZE: 120G
SCCACHE_DIR: /home/runner/.cache/sccache
run: |
${{ github.workspace }}/target/release/manta \
benchmark \
pallet \
--chain=${{ github.workspace }}/tests/data/fork.json \
--pallet=* \
--extrinsic=* \
--repeat=1 \
--steps=2
- name: stop sccache server
run: sccache --stop-server || true
stop-benchmark-checks:
needs: start-benchmark-checks
runs-on: ubuntu-20.04
if: ${{ always() }}
steps:
- continue-on-error: true
uses: audacious-network/[email protected]
with:
mode: stop
github-token: ${{ secrets.SELF_HOSTED_RUNNER_TOKEN }}
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ needs.start-benchmark-checks.outputs.aws-region }}
runner-label: ${{ needs.start-benchmark-checks.outputs.runner-label }}
aws-instance-id: ${{ needs.start-benchmark-checks.outputs.aws-instance-id }}
- name: discard stopper success/failure
run: true
start-unit-test-checks:
timeout-minutes: 480
runs-on: ubuntu-20.04
outputs:
runner-label: ${{ steps.start-self-hosted-runner.outputs.runner-label }}
aws-region: ${{ steps.start-self-hosted-runner.outputs.aws-region }}
aws-instance-id: ${{ steps.start-self-hosted-runner.outputs.aws-instance-id }}
steps:
- id: start-self-hosted-runner
uses: audacious-network/[email protected]
with:
mode: start
github-token: ${{ secrets.SELF_HOSTED_RUNNER_TOKEN }}
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ env.AWS_REGION }}
aws-instance-type: ${{ env.AWS_INSTANCE_TYPE }}
aws-instance-root-volume-size: ${{ env.AWS_INSTANCE_ROOT_VOLUME_SIZE }}
aws-image-search-pattern: ${{ env.AWS_IMAGE_SEARCH_PATTERN }}
aws-image-search-owners: ${{ env.AWS_IMAGE_SEARCH_OWNERS }}
- uses: actions/checkout@v2
- name: install sccache
env:
SCCACHE_RELEASE_URL: https://github.com/mozilla/sccache/releases/download
SCCACHE_VERSION: v0.5.3
run: |
SCCACHE_FILE=sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl
mkdir -p $HOME/.local/bin
curl -L "$SCCACHE_RELEASE_URL/$SCCACHE_VERSION/$SCCACHE_FILE.tar.gz" | tar xz
mv -f $SCCACHE_FILE/sccache $HOME/.local/bin/sccache
chmod +x $HOME/.local/bin/sccache
echo "$HOME/.local/bin" >> $GITHUB_PATH
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
- name: cache cargo registry
uses: actions/cache@v2
with:
path: |
~/.cargo/registry
~/.cargo/git
key: cargo-${{ hashFiles('**/Cargo.lock') }}
restore-keys: |
cargo-
- name: cache sccache
uses: actions/cache@v2
continue-on-error: false
with:
path: /home/runner/.cache/sccache
key: sccache-${{ hashFiles('**/Cargo.lock') }}
restore-keys: |
sccache-
- name: start sccache server
run: sccache --start-server
- name: init
run: |
sudo apt update
sudo apt install -y pkg-config libssl-dev protobuf-compiler
protoc --version
curl -s https://sh.rustup.rs -sSf | sh -s -- -y
source ${HOME}/.cargo/env
rustup update
rustup toolchain install nightly-2023-03-03
rustup default nightly-2023-03-03
rustup target add wasm32-unknown-unknown
df -h
sudo apt remove -y '^dotnet-.*'
sudo apt remove -y '^mongodb-.*'
sudo apt remove -y '^mysql-.*'
sudo apt remove -y '^postgresql-*'
sudo apt remove -y google-cloud-sdk google-chrome-stable firefox powershell mono-devel libgl1-mesa-dri
sudo apt autoremove -y
sudo apt clean
sudo rm -rf /usr/local/lib/android
sudo rm -rf /usr/share/dotnet
df -h
- name: Run Unit Tests
env:
RUST_BACKTRACE: full
RUSTC_WRAPPER: sccache
SCCACHE_CACHE_SIZE: 240
SCCACHE_DIR: /home/runner/.cache/sccache
run: |
source ${HOME}/.cargo/env
RUSTC_BOOTSTRAP=1 cargo test --release --features=runtime-benchmarks,try-runtime --workspace --exclude integration-tests
- name: stop sccache server
run: sccache --stop-server || true
stop-unit-test-checks:
needs: start-unit-test-checks
runs-on: ubuntu-20.04
if: ${{ always() }}
steps:
- continue-on-error: true
uses: audacious-network/[email protected]
with:
mode: stop
github-token: ${{ secrets.SELF_HOSTED_RUNNER_TOKEN }}
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ needs.start-unit-test-checks.outputs.aws-region }}
runner-label: ${{ needs.start-unit-test-checks.outputs.runner-label }}
aws-instance-id: ${{ needs.start-unit-test-checks.outputs.aws-instance-id }}
- name: discard stopper success/failure
run: true
start-congestion-test-checks:
timeout-minutes: 480
strategy:
matrix:
runtime:
- name: manta-runtime
- name: calamari-runtime
runs-on: ubuntu-20.04
outputs:
runner-label: ${{ steps.start-self-hosted-runner.outputs.runner-label }}
aws-region: ${{ steps.start-self-hosted-runner.outputs.aws-region }}
aws-instance-id: ${{ steps.start-self-hosted-runner.outputs.aws-instance-id }}
steps:
- id: start-self-hosted-runner
uses: audacious-network/[email protected]
with:
mode: start
github-token: ${{ secrets.SELF_HOSTED_RUNNER_TOKEN }}
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ env.AWS_REGION }}
aws-instance-type: ${{ env.AWS_INSTANCE_TYPE }}
aws-instance-root-volume-size: ${{ env.AWS_INSTANCE_ROOT_VOLUME_SIZE }}
aws-image-search-pattern: ${{ env.AWS_IMAGE_SEARCH_PATTERN }}
aws-image-search-owners: ${{ env.AWS_IMAGE_SEARCH_OWNERS }}
- uses: actions/checkout@v2
- name: init
run: |
sudo apt update
sudo apt install -y pkg-config libssl-dev protobuf-compiler
protoc --version
curl -s https://sh.rustup.rs -sSf | sh -s -- -y
source ${HOME}/.cargo/env
rustup update
rustup toolchain install nightly-2023-03-03
rustup default nightly-2023-03-03
rustup target add wasm32-unknown-unknown
- name: Run congestion test (allowed to fail)
id: congestion_test
run: |
source ${HOME}/.cargo/env
RUSTC_BOOTSTRAP=1 cargo test --package ${{ matrix.runtime.name }} --lib -- fee::multiplier_tests::multiplier_growth_simulator_and_congestion_budget_test --exact --nocapture --ignored
continue-on-error: true
- name: Comment on PR whether congestion test failed
uses: actions/github-script@v6
with:
script: |
// this uses octokit Issues API from https://octokit.github.io/rest.js/v19#issues
const octokit = github;
const number = context.issue.number;
const owner = context.repo.owner;
const repo = context.repo.repo;
if (!number) return; // Ignore non-PR CI runs
const comment_text = "Congestion test: 1-day congestion cost (${{ matrix.runtime.name }})"
// find or create a comment in the PR
const { data } = await octokit.rest.issues.listComments({ owner, repo, issue_number: number });
let existingComment = data.find((comment) => comment.body.includes(comment_text));
if (!existingComment) {
const { data: newComment } = await octokit.rest.issues.createComment({ owner, repo, issue_number: number, body: comment_text });
existingComment = newComment;
}
// Update comment text with CI status
const status = '${{ job.steps.congestion_test.outcome }}';
const statusIcon = status === 'success' ? ':white_check_mark:' : ':warning:';
const above = status === 'success' ? 'is above' : 'is NOT above';
const updatedComment = `${statusIcon} ${comment_text} ${above} target_daily_congestion_cost_kma`;
await octokit.rest.issues.updateComment({ owner, repo, comment_id: existingComment.id, body: updatedComment });
stop-congestion-test-checks:
timeout-minutes: 15
needs: start-congestion-test-checks
runs-on: ubuntu-20.04
if: ${{ always() }}
steps:
- continue-on-error: true
uses: audacious-network/[email protected]
with:
mode: stop
github-token: ${{ secrets.SELF_HOSTED_RUNNER_TOKEN }}
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ needs.start-congestion-test-checks.outputs.aws-region }}
runner-label: ${{ needs.start-congestion-test-checks.outputs.runner-label }}
aws-instance-id: ${{ needs.start-congestion-test-checks.outputs.aws-instance-id }}
- name: discard stopper success/failure
run: true