Devnet Scenarios #13858
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Devnet Scenarios | |
| on: | |
| schedule: | |
| - cron: '0 */2 * * *' | |
| workflow_dispatch: | |
| jobs: | |
| Run: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test: | |
| - FourValidatorsReconnect | |
| - MultipleValidatorsDown | |
| - FourValidatorsReconnectRmDatabase | |
| - FourValidatorsReconnectSpammer | |
| - MacroBlockProduction | |
| - Validators90sDown | |
| - LowBlockProducerTimeout | |
| include: | |
| - test: FourValidatorsReconnect | |
| devnet_args: -t .github/devnet_topologies/four_validators.toml -dt 30 | |
| - test: MultipleValidatorsDown | |
| devnet_args: -t .github/devnet_topologies/four_validators.toml -k 2 -ut 100 -dt 30 -as | |
| - test: FourValidatorsReconnectRmDatabase | |
| devnet_args: -t .github/devnet_topologies/four_validators.toml -d -ut 100 -dt 30 | |
| - test: FourValidatorsReconnectSpammer | |
| devnet_args: -t .github/devnet_topologies/four_validators_spammer_1.toml -ut 250 -dt 30 | |
| - test: MacroBlockProduction | |
| # The number of blocks per epoch needs to stay the same (because of the ZKP keys) | |
| pre: "grep 'blocks_per_batch: 60,' primitives/src/policy.rs && | |
| grep 'batches_per_epoch: 720,' primitives/src/policy.rs && | |
| sed -i 's/blocks_per_batch: 60,/blocks_per_batch: 60 \\/ 30,/g' primitives/src/policy.rs && | |
| sed -i 's/batches_per_epoch: 720,/batches_per_epoch: 720 \\* 30,/g' primitives/src/policy.rs" | |
| devnet_args: -t .github/devnet_topologies/four_validators_spammer_1.toml -k 0 -ut 100 | |
| - test: Validators90sDown | |
| devnet_args: -t .github/devnet_topologies/four_validators_spammer_1.toml -dt 90 -ut 100 | |
| - test: LowBlockProducerTimeout | |
| pre: "grep 'MIN_PRODUCER_TIMEOUT: u64 = 4 \\* 1000;' primitives/src/policy.rs && | |
| sed -i 's/MIN_PRODUCER_TIMEOUT: u64 = 4 \\* 1000;/MIN_PRODUCER_TIMEOUT: u64 = 1000;/g' primitives/src/policy.rs" | |
| devnet_args: -t .github/devnet_topologies/four_validators_spammer_1.toml -k 0 -ut 100 | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Free Disk Space (Ubuntu) | |
| uses: jlumbroso/free-disk-space@main | |
| with: | |
| tool-cache: true | |
| large-packages: false | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.8' | |
| - name: Install python dependencies | |
| run: pip install scripts/devnet | |
| - name: Set up Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Optionally patch the source | |
| run: ${{ matrix.pre }} | |
| - name: Build the code | |
| run: cargo build | |
| - name: Retrieve initial timestamp | |
| id: initial_ts | |
| run: | | |
| echo "INITIAL_TS=$(date +%s%N | cut -b1-13)" >> $GITHUB_OUTPUT | |
| - name: Execute the test | |
| run: | | |
| python3 scripts/devnet/devnet.py --run-environment ci ${{ matrix.devnet_args }} | |
| env: | |
| NIMIQ_LOKI_URL: ${{ secrets.LOKI_URL }} | |
| NIMIQ_LOKI_EXTRA_FIELDS: nimiq_ci_name=${{ github.workflow }}:nimiq_ci_run=${{ github.run_number }}:nimiq_test=${{ matrix.test }} | |
| - name: Redact Loki URL | |
| if: always() | |
| run: sed -i 's/url = "[^"]*"/url = "<redacted>"/g' temp-logs/*/conf/*/*.toml | |
| - name: Archive test results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.test }}-logs | |
| path: | | |
| temp-logs/ | |
| - name: Retrieve failure reason | |
| if: always() | |
| run: | | |
| if [ -f temp-state/*/RESULT.TXT ]; then | |
| echo "FAIL_REASON=$(cat temp-state/*/RESULT.TXT)" >> $GITHUB_OUTPUT | |
| else | |
| echo "FAIL_REASON=other" >> $GITHUB_OUTPUT | |
| fi | |
| id: reason | |
| - name: Retrieve final timestamp and Run ID | |
| if: always() | |
| id: final_ts_run_id | |
| run: | | |
| echo "FINAL_TS=$(date +%s%N | cut -b1-13)" >> $GITHUB_OUTPUT | |
| echo "RUN_ID=$(cat temp-logs/*/conf/run_id.info)" >> $GITHUB_OUTPUT | |
| - name: Build Grafana dashboard link | |
| if: always() | |
| id: grafana_url | |
| run: | | |
| echo "GRAFANA_URL=http://localhost:3001/d/YbjdvlU7z/nimiq-test?orgId=1&var-env=ci&var-run_id=${{steps.final_ts_run_id.outputs.RUN_ID}}&from=${{steps.initial_ts.outputs.INITIAL_TS}}&to=${{steps.final_ts_run_id.outputs.FINAL_TS}}" >> $GITHUB_OUTPUT | |
| - name: Report potential deadlocks to slack | |
| if: always() && contains(steps.reason.outputs.FAIL_REASON, 'DEADLOCK') | |
| uses: ravsamhq/notify-slack-action@v2 | |
| with: | |
| status: ${{ job.status }} | |
| notification_title: 'Potential deadlock detected in ${{ matrix.test }}' | |
| footer: '<{run_url}|View Run>' | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }} | |
| # - name: Report long lock hold times | |
| # if: always() && contains(steps.reason.outputs.FAIL_REASON, 'LONG_LOCK_HOLD_TIME') | |
| # uses: ravsamhq/notify-slack-action@v2 | |
| # with: | |
| # status: ${{ job.status }} | |
| # notification_title: 'Long lock hold time in ${{ matrix.test }}' | |
| # footer: '<{run_url}|View Run>' | |
| # env: | |
| # SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }} | |
| # - name: Report slow lock acquisitions | |
| # if: always() && contains(steps.reason.outputs.FAIL_REASON, 'SLOW_LOCK_ACQUISITION') | |
| # uses: ravsamhq/notify-slack-action@v2 | |
| # with: | |
| # status: ${{ job.status }} | |
| # notification_title: 'Slow lock acquisition detected in ${{ matrix.test }}' | |
| # footer: '<{run_url}|View Run>' | |
| # env: | |
| # SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }} | |
| - name: Report Status to Slack | |
| if: always() && github.ref == 'refs/heads/albatross' | |
| uses: ravsamhq/notify-slack-action@v2 | |
| with: | |
| status: ${{ job.status }} | |
| notify_when: 'failure' | |
| notification_title: '${{ matrix.test }} failed because of ${{ steps.reason.outputs.FAIL_REASON }}' | |
| footer: '<{run_url}|View Run> | <${{steps.grafana_url.outputs.GRAFANA_URL}}|Grafana dashboard>' | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }} |