Reduce CI flakiness by using cargo-nextest (#3544)

* Try to reduce CI flakiness * Use cargo-nextest for all other CI jobs * Use cargo-nextest for multi-chains test and unit tests * Revert using action to install nextest * Increase timeout for multi-chains test * Use --failure-output to output logs only on failure * Use --failure-output in multi-chains CI * Add name to multi-chains integration test step --------- Co-authored-by: Romain Ruetschi <[email protected]>
informalsystems · Aug 18, 2023 · 92e0ac0 · 92e0ac0
1 parent 6113050
commit 92e0ac0
Show file tree

Hide file tree

Showing 3 changed files with 78 additions and 50 deletions.
diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml
@@ -99,20 +99,19 @@ jobs:
         with:
           command: test
           args: -p ibc-integration-test --no-fail-fast --no-run
-      - uses: nick-fields/retry@v2
+      - name: Install cargo-nextest
+        run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin
+      - name: Run integration test
         env:
           RUST_LOG: info
           RUST_BACKTRACE: 1
           NO_COLOR_LOG: 1
+          NEXTEST_RETRIES: 2
           CHAIN_COMMAND_PATHS: ${{ matrix.chain.command }}
           ACCOUNT_PREFIXES: ${{ matrix.chain.account_prefix }}
-        with:
-           max_attempts: 2
-           timeout_minutes: 90
-           command: |
-             nix shell .#python .#${{ matrix.chain.package }} -c cargo \
-               test -p ibc-integration-test --no-fail-fast -- \
-               --nocapture --test-threads=2
+        run: |
+            nix shell .#python .#${{ matrix.chain.package }} -c \
+              cargo nextest run -p ibc-integration-test --no-fail-fast --failure-output final --test-threads=2
 
   ordered-channel-test:
     runs-on: ubuntu-20.04
@@ -137,14 +136,17 @@ jobs:
         with:
           command: test
           args: -p ibc-integration-test --no-fail-fast --no-run
+      - name: Install cargo-nextest
+        run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin
       - env:
           RUST_LOG: info
           RUST_BACKTRACE: 1
           NO_COLOR_LOG: 1
+          NEXTEST_RETRIES: 2
         run: |
-          nix shell .#python .#gaia6-ordered -c cargo \
-            test -p ibc-integration-test --features ordered --no-fail-fast -- \
-            --nocapture --test-threads=1 test_ordered_channel
+          nix shell .#python .#gaia6-ordered -c \
+            cargo nextest run -p ibc-integration-test --no-fail-fast --failure-output final --test-threads=2 \
+            --features ordered test_ordered_channel
 
   ica-filter-test:
     runs-on: ubuntu-20.04
@@ -178,16 +180,19 @@ jobs:
         with:
           command: test
           args: -p ibc-integration-test --no-fail-fast --no-run
+      - name: Install cargo-nextest
+        run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin
       - env:
           RUST_LOG: info
           RUST_BACKTRACE: 1
           NO_COLOR_LOG: 1
+          NEXTEST_RETRIES: 2
           CHAIN_COMMAND_PATHS: ${{ matrix.chain.command }}
           ACCOUNT_PREFIXES: ${{ matrix.chain.account_prefix }}
         run: |
-          nix shell .#${{ matrix.chain.package }} -c cargo \
-            test -p ibc-integration-test --features ica --no-fail-fast -- \
-            --nocapture --test-threads=1 test_ica_filter
+          nix shell .#${{ matrix.chain.package }} -c \
+            cargo nextest run -p ibc-integration-test --no-fail-fast --failure-output final --test-threads=2 \
+            --features ica test_ica_filter
 
   ics29-fee-test:
     runs-on: ubuntu-20.04
@@ -227,16 +232,19 @@ jobs:
         with:
           command: test
           args: -p ibc-integration-test --features ics29-fee --no-fail-fast --no-run
+      - name: Install cargo-nextest
+        run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin
       - env:
           RUST_LOG: info
           RUST_BACKTRACE: 1
           NO_COLOR_LOG: 1
+          NEXTEST_RETRIES: 2
           CHAIN_COMMAND_PATHS: ${{ matrix.chain.command }}
           ACCOUNT_PREFIXES: ${{ matrix.chain.account_prefix }}
         run: |
-          nix shell .#${{ matrix.chain.package }} -c cargo \
-            test -p ibc-integration-test --features ics29-fee --no-fail-fast -- \
-            --nocapture --test-threads=1 fee::
+          nix shell .#${{ matrix.chain.package }} -c \
+            cargo nextest run -p ibc-integration-test --no-fail-fast --failure-output final --test-threads=2 \
+            --features ics29-fee fee::
 
   forward-packet:
     runs-on: ubuntu-20.04
@@ -273,16 +281,19 @@ jobs:
         with:
           command: test
           args: -p ibc-integration-test --features forward-packet --no-fail-fast --no-run
+      - name: Install cargo-nextest
+        run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin
       - env:
           RUST_LOG: info
           RUST_BACKTRACE: 1
           NO_COLOR_LOG: 1
+          NEXTEST_RETRIES: 2
           CHAIN_COMMAND_PATHS: ${{ matrix.chain.command }}
           ACCOUNT_PREFIXES: ${{ matrix.chain.account_prefix }}
         run: |
-          nix shell .#${{ matrix.chain.package }} -c cargo \
-            test -p ibc-integration-test --features forward-packet --no-fail-fast -- \
-            --nocapture --test-threads=1 forward::
+          nix shell .#${{ matrix.chain.package }} -c \
+            cargo nextest run -p ibc-integration-test --no-fail-fast --failure-output final --test-threads=2 \
+            --features forward-packet forward::
 
   ics31:
     runs-on: ubuntu-20.04
@@ -313,16 +324,19 @@ jobs:
         with:
           command: test
           args: -p ibc-integration-test --features ics31 --no-fail-fast --no-run
+      - name: Install cargo-nextest
+        run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin
       - env:
           RUST_LOG: info
           RUST_BACKTRACE: 1
           NO_COLOR_LOG: 1
+          NEXTEST_RETRIES: 2
           CHAIN_COMMAND_PATHS: ${{ matrix.chain.command }}
           ACCOUNT_PREFIXES: ${{ matrix.chain.account_prefix }}
         run: |
-          nix shell ${{ matrix.chain.package }} -c cargo \
-            test -p ibc-integration-test --features ics31 --no-fail-fast -- \
-            --nocapture --test-threads=1 ics31::
+          nix shell ${{ matrix.chain.package }} -c \
+            cargo nextest run -p ibc-integration-test --no-fail-fast --failure-output final --test-threads=2 \
+            --features ics31 ics31::
   fee-grant:
     runs-on: ubuntu-20.04
     strategy:
@@ -355,16 +369,18 @@ jobs:
         with:
           command: test
           args: -p ibc-integration-test --features fee-grant --no-fail-fast --no-run
+      - name: Install cargo-nextest
+        run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin
       - env:
           RUST_LOG: info
           RUST_BACKTRACE: 1
           NO_COLOR_LOG: 1
           CHAIN_COMMAND_PATHS: ${{ matrix.chain.command }}
           ACCOUNT_PREFIXES: ${{ matrix.chain.account_prefix }}
         run: |
-          nix shell .#${{ matrix.chain.package }} -c cargo \
-            test -p ibc-integration-test --features fee-grant --no-fail-fast -- \
-            --nocapture --test-threads=1 fee_grant::
+          nix shell .#${{ matrix.chain.package }} -c \
+            cargo nextest run -p ibc-integration-test --no-fail-fast --failure-output final --test-threads=2 \
+            --features fee-grant fee_grant::
 
   clean-workers:
     runs-on: ubuntu-20.04
@@ -395,16 +411,19 @@ jobs:
         with:
           command: test
           args: -p ibc-integration-test --features ics31 --no-fail-fast --no-run
+      - name: Install cargo-nextest
+        run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin
       - env:
           RUST_LOG: info
           RUST_BACKTRACE: 1
           NO_COLOR_LOG: 1
+          NEXTEST_RETRIES: 2
           CHAIN_COMMAND_PATHS: ${{ matrix.chain.command }}
           ACCOUNT_PREFIXES: ${{ matrix.chain.account_prefix }}
         run: |
-          nix shell .#${{ matrix.chain.package }} -c cargo \
-            test -p ibc-integration-test --features clean-workers --no-fail-fast -- \
-            --nocapture --test-threads=1 clean_workers::
+          nix shell .#${{ matrix.chain.package }} -c \
+            cargo nextest run -p ibc-integration-test --no-fail-fast --failure-output final --test-threads=2 \
+            --features clean-workers clean_workers::
 
   interchain-security-no-ica:
     runs-on: ubuntu-20.04
@@ -435,16 +454,19 @@ jobs:
         with:
           command: test
           args: -p ibc-integration-test --features interchain-security --no-fail-fast --no-run
+      - name: Install cargo-nextest
+        run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin
       - env:
           RUST_LOG: info
           RUST_BACKTRACE: 1
           NO_COLOR_LOG: 1
+          NEXTEST_RETRIES: 2
           CHAIN_COMMAND_PATHS: gaiad,${{ matrix.chain.command }}
           ACCOUNT_PREFIXES: cosmos,${{ matrix.chain.account_prefix }}
         run: |
-          nix shell .#gaia9 .#${{ matrix.chain.package }} -c cargo \
-            test -p ibc-integration-test --features interchain-security --no-fail-fast -- \
-            --nocapture --test-threads=1 interchain_security::
+          nix shell .#gaia9 .#${{ matrix.chain.package }} -c \
+            cargo nextest run -p ibc-integration-test --no-fail-fast --failure-output final --test-threads=2 \
+            --features interchain-security interchain_security::
 
   interchain-security-ica:
     runs-on: ubuntu-20.04
@@ -475,16 +497,19 @@ jobs:
         with:
           command: test
           args: -p ibc-integration-test --features interchain-security --no-fail-fast --no-run
+      - name: Install cargo-nextest
+        run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin
       - env:
           RUST_LOG: info
           RUST_BACKTRACE: 1
           NO_COLOR_LOG: 1
+          NEXTEST_RETRIES: 2
           CHAIN_COMMAND_PATHS: gaiad,${{ matrix.chain.command }}
           ACCOUNT_PREFIXES: cosmos,${{ matrix.chain.account_prefix }}
         run: |
-          nix shell .#gaia9 .#${{ matrix.chain.package }} -c cargo \
-            test -p ibc-integration-test --features interchain-security,ica --no-fail-fast -- \
-            --nocapture --test-threads=1 interchain_security::
+          nix shell .#gaia9 .#${{ matrix.chain.package }} -c \
+            cargo nextest run -p ibc-integration-test --no-fail-fast --failure-output final --test-threads=2 \
+            --features interchain-security,ica interchain_security::
 
   interchain-security-icq:
     runs-on: ubuntu-20.04
@@ -515,16 +540,18 @@ jobs:
         with:
           command: test
           args: -p ibc-integration-test --features interchain-security --no-fail-fast --no-run
+      - name: Install cargo-nextest
+        run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin
       - env:
           RUST_LOG: info
           RUST_BACKTRACE: 1
           NO_COLOR_LOG: 1
           CHAIN_COMMAND_PATHS: gaiad,${{ matrix.chain.command }}
           ACCOUNT_PREFIXES: cosmos,${{ matrix.chain.account_prefix }}
         run: |
-          nix shell .#gaia9 .#${{ matrix.chain.package }} -c cargo \
-            test -p ibc-integration-test --features interchain-security,ics31 --no-fail-fast -- \
-            --nocapture --test-threads=1 interchain_security::
+          nix shell .#gaia9 .#${{ matrix.chain.package }} -c \
+            cargo nextest run -p ibc-integration-test --no-fail-fast --failure-output final --test-threads=2 \
+            --features interchain-security,ics31 interchain_security::
 
   model-based-test:
     runs-on: ubuntu-20.04
@@ -564,4 +591,4 @@ jobs:
       #       .#apalache \
       #       -c cargo \
       #       test -p ibc-integration-test --features mbt --no-fail-fast -- \
-      #       --nocapture --test-threads=1 mbt
+      #       --failure-output final --test-threads=2 --test-threads=1 mbt
diff --git a/.github/workflows/multi-chains.yaml b/.github/workflows/multi-chains.yaml
@@ -53,7 +53,7 @@ jobs:
       ) || (
         github.event.action == 'labeled' && github.event.label.name == 'CI: multi-chains'
       )
-    timeout-minutes: 60
+    timeout-minutes: 120
     strategy:
       fail-fast: false
       matrix:
@@ -101,17 +101,16 @@ jobs:
         with:
           command: test
           args: -p ibc-integration-test --no-fail-fast --no-run
-      - uses: nick-fields/retry@v2
+      - name: Install cargo-nextest
+        run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin
+      - name: Run multi-chains integration tests
         env:
           RUST_LOG: info
           RUST_BACKTRACE: 1
           NO_COLOR_LOG: 1
+          NEXTEST_RETRIES: 2
           ACCOUNT_PREFIXES: ${{ matrix.first-package.account_prefix }},${{ matrix.second-package.account_prefix }}
-        with:
-           max_attempts: 2
-           timeout_minutes: 60
-           command: |
+        run: |
              CHAIN_COMMAND_PATHS=$(nix shell .#${{ matrix.first-package.package }} -c which ${{ matrix.first-package.command }}),$(nix shell .#${{ matrix.second-package.package }} -c which ${{ matrix.second-package.command }}) \
-              nix shell .#python -c cargo \
-              test -p ibc-integration-test --no-fail-fast -- \
-              --nocapture --test-threads=2
+               nix shell .#python -c \
+                 cargo nextest run -p ibc-integration-test --no-fail-fast --failure-output final --test-threads=2
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -89,14 +89,16 @@ jobs:
           toolchain: stable
           override: true
       - uses: Swatinem/rust-cache@v2
+      - name: Install cargo-nextest
+        run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin
       - uses: actions-rs/cargo@v1
         with:
           command: test
           args: --all-features --no-fail-fast --no-run
       - uses: actions-rs/cargo@v1
         with:
-          command: test
-          args: --all-features --no-fail-fast --workspace --exclude ibc-integration-test -- --nocapture
+          command: nextest
+          args: run --all-features --no-fail-fast --workspace --exclude ibc-integration-test --no-capture
 
   # test-coverage:
   #   runs-on: ubuntu-latest