nightly_wan.yml

name: Nightly -- Full WAN Network Tests

on:
  schedule:
    - cron: "0 0 * * *"
# enable as below for testing purpose.
  # pull_request:
  #   branches: ["*"]
  workflow_dispatch:

env:
  CARGO_INCREMENTAL: 0 # bookkeeping for incremental builds has overhead, not useful in CI.
  NETWORK_NAME: DEV-01
  WORKFLOW_URL: https://github.com/maidsafe/stableset_net/actions/runs
jobs:
  e2e:
    name: E2E tests
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest]
    steps:
      - uses: actions/checkout@v4

      - name: Install Rust
        uses: dtolnay/rust-toolchain@stable

      - uses: Swatinem/rust-cache@v2
        continue-on-error: true

      - name: Build safe
        run: cargo build --release --bin safe
        timeout-minutes: 30

      - name: setup testnet-deploy
        uses: maidsafe/sn-testnet-control-action/init-testnet-deploy@main
        with:
          ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }}
          aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }}
          aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }}
          aws-region: eu-west-2
          do-token: ${{ secrets.SN_TESTNET_DO_PAT }}
          ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }}

      - name: launch ${{ env.NETWORK_NAME }}
        uses: maidsafe/sn-testnet-control-action/launch-network@main
        with:
          ansible-forks: ${{ env.ANSIBLE_FORKS }}
          environment-type: development
          node-vm-count: 10
          node-count: 20
          uploader-vm-count: 0
          bootstrap-node-vm-count: 0
          log-format: json
          network-name: ${{ env.NETWORK_NAME }}
          provider: digital-ocean
          safe-network-branch: main
          safe-network-user: maidsafe

      - name: Check env variables
        shell: bash
        run: |
           echo "Peer is $SAFE_PEERS"
           echo "Deployment inventory is $SN_INVENTORY"

      - name: start faucet
        uses: maidsafe/sn-testnet-control-action/start-faucet@main
        with:
            network-name: ${{ env.NETWORK_NAME }}
               
      - name: Obtain the funds from the faucet
        run: |
          set -e

          # read the inventory file
          inventory_path=/home/runner/.local/share/safe/testnet-deploy/${{ env.NETWORK_NAME }}-inventory.json
          echo "Inventory Path: $inventory_path"
          faucet_address=$(jq -r '.faucet_address' $inventory_path)
          cargo run --bin safe --release -- wallet get-faucet ${faucet_address}
        env:
          SN_LOG: "all"
        timeout-minutes: 2
      
      - name: Start a client to carry out chunk actions
        run: |
          set -e
          cargo run --bin safe --release -- --log-output-dest=data-dir files upload "./resources" --retry-strategy quick
        env:
          SN_LOG: "all"
        timeout-minutes: 2
      
      - name: Start a client to create a register
        run: |
          set -e
          cargo run --bin safe --release -- --log-output-dest=data-dir register create -n baobao
        env:
          SN_LOG: "all"
        timeout-minutes: 2
      
      - name: Start a client to get a register
        run: |
          set -e
          cargo run --bin safe --release -- --log-output-dest=data-dir register get -n baobao
        env:
          SN_LOG: "all"
        timeout-minutes: 2
      
      - name: Start a client to edit a register
        run: |
          set -e
          cargo run --bin safe --release -- --log-output-dest=data-dir register edit -n baobao wood
        env:
          SN_LOG: "all"
        timeout-minutes: 2
      
      # - name: Fetch network logs
      #   uses: maidsafe/sn-testnet-control-action/fetch-logs@main
      #   with:
      #     re-attempts: 3
      #     rust-log: debug
      #     provider: digital-ocean
      #     network-name: ${{ env.NETWORK_NAME }}
    
      - name: Upload local logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: local_logs_NightlyE2E
          path: |
            ~/.local/share/safe/node/*/logs/*.log*
            ~/.local/share/safe/*/*/*.log*
            ~/.local/share/safe/client/logs/*/*.log*

      - name: destroy network
        if: always()
        uses: maidsafe/sn-testnet-control-action/destroy-network@main
        with:
          network-name: ${{ env.NETWORK_NAME }}
          provider: digital-ocean

      - name: post notification to slack on failure
        if: ${{ failure() }}
        uses: bryannice/gitactions-slack-notification@2.0.0
        env:
          SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }}
          SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}"
          SLACK_TITLE: "Nightly E2E Test Run Failed"

  # spend_test:
  #   name: Spend tests against network
  #   runs-on: ${{ matrix.os }}
  #   strategy:
  #     matrix:
  #       os: [ubuntu-latest]
  #   steps:
  #     - uses: actions/checkout@v4
  
  #     - name: Install Rust
  #       uses: dtolnay/rust-toolchain@stable
  
  #     - uses: Swatinem/rust-cache@v2
  #       continue-on-error: true
  
  #     - name: Build testing executable
  #       run: cargo test --release -p sn_node --features=local-discovery --test sequential_transfers --test storage_payments --test double_spend --test spend_simulation --no-run
  #       timeout-minutes: 40
  
  #     - name: setup testnet-deploy
  #       uses: maidsafe/sn-testnet-control-action/init-testnet-deploy@main
  #       with:
  #         ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }}
  #         aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }}
  #         aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }}
  #         aws-region: eu-west-2
  #         do-token: ${{ secrets.SN_TESTNET_DO_PAT }}
  #         ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }}

  #     - name: launch ${{ env.NETWORK_NAME }}
  #       uses: maidsafe/sn-testnet-control-action/launch-network@main
  #       with:
  #         ansible-forks: ${{ env.ANSIBLE_FORKS }}
  #         beta-encryption-key: ${{ env.DEFAULT_PAYMENT_FORWARD_SK }}
  #         environment-type: development
  #         faucet-version: ${{ env.FAUCET_VERSION }}
  #         log-format: json
  #         network-name: ${{ env.NETWORK_NAME }}
  #         network-contacts-file-name: ${{ env.NETWORK_CONTACTS_FILE_NAME }}
  #         provider: digital-ocean
  #         safe-network-branch: main
  #         safe-network-user: maidsafe
  
  #     - name: Check env variables
  #       shell: bash
  #       run: |
  #         echo "Peer is $SAFE_PEERS"
  #         echo "Deployment inventory is $SN_INVENTORY"
  
  #     - name: execute the sequential transfers test
  #       run: cargo test --release -p sn_node --test sequential_transfers -- --nocapture --test-threads=1
  #       env:
  #         SN_LOG: "all"
  #       timeout-minutes: 45
  
  #     - name: execute the storage payment tests
  #       run: cargo test --release -p sn_node --test storage_payments -- --nocapture --test-threads=1
  #       env:
  #         SN_LOG: "all"
  #       timeout-minutes: 45
  
  #     - name: execute the double spend tests
  #       run: cargo test --release -p sn_node --test double_spend -- --nocapture --test-threads=1
  #       timeout-minutes: 45
  
  
  #     - name: execute the spend simulation tests
  #       run: cargo test --release -p sn_node --test spend_simulation -- --nocapture --test-threads=1
  #       timeout-minutes: 45
  
  #     - name: Small wait to allow reward receipt
  #       run: sleep 30
  #       timeout-minutes: 1
  
  #     - name: Fetch network logs
  #       uses: ermineJose/sn-testnet-control-action/fetch-logs@feat-add_fetch-logs-action
  #       with:
  #         re-attempts: 3
  #         rust-log: debug
  #         provider: digital-ocean
  #         testnet-name: ${{ env.NETWORK_NAME }}

  #     - name: Upload local logs
  #       if: always()
  #       uses: actions/upload-artifact@v4
  #       with:
  #         name: local_logs_NightlySpendTest
  #         path: |
  #           ~/.local/share/safe/node/*/logs/*.log*
  #           ~/.local/share/safe/*/*/*.log*
  #           ~/.local/share/safe/client/logs/*/*.log*
 
  #     - name: destroy network
  #       uses: maidsafe/sn-testnet-control-action/destroy-network@main
  #       with:
  #         network-name: ${{ env.NETWORK_NAME }}
  #         provider: digital-ocean
  
  #     - name: post notification to slack on failure
  #       if: ${{ failure() }}
  #       uses: bryannice/gitactions-slack-notification@2.0.0
  #       env:
  #         SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }}
  #         SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}"
  #         SLACK_TITLE: "Nightly Spend Test Run Failed"
  
  # churn:
  #   name: Network churning tests
  #   runs-on: ${{ matrix.os }}
  #   strategy:
  #     matrix:
  #       include:
  #         - os: ubuntu-latest
  #           wan_logs_path: /home/runner/sn-testnet-deploy/logs
  #           local_safe_path: /home/runner/.local/share/safe
  #         # - os: windows-latest
  #         #   node_data_path: C:\\Users\\runneradmin\\AppData\\Roaming\\safe\\node
  #         #   safe_path: C:\\Users\\runneradmin\\AppData\\Roaming\\safe
  #         # - os: macos-latest
  #         #   node_data_path: /Users/runner/Library/Application Support/safe/node
  #         #   safe_path: /Users/runner/Library/Application Support/safe
  #   steps:
  #     - uses: actions/checkout@v4
  #
  #     - name: Install Rust
  #       uses: dtolnay/rust-toolchain@stable
  #
  #     - name: install ripgrep
  #       run: sudo apt-get -y install ripgrep
  #
  #     - uses: Swatinem/rust-cache@v2
  #       continue-on-error: true
  #
  #     - name: Build churn tests
  #       run: cargo test --release -p sn_node --test data_with_churn --no-run
  #       timeout-minutes: 30
  #
  #     - name: Start a WAN network
  #       uses: maidsafe/sn-testnet-action@main
  #       with:
  #         action: create
  #         re-attempts: 3
  #         rust-log: debug
  #         ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }}
  #         aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }}
  #         aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }}
  #         aws-region: eu-west-2
  #         do-token: ${{ secrets.SN_TESTNET_DO_PAT }}
  #         ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }}
  #         security-group-id: sg-0d47df5b3f0d01e2a
  #         subnet-id: subnet-018f2ab26755df7f9
  #         node-count: 20
  #         vm-count: 1
  #         provider: digital-ocean
  #         testnet-name: NightlyChurnTest
  #         safe-network-user: maidsafe
  #         safe-network-branch: main
  #
  #     - name: Check env variables
  #       shell: bash
  #       run: |
  #         echo "Peer is $SAFE_PEERS"
  #         echo "Deployment inventory is $SN_INVENTORY"
  #
  #     - name: Chunks data integrity during nodes churn
  #       run: cargo test --release -p sn_node --test data_with_churn -- --nocapture
  #       env:
  #         # TEST_DURATION_MINS: 60
  #         # TEST_CHURN_CYCLES: 6
  #         # SN_LOG: "all"
  #         # todo: lower time for testing
  #         TEST_DURATION_MINS: 10
  #         TEST_CHURN_CYCLES: 2
  #         SN_LOG: "all"
  #       timeout-minutes: 90
  #
  #     - name: Fetch network logs
  #       if: always()
  #       uses: maidsafe/sn-testnet-action@main
  #       with:
  #         action: logs
  #         re-attempts: 3
  #         rust-log: debug
  #         ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }}
  #         aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }}
  #         aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }}
  #         aws-region: eu-west-2
  #         do-token: ${{ secrets.SN_TESTNET_DO_PAT }}
  #         ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }}
  #         node-count: 20
  #         vm-count: 1
  #         provider: digital-ocean
  #         testnet-name: NightlyChurnTest
  #         safe-network-user: maidsafe
  #         safe-network-branch: main
  #
  #     - name: Upload local logs
  #       if: always()
  #       uses: actions/upload-artifact@v4
  #       with:
  #         name: local_logs_NightlyChurnTest
  #         path: |
  #           ~/.local/share/safe/node/*/logs/*.log*
  #           ~/.local/share/safe/*/*/*.log*
  #           ~/.local/share/safe/client/logs/*/*.log*
  #
  #     - name: Stop the WAN network
  #       if: always()
  #       uses: maidsafe/sn-testnet-action@main
  #       with:
  #         action: destroy
  #         re-attempts: 3
  #         rust-log: debug
  #         ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }}
  #         aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }}
  #         aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }}
  #         aws-region: eu-west-2
  #         do-token: ${{ secrets.SN_TESTNET_DO_PAT }}
  #         ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }}
  #         node-count: 20
  #         vm-count: 1
  #         provider: digital-ocean
  #         testnet-name: NightlyChurnTest
  #         safe-network-user: maidsafe
  #         safe-network-branch: main
  #
  #     # TODO: re-enable the below scripts once we have proper way to restart nodes.
  #     # Currently on remote network (not local), the nodes do not handle restart RPC cmd well. They reuse the same
  #     # log location and the logs are over written. Hence the scripts might give false outputs.
  #
  #     # - name: Verify restart of nodes using rg
  #     #   shell: bash
  #     #   timeout-minutes: 1
  #     #   # get the counts, then the specific line, and then the digit count only
  #     #   # then check we have an expected level of restarts
  #     #   # TODO: make this use an env var, or relate to testnet size
  #     #   run : |
  #     #     restart_count=$(rg "Node is restarting in" "${{ matrix.wan_logs_path }}" -c --stats | \
  #     #       rg "(\d+) matches" | rg "\d+" -o)
  #     #     echo "Restart $restart_count nodes"
  #     #     peer_removed=$(rg "PeerRemovedFromRoutingTable" "${{ matrix.wan_logs_path }}" -c --stats | \
  #     #       rg "(\d+) matches" | rg "\d+" -o)
  #     #     echo "PeerRemovedFromRoutingTable $peer_removed times"
  #     #     if [ $peer_removed -lt $restart_count ]; then
  #     #       echo "PeerRemovedFromRoutingTable times of: $peer_removed is less than the restart count of: $restart_count"
  #     #       exit 1
  #     #     fi
  #     #     node_count=$(find "${{ matrix.wan_logs_path }}" -type d | awk -F/ 'NF==9' | grep -E "/12D3KooW" | wc -l)
  #     #     echo "Node dir count is $node_count"
  #     #   # TODO: reenable this once the testnet dir creation is tidied up to avoid a large count here
  #
  #     #   # if [ $restart_count -lt $node_count ]; then
  #     #   #   echo "Restart count of: $restart_count is less than the node count of: $node_count"
  #     #   #   exit 1
  #     #   # fi
  #
  #     # - name: Verify data replication using rg
  #     #   shell: bash
  #     #   timeout-minutes: 1
  #     #   # get the counts, then the specific line, and then the digit count only
  #     #   # then check we have an expected level of replication
  #     #   # TODO: make this use an env var, or relate to testnet size
  #     #   run : |
  #     #     fetching_attempt_count=$(rg "FetchingKeysForReplication" "${{ matrix.wan_logs_path }}" -c --stats | \
  #     #       rg "(\d+) matches" | rg "\d+" -o)
  #     #     echo "Carried out $fetching_attempt_count fetching attempts"
  #     #     node_count=$(find "${{ matrix.wan_logs_path }}" -type d | awk -F/ 'NF==9' | grep -E "/12D3KooW" | wc -l)
  #     #     if [ $fetching_attempt_count -lt $node_count ]; then
  #     #       echo "Replication fetching attempts of: $fetching_attempt_count is less than the node count of: $node_count"
  #     #       exit 1
  #     #     fi
  #
  #     # Only error out after uploading the logs
  #     - name: Don't log raw data
  #       if: always() && matrix.os != 'windows-latest' # causes error
  #       shell: bash
  #       timeout-minutes: 10
  #       run: |
  #         if ! rg '^' "${{ matrix.local_safe_path }}"/client/logs | awk 'length($0) > 15000 { print; exit 1 }'
  #         then
  #           echo "We are logging an extremely large data"
  #           exit 1
  #         fi
  #         #  node dir structure: ~/sn-testnet-deploy/logs/NightlyChurnTest/NightlyChurnTest-genesis/safenode1/safenode.log
  #         #faucet dir structure: ~/sn-testnet-deploy/logs/NightlyChurnTest/NightlyChurnTest-genesis/faucet/logs/faucet.log
  #         if ! rg '^' "${{ matrix.wan_logs_path }}"/*/*/*/ | awk 'length($0) > 15000 { print; exit 1 }'
  #         then
  #           echo "We are logging an extremely large data"
  #           exit 1
  #         fi
  #
  #         # sanity check
  #         if ! rg '^' "${{ matrix.local_safe_path }}"/client/logs | awk 'length($0) > 1000 { print; exit 1 }'
  #         then
  #           echo "Sanity check pass for local safe path"
  #         fi
  #         #  node dir structure: ~/sn-testnet-deploy/logs/NightlyChurnTest/NightlyChurnTest-genesis/safenode1/safenode.log
  #         #faucet dir structure: ~/sn-testnet-deploy/logs/NightlyChurnTest/NightlyChurnTest-genesis/faucet/logs/faucet.log
  #         if ! rg '^' "${{ matrix.wan_logs_path }}"/*/*/*/ | awk 'length($0) > 1000 { print; exit 1 }'
  #         then
  #           echo "Sanity check pass for wan logs path"
  #         fi
  #
  #     # - name: post notification to slack on failure
  #     #   if: ${{ failure() }}
  #     #   uses: bryannice/gitactions-slack-notification@2.0.0
  #     #   env:
  #     #     SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }}
  #     #     SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}"
  #     #     SLACK_TITLE: "Nightly Churn Test Run Failed"
  #
  # verify_data_location_routing_table:
  #   name: Verify data location and Routing Table
  #   runs-on: ${{ matrix.os }}
  #   strategy:
  #     matrix:
  #       include:
  #         - os: ubuntu-latest
  #           wan_logs_path: /home/runner/sn-testnet-deploy/logs
  #           local_safe_path: /home/runner/.local/share/safe
  #         # - os: windows-latest
  #         #   node_data_path: C:\\Users\\runneradmin\\AppData\\Roaming\\safe\\node
  #         #   safe_path: C:\\Users\\runneradmin\\AppData\\Roaming\\safe
  #         # - os: macos-latest
  #         #   node_data_path: /Users/runner/Library/Application Support/safe/node
  #         #   safe_path: /Users/runner/Library/Application Support/safe
  #   steps:
  #     - uses: actions/checkout@v4
  #
  #     - name: Install Rust
  #       uses: dtolnay/rust-toolchain@stable
  #
  #     - name: install ripgrep
  #       run: sudo apt-get -y install ripgrep
  #
  #     - uses: Swatinem/rust-cache@v2
  #       continue-on-error: true
  #
  #     - name: Build data location and routing table tests
  #       run: cargo test --release -p sn_node --test verify_data_location --test verify_routing_table --no-run
  #       timeout-minutes: 30
  #
  #     - name: Start a WAN network
  #       uses: maidsafe/sn-testnet-action@main
  #       with:
  #         action: create
  #         re-attempts: 3
  #         rust-log: debug
  #         ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }}
  #         aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }}
  #         aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }}
  #         aws-region: eu-west-2
  #         do-token: ${{ secrets.SN_TESTNET_DO_PAT }}
  #         ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }}
  #         security-group-id: sg-0d47df5b3f0d01e2a
  #         subnet-id: subnet-018f2ab26755df7f9
  #         node-count: 20
  #         vm-count: 1
  #         testnet-name: NightlyDataLocationTest
  #         safe-network-user: maidsafe
  #         safe-network-branch: main
  #
  #     - name: Check env variables
  #       shell: bash
  #       run: |
  #         echo "Peer is $SAFE_PEERS"
  #         echo "Deployment inventory is $SN_INVENTORY"
  #
  #     - name: Verify the Routing table of the nodes
  #       run: cargo test --release -p sn_node --test verify_routing_table -- --nocapture
  #       timeout-minutes: 5
  #
  #     - name: Verify the location of the data on the network
  #       run: cargo test --release -p sn_node --test verify_data_location -- --nocapture
  #       env:
  #         SN_LOG: "all"
  #       timeout-minutes: 90
  #
  #     - name: Verify the routing tables of the nodes
  #       run: cargo test --release -p sn_node --test verify_routing_table -- --nocapture
  #       timeout-minutes: 5
  #
  #     - name: Fetch network logs
  #       if: always()
  #       uses: maidsafe/sn-testnet-action@main
  #       with:
  #         action: logs
  #         re-attempts: 3
  #         rust-log: debug
  #         ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }}
  #         aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }}
  #         aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }}
  #         aws-region: eu-west-2
  #         do-token: ${{ secrets.SN_TESTNET_DO_PAT }}
  #         ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }}
  #         node-count: 20
  #         vm-count: 1
  #         provider: digital-ocean
  #         testnet-name: NightlyDataLocationTest
  #         safe-network-user: maidsafe
  #         safe-network-branch: main
  #
  #     - name: Upload local logs
  #       if: always()
  #       uses: actions/upload-artifact@v4
  #       with:
  #         name: local_logs_NightlyDataLocationTest
  #         path: |
  #           ~/.local/share/safe/node/*/logs/*.log*
  #           ~/.local/share/safe/*/*/*.log*
  #           ~/.local/share/safe/client/logs/*/*.log*
  #
  #     - name: Stop the WAN network
  #       if: always()
  #       uses: maidsafe/sn-testnet-action@main
  #       with:
  #         action: destroy
  #         re-attempts: 3
  #         rust-log: debug
  #         ansible-vault-password: ${{ secrets.SN_TESTNET_ANSIBLE_VAULT_PASSWORD }}
  #         aws-access-key-id: ${{ secrets.SN_TESTNET_AWS_ACCESS_KEY_ID }}
  #         aws-access-key-secret: ${{ secrets.SN_TESTNET_AWS_SECRET_ACCESS_KEY }}
  #         aws-region: eu-west-2
  #         do-token: ${{ secrets.SN_TESTNET_DO_PAT }}
  #         ssh-secret-key: ${{ secrets.SN_TESTNET_SSH_KEY }}
  #         node-count: 20
  #         vm-count: 1
  #         provider: digital-ocean
  #         testnet-name: NightlyDataLocationTest
  #         safe-network-user: maidsafe
  #         safe-network-branch: main
  #
  #     # TODO: re-enable the below scripts once we have proper way to restart nodes.
  #     # Currently on remote network (not local), the nodes do not handle restart RPC cmd well. They reuse the same
  #     # log location and the logs are over written. Hence the scripts might give false outputs.
  #
  #     # - name: Verify restart of nodes using rg
  #     #   shell: bash
  #     #   timeout-minutes: 1
  #     #   # get the counts, then the specific line, and then the digit count only
  #     #   # then check we have an expected level of restarts
  #     #   # TODO: make this use an env var, or relate to testnet size
  #     #   run : |
  #     #     restart_count=$(rg "Node is restarting in" "${{ matrix.wan_logs_path }}" -c --stats | \
  #     #       rg "(\d+) matches" | rg "\d+" -o)
  #     #     echo "Restart $restart_count nodes"
  #     #     peer_removed=$(rg "PeerRemovedFromRoutingTable" "${{ matrix.wan_logs_path }}" -c --stats | \
  #     #       rg "(\d+) matches" | rg "\d+" -o)
  #     #     echo "PeerRemovedFromRoutingTable $peer_removed times"
  #     #     if [ $peer_removed -lt $restart_count ]; then
  #     #       echo "PeerRemovedFromRoutingTable times of: $peer_removed is less than the restart count of: $restart_count"
  #     #       exit 1
  #     #     fi
  #     #     node_count=$(find "${{ matrix.wan_logs_path }}" -type d | awk -F/ 'NF==9' | grep -E "/12D3KooW" | wc -l)
  #     #     echo "Node dir count is $node_count"
  #     #   # TODO: reenable this once the testnet dir creation is tidied up to avoid a large count here
  #
  #     #   # if [ $restart_count -lt $node_count ]; then
  #     #   #   echo "Restart count of: $restart_count is less than the node count of: $node_count"
  #     #   #   exit 1
  #     #   # fi
  #
  #     # Only error out after uploading the logs
  #     - name: Don't log raw data
  #       if: always() && matrix.os != 'windows-latest' # causes error
  #       shell: bash
  #       timeout-minutes: 10
  #       run: |
  #         if ! rg '^' "${{ matrix.local_safe_path }}"/client/logs | awk 'length($0) > 15000 { print; exit 1 }'
  #         then
  #           echo "We are logging an extremely large data"
  #           exit 1
  #         fi
  #         #  node dir structure: ~/sn-testnet-deploy/logs/NightlyChurnTest/NightlyChurnTest-genesis/safenode1/safenode.log
  #         #faucet dir structure: ~/sn-testnet-deploy/logs/NightlyChurnTest/NightlyChurnTest-genesis/faucet/logs/faucet.log
  #         if ! rg '^' "${{ matrix.wan_logs_path }}"/*/*/*/ | awk 'length($0) > 15000 { print; exit 1 }'
  #         then
  #           echo "We are logging an extremely large data"
  #           exit 1
  #         fi
  #
  #         # sanity check
  #         if ! rg '^' "${{ matrix.local_safe_path }}"/client/logs | awk 'length($0) > 1000 { print; exit 1 }'
  #         then
  #           echo "Sanity check pass for local safe path"
  #         fi
  #         #  node dir structure: ~/sn-testnet-deploy/logs/NightlyChurnTest/NightlyChurnTest-genesis/safenode1/safenode.log
  #         #faucet dir structure: ~/sn-testnet-deploy/logs/NightlyChurnTest/NightlyChurnTest-genesis/faucet/logs/faucet.log
  #         if ! rg '^' "${{ matrix.wan_logs_path }}"/*/*/*/ | awk 'length($0) > 1000 { print; exit 1 }'
  #         then
  #           echo echo "Sanity check pass for wan logs path"
  #         fi
  #
  #     # - name: post notification to slack on failure
  #     #   if: ${{ failure() }}
  #     #   uses: bryannice/gitactions-slack-notification@2.0.0
  #     #   env:
  #     #     SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }}
  #     #     SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}"
  #     #     SLACK_TITLE: "Nightly Data Location Test Run Failed"