draft for integration testing

.github/workflows/cron_jerigon_zero_testing.yml

@@ -0,0 +1,86 @@
+name: Jerigon Zero Testing
+
+on:
+  # TODO - Uncomment when ready to run on a schedule
+  # schedule:
+  #   # Run every Sunday at 12:00 AM (UTC)
+  #   - cron: "0 0 * * 0"
+  push:
+    branches: [develop, main]
+  pull_request:
+    branches:
+      - "**"
+  workflow_dispatch:
+    branches:
+      - "**"
+
+env:
+  CARGO_TERM_COLOR: always
+  REGISTRY: ghcr.io
+
+jobs:
+  jerigon_zero_testing:
+    name: Jerigon Zero Testing - Integration and Benchmarking
+    runs-on: zero-ci
+    steps:
+      - name: Checkout zk_evm code
+        uses: actions/checkout@v4
+
+      - name: Setup Rust Toolchain
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+
+      - name: Set up rust cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: true
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # TODO - Find a better way to store erigon-data.tar.gz. Currently storing it in zk evm repo temporarily. Maybe download it from a repo and use it here. Also, extract the data in a temp file maybe
+      - name: Run erigon network
+        run: |
+          cd ..
+          tar xf "$(pwd)/zk_evm/test-data/erigon-data.tar.gz"
+          docker pull ghcr.io/0xpolygonzero/erigon:feat-zero
+          docker run -d -p 8545:8545 -v $(pwd):/data \
+            ghcr.io/0xpolygonzero/erigon:feat-zero \
+            --datadir=/data/erigon/execution-data --http.api=eth,erigon,engine,web3,net,debug,trace,txpool,admin \
+            --http.vhosts=* --ws --http --http.addr=0.0.0.0 --http.corsdomain=* --http.port=8545 \
+            --metrics --metrics.addr=0.0.0.0 --metrics.port=9001 --db.size.limit=3000MB
+
+      # TODO - Decide the number of blocks we want to test with
+      - name: Regression test with zero tracer in real mode
+        run: |
+          export ETH_RPC_URL="http://localhost:8545"
+          rm -rf proofs/* circuits/* ./proofs.json test.out verify.out leader.out
+          random_numbers=($(shuf -i 1-500 -n 10))
+          for number in "${random_numbers[@]}"; do
+            echo $number
+            hex_number="0x$(echo "obase=16; $number" | bc)"
+            echo $hex_number
+            OUTPUT_TO_TERMINAL=true RUN_VERIFICATION=true ./scripts/prove_rpc.sh $hex_number $hex_number $ETH_RPC_URL jerigon true 3000 100
+          done
+
+      # - name: Download previous results for becnhmarking
+      #   uses: dawidd6/action-download-artifact@v6
+      #   with:
+      #     workflow: cron_jerigon_zero_testing.yml
+      #     workflow_conclusion: success
+      #     name: jerigon_zero_testing_benchmark
+      #     path: ./
+      #     if_no_artifact_found: ignore
+
+      # - name: Benchmarking test with zero tracer in real mode
+      #   run: |
+      #     export ETH_RPC_URL="http://localhost:8545"
+      #     rm -rf proofs/* circuits/* ./proofs.json test.out verify.out leader.out
+      #     echo "Running the benchmarking script..."
+      #     ./scripts/jerigon_zero_benchmark.sh | tee benchmark_output.log

scripts/jerigon_zero_benchmark.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+# ------------------------------------------------------------------------------
+set -exo pipefail
+
+# We're going to set the parallelism in line with the total cpu count
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    num_procs=$(sysctl -n hw.physicalcpu)
+else
+    num_procs=$(nproc)
+fi
+
+# Force the working directory to always be the `tools/` directory.
+REPO_ROOT=$(git rev-parse --show-toplevel)
+PROOF_OUTPUT_DIR="${REPO_ROOT}/proofs"
+
+BLOCK_BATCH_SIZE="${BLOCK_BATCH_SIZE:-8}"
+echo "Block batch size: $BLOCK_BATCH_SIZE"
+
+OUTPUT_LOG="${REPO_ROOT}/output.log"
+PROOFS_FILE_LIST="${PROOF_OUTPUT_DIR}/proof_files.json"
+TEST_OUT_PATH="${REPO_ROOT}/test.out"
+
+# Configured Rayon and Tokio with rough defaults
+export RAYON_NUM_THREADS=$num_procs
+export TOKIO_WORKER_THREADS=$num_procs
+
+export RUST_MIN_STACK=33554432
+export RUST_BACKTRACE=full
+export RUST_LOG=info
+
+INPUT_FILE=$1
+
+if [[ $INPUT_FILE == "" ]]; then
+    echo "Please provide witness json input file, e.g. artifacts/witness_b19240705.json"
+    exit 1
+fi
+
+start_time=$(date +%s%N)
+perf stat -e cycles "${REPO_ROOT}/target/release/leader" --runtime in-memory --load-strategy monolithic --block-batch-size $BLOCK_BATCH_SIZE \
+ --proof-output-dir $PROOF_OUTPUT_DIR stdio < $INPUT_FILE &> $OUTPUT_LOG
+end_time=$(date +%s%N)
+
+set +o pipefail
+cat $OUTPUT_LOG | grep "Successfully wrote to disk proof file " | awk '{print $NF}' | tee $PROOFS_FILE_LIST
+if [ ! -s "$PROOFS_FILE_LIST" ]; then
+  # Some error occurred, display the logs and exit.
+  cat $OUTPUT_LOG
+  echo "Proof list not generated, some error happened. For more details check the log file $OUTPUT_LOG"
+  exit 1
+fi
+
+duration_ns=$((end_time - start_time))
+duration_sec=$(echo "$duration_ns / 1000000000" | bc -l)
+
+echo "Success!"
+echo "Proving duration:" $duration_sec " seconds"
+echo "Note, this duration is inclusive of circuit handling and overall process initialization";