Merge upstream v2.60.0 beta8

.github/scripts/cpu_monitor.sh

@@ -0,0 +1,136 @@
+#!/bin/bash
+
+# Configuration
+THRESHOLD=80
+MEASUREMENTS_FILE="/tmp/cpu_measurements.txt"
+MONITOR_INTERVAL=5  # seconds
+PROCESS_NAME="cdk-erigon"
+DETAILED_LOG="/tmp/cpu_detailed.log"
+
+# Function to get CPU usage for all matching processes
+get_process_cpu() {
+    # Clear previous detailed log
+    > "$DETAILED_LOG"
+
+    # Get PIDs of cdk-erigon processes
+    pids=$(pgrep -f "[c]dk-erigon")
+
+    if [ -n "$pids" ]; then
+        # Use top in batch mode for each PID to get current CPU usage
+        for pid in $pids; do
+            # Get process command
+            if [[ "$OSTYPE" == "darwin"* ]]; then
+                cmd=$(ps -p $pid -o command=)
+                cpu=$(top -l 1 -pid $pid | tail -1 | awk '{print $3}')
+            else
+                cmd=$(ps -p $pid -o cmd=)
+                cpu=$(top -b -n 1 -p $pid | tail -1 | awk '{print $9}')
+            fi
+            # Get current CPU usage
+            echo "$pid $cpu $cmd" >> "$DETAILED_LOG"
+        done
+    fi
+
+    # Sum total CPU usage
+    total_cpu=$(awk '{sum += $2} END {printf "%.1f", sum}' "$DETAILED_LOG")
+
+    # Return 0 if no process found
+    if [ -z "$total_cpu" ]; then
+        echo "0.0"
+    else
+        echo "$total_cpu"
+    fi
+}
+
+# Function to show current process details
+show_process_details() {
+    if [ -s "$DETAILED_LOG" ]; then
+        echo "Individual process details:"
+        printf "%-10s %-8s %-s\n" "PID" "CPU%" "Command"
+        echo "----------------------------------------"
+        while read -r line; do
+            pid=$(echo "$line" | awk '{print $1}')
+            cpu=$(echo "$line" | awk '{print $2}')
+            cmd=$(echo "$line" | cut -d' ' -f3-)
+            printf "%-10s %-8.1f %-s\n" "$pid" "$cpu" "$cmd"
+        done < "$DETAILED_LOG"
+        echo "----------------------------------------"
+    else
+        echo "No $PROCESS_NAME processes found"
+    fi
+}
+
+# Function to analyze CPU measurements
+analyze_cpu() {
+    if [ -f "$MEASUREMENTS_FILE" ]; then
+        # Calculate statistics
+        avg_cpu=$(awk '{ sum += $1 } END { print sum/NR }' "$MEASUREMENTS_FILE")
+        avg_cpu_rounded=$(printf "%.1f" "$avg_cpu")
+        max_cpu=$(awk 'BEGIN{max=0} {if($1>max) max=$1} END{print max}' "$MEASUREMENTS_FILE")
+        measurement_count=$(wc -l < "$MEASUREMENTS_FILE")
+
+        echo ""
+        echo "=== CPU Usage Analysis for all $PROCESS_NAME processes ==="
+        echo "Number of measurements: $measurement_count"
+        echo "Average Combined CPU Usage: $avg_cpu_rounded%"
+        echo "Peak Combined CPU Usage: $max_cpu%"
+        echo "Threshold: $THRESHOLD%"
+
+        # Get final process details for the report
+        echo ""
+        echo "Final process state:"
+        show_process_details
+
+        # Compare with threshold
+        if [ "$(echo "$avg_cpu > $THRESHOLD" | bc -l)" -eq 1 ]; then
+            echo ""
+            echo "ERROR: Average CPU usage ($avg_cpu_rounded%) exceeded threshold of $THRESHOLD%"
+            cleanup_and_exit 1
+        else
+            echo ""
+            echo "SUCCESS: CPU usage ($avg_cpu_rounded%) is within threshold of $THRESHOLD%"
+            cleanup_and_exit 0
+        fi
+    else
+        echo "ERROR: No CPU measurements found at $MEASUREMENTS_FILE"
+        cleanup_and_exit 1
+    fi
+}
+
+# Function to clean up and exit
+cleanup_and_exit() {
+    exit_code=$1
+    rm -f "$DETAILED_LOG"
+    exit $exit_code
+}
+
+# Function to handle interruption
+handle_interrupt() {
+    echo ""
+    echo "Monitoring interrupted. Analyzing collected data..."
+    analyze_cpu
+}
+
+# Set up trap for various signals
+trap handle_interrupt TERM INT
+
+# Clear measurements file
+> "$MEASUREMENTS_FILE"
+> "$DETAILED_LOG"
+
+echo "Starting CPU monitoring for all '$PROCESS_NAME' processes"
+echo "Storing measurements in $MEASUREMENTS_FILE"
+echo "Monitoring interval: ${MONITOR_INTERVAL}s"
+echo "Press Ctrl+C to stop monitoring and see analysis"
+echo ""
+
+# Start monitoring loop
+while true; do
+    # Get CPU usage for all matching processes
+    cpu_usage=$(get_process_cpu)
+    echo "$cpu_usage" >> "$MEASUREMENTS_FILE"
+    echo "$(date '+%Y-%m-%d %H:%M:%S') - Combined CPU Usage: $cpu_usage%"
+    show_process_details
+    echo ""
+    sleep "$MONITOR_INTERVAL"
+done

.github/workflows/ci_zkevm.yml

@@ -116,6 +116,23 @@ jobs:
         run: | 
           kurtosis run --enclave cdk-v1 --image-download always . '{"args": {"data_availability_mode": "${{ matrix.da-mode }}", "cdk_erigon_node_image": "cdk-erigon:local"}}'
 
+      - name: Run process with CPU monitoring
+        working-directory: ./cdk-erigon
+        run: |
+          # Start monitoring in background
+          bash ./.github/scripts/cpu_monitor.sh &
+          monitor_pid=$!
+
+          # Wait for 30 seconds
+          sleep 30
+
+          # Stop monitoring and get analysis
+          kill -TERM $monitor_pid
+          wait $monitor_pid || {
+            echo "CPU usage exceeded threshold!"
+            exit 1
+          }
+
       - name: Monitor verified batches
         working-directory: ./kurtosis-cdk
         shell: bash

Makefile

@@ -194,6 +194,9 @@ lint:
 	@./erigon-lib/tools/golangci_lint.sh
 	@./erigon-lib/tools/mod_tidy_check.sh
 
+cpu_monitor:
+	@.github/scripts/cpu_monitor.sh
+
 ## clean:                             cleans the go cache, build dir, libmdbx db dir
 clean:
 	go clean -cache

README.md

@@ -208,6 +208,15 @@ Useful config entries:
 - `zkevm.sync-limit`: This will ensure the network only syncs to a given block height.
 - `debug.timers`: This will enable debug timers in the logs to help with performance tuning. Recording timings of witness generation, etc. at INFO level.
 
+Metrics and pprof configuration flags:
+
+- `metrics:` Enables or disables the metrics collection. Set to true to enable.
+- `metrics.addr`: The address on which the metrics server will listen. Default is "0.0.0.0".
+- `metrics.port`: The port on which the metrics server will listen. Default is 6060.
+- `pprof`: Enables or disables the pprof profiling. Set to true to enable.
+- `pprof.addr`: The address on which the pprof server will listen. Default is "0.0.0.0".
+- `pprof.port`: The port on which the pprof server will listen. Default is 6061.
+
 ***
 
 

cmd/rpcdaemon/health/check_time.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"net/http"
 
+	"github.com/ledgerwatch/erigon-lib/common/hexutil"
+
 	"github.com/ledgerwatch/erigon/rpc"
 )
 
@@ -20,13 +22,13 @@ func checkTime(
 	if err != nil {
 		return err
 	}
-	timestamp := 0
+	timestamp := uint64(0)
 	if ts, ok := i["timestamp"]; ok {
-		if cs, ok := ts.(uint64); ok {
-			timestamp = int(cs)
+		if cs, ok := ts.(hexutil.Uint64); ok {
+			timestamp = cs.Uint64()
 		}
 	}
-	if timestamp < seconds {
+	if timestamp < uint64(seconds) {
 		return fmt.Errorf("%w: got ts: %d, need: %d", errTimestampTooOld, timestamp, seconds)
 	}
 

cmd/rpcdaemon/health/health_test.go

@@ -245,7 +245,7 @@ func TestProcessHealthcheckIfNeeded_HeadersTests(t *testing.T) {
 			netApiResponse: hexutil.Uint(1),
 			netApiError:    nil,
 			ethApiBlockResult: map[string]interface{}{
-				"timestamp": uint64(time.Now().Add(-10 * time.Second).Unix()),
+				"timestamp": hexutil.Uint64(time.Now().Add(-10 * time.Second).Unix()),
 			},
 			ethApiBlockError:    nil,
 			ethApiSyncingResult: false,
@@ -264,7 +264,7 @@ func TestProcessHealthcheckIfNeeded_HeadersTests(t *testing.T) {
 			netApiResponse: hexutil.Uint(1),
 			netApiError:    nil,
 			ethApiBlockResult: map[string]interface{}{
-				"timestamp": uint64(time.Now().Add(-1 * time.Hour).Unix()),
+				"timestamp": hexutil.Uint64(time.Now().Add(-1 * time.Hour).Unix()),
 			},
 			ethApiBlockError:    nil,
 			ethApiSyncingResult: false,
@@ -283,7 +283,7 @@ func TestProcessHealthcheckIfNeeded_HeadersTests(t *testing.T) {
 			netApiResponse: hexutil.Uint(1),
 			netApiError:    nil,
 			ethApiBlockResult: map[string]interface{}{
-				"timestamp": uint64(time.Now().Add(1 * time.Hour).Unix()),
+				"timestamp": hexutil.Uint64(time.Now().Add(1 * time.Hour).Unix()),
 			},
 			ethApiBlockError:    nil,
 			ethApiSyncingResult: false,
@@ -319,7 +319,7 @@ func TestProcessHealthcheckIfNeeded_HeadersTests(t *testing.T) {
 			netApiResponse: hexutil.Uint(10),
 			netApiError:    nil,
 			ethApiBlockResult: map[string]interface{}{
-				"timestamp": uint64(time.Now().Add(1 * time.Second).Unix()),
+				"timestamp": hexutil.Uint64(time.Now().Add(1 * time.Second).Unix()),
 			},
 			ethApiBlockError:    nil,
 			ethApiSyncingResult: false,

cmd/utils/flags.go

@@ -758,6 +758,11 @@ var (
 		Usage: "Seal the batch immediately when detecting a counter overflow",
 		Value: false,
 	}
+	MockWitnessGeneration = cli.BoolFlag{
+		Name:  "zkevm.mock-witness-generation",
+		Usage: "Mock the witness generation",
+		Value: false,
+	}
 	ACLPrintHistory = cli.IntFlag{
 		Name:  "acl.print-history",
 		Usage: "Number of entries to print from the ACL history on node start up",

docs/endpoints/endpoints.md

@@ -200,6 +200,8 @@ If the endpoint is not in the list below, it means this specific endpoint is not
 - zkevm_getL2BlockInfoTree
 - zkevm_getLatestGlobalExitRoot
 - zkevm_getProverInput
+- zkevm_getRollupAddress
+- zkevm_getRollupManagerAddress
 - zkevm_getVersionHistory
 - zkevm_getWitness
 - zkevm_isBlockConsolidated

eth/ethconfig/config_zkevm.go

@@ -96,6 +96,7 @@ type Zk struct {
 	InfoTreeUpdateInterval         time.Duration
 	BadBatches                     []uint64
 	SealBatchImmediatelyOnOverflow bool
+	MockWitnessGeneration          bool
 }
 
 var DefaultZkConfig = Zk{

test/Makefile

@@ -83,10 +83,11 @@ all: ## Runs a full node
 	$(RUN_DOCKER_BRIDGE_COIN_KAFKA)
 	sleep 3
 	$(RUN_DOCKER_DATA_AVAILABILITY)
+	$(RUN_DOCKER_APPROVE)
 
 	# app services
 	#sleep 3
-	#$(RUN_DOCKER_STATELESS_EXECUTOR)
+	$(RUN_DOCKER_STATELESS_EXECUTOR)
 	$(RUN_DOCKER_SEQ)
 	$(RUN_DOCKER_PROVER)
 	sleep 10
@@ -97,7 +98,7 @@ all: ## Runs a full node
 	$(RUN_DOCKER_RPC)
 
 	# bridge services
-	sleep 3
+	sleep 30
 	$(RUN_DOCKER_BRIDGE_SERVICE)
 	sleep 3
 	$(RUN_DOCKER_BRIDGE_UI)

turbo/cli/default_flags.go

@@ -328,4 +328,5 @@ var DefaultFlags = []cli.Flag{
 	&utils.ACLPrintHistory,
 	&utils.InfoTreeUpdateInterval,
 	&utils.SealBatchImmediatelyOnOverflow,
+	&utils.MockWitnessGeneration,
 }

turbo/cli/flags_zkevm.go

@@ -210,6 +210,7 @@ func ApplyFlagsForZkConfig(ctx *cli.Context, cfg *ethconfig.Config) {
 		ACLPrintHistory:                        ctx.Int(utils.ACLPrintHistory.Name),
 		InfoTreeUpdateInterval:                 ctx.Duration(utils.InfoTreeUpdateInterval.Name),
 		SealBatchImmediatelyOnOverflow:         ctx.Bool(utils.SealBatchImmediatelyOnOverflow.Name),
+		MockWitnessGeneration:                  ctx.Bool(utils.MockWitnessGeneration.Name),
 	}
 
 	// For X Layer

turbo/jsonrpc/bor_helper.go

@@ -58,7 +58,7 @@ func getHeaderByNumber(ctx context.Context, number rpc.BlockNumber, api *BorImpl
 		return block.Header(), nil
 	}
 
-	blockNum, _, _, err := rpchelper.GetBlockNumber(rpc.BlockNumberOrHashWithNumber(number), tx, api.filters)
+	blockNum, _, _, err := rpchelper.GetBlockNumber_zkevm(rpc.BlockNumberOrHashWithNumber(number), tx, api.filters)
 	if err != nil {
 		return nil, err
 	}

turbo/jsonrpc/debug_api.go

@@ -375,7 +375,8 @@ func (api *PrivateDebugAPIImpl) GetRawHeader(ctx context.Context, blockNrOrHash
 		return nil, err
 	}
 	defer tx.Rollback()
-	n, h, _, err := rpchelper.GetBlockNumber(blockNrOrHash, tx, api.filters)
+
+	n, h, _, err := rpchelper.GetBlockNumber_zkevm(blockNrOrHash, tx, api.filters)
 	if err != nil {
 		return nil, err
 	}
@@ -395,7 +396,7 @@ func (api *PrivateDebugAPIImpl) GetRawBlock(ctx context.Context, blockNrOrHash r
 		return nil, err
 	}
 	defer tx.Rollback()
-	n, h, _, err := rpchelper.GetBlockNumber(blockNrOrHash, tx, api.filters)
+	n, h, _, err := rpchelper.GetBlockNumber_zkevm(blockNrOrHash, tx, api.filters)
 	if err != nil {
 		return nil, err
 	}

turbo/jsonrpc/erigon_block.go

@@ -43,7 +43,7 @@ func (api *ErigonImpl) GetHeaderByNumber(ctx context.Context, blockNumber rpc.Bl
 	}
 	defer tx.Rollback()
 
-	blockNum, _, _, err := rpchelper.GetBlockNumber(rpc.BlockNumberOrHashWithNumber(blockNumber), tx, api.filters)
+	blockNum, _, _, err := rpchelper.GetBlockNumber_zkevm(rpc.BlockNumberOrHashWithNumber(blockNumber), tx, api.filters)
 	if err != nil {
 		return nil, err
 	}
@@ -213,7 +213,7 @@ func (api *ErigonImpl) GetBalanceChangesInBlock(ctx context.Context, blockNrOrHa
 		return nil, err
 	}
 
-	blockNumber, _, _, err := rpchelper.GetBlockNumber(blockNrOrHash, tx, api.filters)
+	blockNumber, _, _, err := rpchelper.GetBlockNumber_zkevm(blockNrOrHash, tx, api.filters)
 	if err != nil {
 		return nil, err
 	}

turbo/jsonrpc/erigon_receipts.go

@@ -407,7 +407,7 @@ func (api *ErigonImpl) GetBlockReceiptsByBlockHash(ctx context.Context, cannonic
 		}
 	}
 
-	blockNum, _, _, err := rpchelper.GetBlockNumber(rpc.BlockNumberOrHashWithHash(cannonicalBlockHash, true), tx, api.filters)
+	blockNum, _, _, err := rpchelper.GetBlockNumber_zkevm(rpc.BlockNumberOrHashWithHash(cannonicalBlockHash, true), tx, api.filters)
 	if err != nil {
 		return nil, err
 	}

turbo/jsonrpc/eth_api.go

@@ -288,7 +288,7 @@ func (api *BaseAPI) pendingBlock() *types.Block {
 }
 
 func (api *BaseAPI) blockByRPCNumber(ctx context.Context, number rpc.BlockNumber, tx kv.Tx) (*types.Block, error) {
-	n, h, _, err := rpchelper.GetBlockNumber(rpc.BlockNumberOrHashWithNumber(number), tx, api.filters)
+	n, h, _, err := rpchelper.GetBlockNumber_zkevm(rpc.BlockNumberOrHashWithNumber(number), tx, api.filters)
 	if err != nil {
 		return nil, err
 	}
@@ -299,7 +299,7 @@ func (api *BaseAPI) blockByRPCNumber(ctx context.Context, number rpc.BlockNumber
 }
 
 func (api *BaseAPI) headerByRPCNumber(ctx context.Context, number rpc.BlockNumber, tx kv.Tx) (*types.Header, error) {
-	n, h, _, err := rpchelper.GetBlockNumber(rpc.BlockNumberOrHashWithNumber(number), tx, api.filters)
+	n, h, _, err := rpchelper.GetBlockNumber_zkevm(rpc.BlockNumberOrHashWithNumber(number), tx, api.filters)
 	if err != nil {
 		return nil, err
 	}
@@ -320,7 +320,7 @@ func (api *BaseAPI) checkPruneHistory(tx kv.Tx, block uint64) error {
 		return nil
 	}
 	if p.History.Enabled() {
-		latest, _, _, err := rpchelper.GetBlockNumber(rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber), tx, api.filters)
+		latest, _, _, err := rpchelper.GetBlockNumber_zkevm(rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber), tx, api.filters)
 		if err != nil {
 			return err
 		}