Introduce ledger-tool simulate-block-production (#2733)

* Introduce ledger-tool simulate-block-production

* Move counting code out of time-sensitive loop

* Avoid misleading ::clone() altogether

* Use while instead of loop+break

* Add comment of using BTreeMap

* Reduce simulation jitter due to mem deallocs

* Rename to CostTracker::new_from_parent_limits()

* Make ::load() take a slice

* Clean up retracer code a bit

* Add comment about BaningTracer even inside sim

* Remove redundant dcou dev-dependencies

* Apply suggestions from code review

Co-authored-by: Andrew Fitzgerald <[email protected]>

* Fix up and promote to doc comments

* Make warm-up code and doc simpler

* Further clean up timed_batches_to_send

* Fix wrong units...

* Replace new_with_dummy_keypair() with traits

* Tweak --no-block-cost-limits description

* Remove redundant dev-dependencies

* Use RwLock to mimic real ClusterInfo

* Fix typo

* Refactor too long BankingSimulator::start()

* Reduce indent

* Calculate required_duration in advance

* Use correct format specifier instead of cast

* Align formatting by using ::*

* Make envs overridable

* Add comment for SOLANA_VALIDATOR_EXIT_TIMEOUT

* Clarify comment a bit

* Fix typoss

* Fix typos

Co-authored-by: Andrew Fitzgerald <[email protected]>

* Use correct variant name: DeserializeError

* Remove SimulatorLoopLogger::new()

* Fix typos more

* Add explicit _batch in field names

* Avoid unneeded events: Vec<_> buffering

* Manually adjust logging code styles

* Align name: spawn_sender_loop/enter_simulator_loop

* Refactor by introducing {Sender,Simulator}Loop

* Fix out-of-sync sim due to timed preprocessing

* Fix too-early base_simulation_time creation

* Don't log confusing info! after leader slots

* Add justification comment of BroadcastStage

* Align timeout values

* Comment about snapshot_slot=50

* Don't squash all errors unconditionally

* Remove repetitive exitence check

* Promote no_block_cost_limits logging level

* Make ci/run-sanity.sh more robust

* Improve wordking of --enable-hash-overrides

* Remove marker-file based abortion mechanism

* Remove needless touch

---------

Co-authored-by: Andrew Fitzgerald <[email protected]>

ci/run-sanity.sh

@@ -5,9 +5,27 @@ cd "$(dirname "$0")/.."
 # shellcheck source=multinode-demo/common.sh
 source multinode-demo/common.sh
 
+if [[ -z $CI ]]; then
+  # Build eargerly if needed for local development. Otherwise, odd timing error occurs...
+  $solana_keygen --version
+  $solana_genesis --version
+  $solana_faucet --version
+  $solana_cli --version
+  $agave_validator --version
+  $solana_ledger_tool --version
+fi
+
 rm -rf config/run/init-completed config/ledger config/snapshot-ledger
 
-SOLANA_RUN_SH_VALIDATOR_ARGS="--full-snapshot-interval-slots 200" timeout 120 ./scripts/run.sh &
+# Sanity-check that agave-validator can successfully terminate itself without relying on
+# process::exit() by extending the timeout...
+# Also the banking_tracer thread needs some extra time to flush due to
+# unsynchronized and buffered IO.
+validator_timeout="${SOLANA_VALIDATOR_EXIT_TIMEOUT:-120}"
+SOLANA_RUN_SH_VALIDATOR_ARGS="${SOLANA_RUN_SH_VALIDATOR_ARGS} --full-snapshot-interval-slots 200" \
+  SOLANA_VALIDATOR_EXIT_TIMEOUT="$validator_timeout" \
+  timeout "$validator_timeout" ./scripts/run.sh &
+
 pid=$!
 
 attempts=20
@@ -21,7 +39,10 @@ while [[ ! -f config/run/init-completed ]]; do
   fi
 done
 
-snapshot_slot=1
+# Needs bunch of slots for simulate-block-production.
+# Better yet, run ~20 secs to run longer than its warm-up.
+# As a bonus, this works as a sanity test of general slot-rooting behavior.
+snapshot_slot=50
 latest_slot=0
 
 # wait a bit longer than snapshot_slot
@@ -39,5 +60,18 @@ $solana_ledger_tool create-snapshot --ledger config/ledger "$snapshot_slot" conf
 cp config/ledger/genesis.tar.bz2 config/snapshot-ledger
 $solana_ledger_tool copy --ledger config/ledger \
   --target-db config/snapshot-ledger --starting-slot "$snapshot_slot" --ending-slot "$latest_slot"
-$solana_ledger_tool verify --ledger config/snapshot-ledger --block-verification-method blockstore-processor
-$solana_ledger_tool verify --ledger config/snapshot-ledger --block-verification-method unified-scheduler
+$solana_ledger_tool verify --abort-on-invalid-block \
+  --ledger config/snapshot-ledger --block-verification-method blockstore-processor
+$solana_ledger_tool verify --abort-on-invalid-block \
+  --ledger config/snapshot-ledger --block-verification-method unified-scheduler
+
+first_simulated_slot=$((latest_slot / 2))
+purge_slot=$((first_simulated_slot + latest_slot / 4))
+echo "First simulated slot: ${first_simulated_slot}"
+# Purge some slots so that later verify fails if sim is broken
+$solana_ledger_tool purge --ledger config/ledger "$purge_slot"
+$solana_ledger_tool simulate-block-production --ledger config/ledger \
+  --first-simulated-slot $first_simulated_slot
+# Slots should be available and correctly replayable upto snapshot_slot at least.
+$solana_ledger_tool verify --abort-on-invalid-block \
+  --ledger config/ledger --enable-hash-overrides --halt-at-slot "$snapshot_slot"

core/Cargo.toml

@@ -105,7 +105,6 @@ solana-ledger = { workspace = true, features = ["dev-context-only-utils"] }
 solana-logger = { workspace = true }
 solana-poh = { workspace = true, features = ["dev-context-only-utils"] }
 solana-program-runtime = { workspace = true }
-solana-runtime = { workspace = true, features = ["dev-context-only-utils"] }
 solana-sdk = { workspace = true, features = ["dev-context-only-utils"] }
 solana-stake-program = { workspace = true }
 solana-unified-scheduler-pool = { workspace = true, features = [
@@ -123,7 +122,9 @@ sysctl = { workspace = true }
 rustc_version = { workspace = true, optional = true }
 
 [features]
-dev-context-only-utils = []
+dev-context-only-utils = [
+    "solana-runtime/dev-context-only-utils",
+]
 frozen-abi = [
     "dep:rustc_version",
     "dep:solana-frozen-abi",

core/benches/forwarder.rs

@@ -34,7 +34,7 @@ use {
 struct BenchSetup {
     exit: Arc<AtomicBool>,
     poh_service: PohService,
-    forwarder: Forwarder,
+    forwarder: Forwarder<Arc<ClusterInfo>>,
     unprocessed_packet_batches: UnprocessedTransactionStorage,
     tracker: LeaderSlotMetricsTracker,
     stats: BankingStageStats,