From 4a44bd0f54fccecf8fd8fa0f529717eb6c650ff3 Mon Sep 17 00:00:00 2001 From: greg Date: Wed, 21 Aug 2024 23:16:06 +0000 Subject: [PATCH 1/2] add flag to bake non bootstrap validator stakes into genesis --- genesis/src/main.rs | 172 +++++++++++++++++++++++++++++++------------- 1 file changed, 123 insertions(+), 49 deletions(-) diff --git a/genesis/src/main.rs b/genesis/src/main.rs index 46b0ceadd77614..66c4a4225c7f91 100644 --- a/genesis/src/main.rs +++ b/genesis/src/main.rs @@ -49,6 +49,7 @@ use { io::{self, Read}, path::PathBuf, process, + slice::Iter, str::FromStr, time::Duration, }, @@ -176,6 +177,69 @@ fn features_to_deactivate_for_cluster( Ok(features_to_deactivate) } +fn validate_pubkeys(matches: &ArgMatches, arg_name: &str) -> Vec { + // This method will only return an empty vector in the case where `--validator` is not provided + // Clap will enforce that `--bootstrap` flag is provided + let pubkeys = pubkeys_of(matches, arg_name).unwrap_or_default(); + if !pubkeys.is_empty() { + assert_eq!(pubkeys.len() % 3, 0); + + // Ensure there are no duplicated pubkeys in the list + let mut v = pubkeys.clone(); + v.sort(); + v.dedup(); + if v.len() != pubkeys.len() { + eprintln!("Error: --{arg_name} pubkeys cannot be duplicated"); + process::exit(1); + } + } + pubkeys +} + +fn add_validator_accounts( + genesis_config: &mut GenesisConfig, + pubkeys_iter: &mut Iter, + lamports: u64, + stake_lamports: u64, + commission: u8, + rent: &Rent, + authorized_pubkey: Option<&Pubkey>, +) { + loop { + let Some(identity_pubkey) = pubkeys_iter.next() else { + break; + }; + let vote_pubkey = pubkeys_iter.next().unwrap(); + let stake_pubkey = pubkeys_iter.next().unwrap(); + + genesis_config.add_account( + *identity_pubkey, + AccountSharedData::new(lamports, 0, &system_program::id()), + ); + + let vote_account = vote_state::create_account_with_authorized( + identity_pubkey, + identity_pubkey, + identity_pubkey, + commission, + VoteState::get_rent_exempt_reserve(rent).max(1), + ); + + genesis_config.add_account( + *stake_pubkey, + stake_state::create_account( + authorized_pubkey.unwrap_or(identity_pubkey), + vote_pubkey, + &vote_account, + rent, + stake_lamports, + ), + ); + + genesis_config.add_account(*vote_pubkey, vote_account); + } +} + #[allow(clippy::cognitive_complexity)] fn main() -> Result<(), Box> { let default_faucet_pubkey = solana_cli_config::Config::default().keypair_path; @@ -214,6 +278,9 @@ fn main() -> Result<(), Box> { .max(rent.minimum_balance(StakeStateV2::size_of())) .to_string(); + let default_validator_lamports = default_bootstrap_validator_lamports; + let default_validator_stake_lamports = default_bootstrap_validator_stake_lamports; + let default_target_tick_duration = PohConfig::default().target_tick_duration; let default_ticks_per_slot = &clock::DEFAULT_TICKS_PER_SLOT.to_string(); let default_cluster_type = "mainnet-beta"; @@ -242,6 +309,17 @@ fn main() -> Result<(), Box> { .required(true) .help("The bootstrap validator's identity, vote and stake pubkeys"), ) + .arg( + Arg::with_name("internal_validator") + .short("v") + .long("internal-validator") + .value_name("IDENTITY_PUBKEY VOTE_PUBKEY STAKE_PUBKEY") + .takes_value(true) + .validator(is_pubkey_or_keypair) + .number_of_values(3) + .multiple(true) + .help("An internal (non bootstrap) validator's identity, vote, and stake pubkeys and its stake in lamports"), + ) .arg( Arg::with_name("ledger_path") .short("l") @@ -297,6 +375,22 @@ fn main() -> Result<(), Box> { .default_value(default_bootstrap_validator_stake_lamports) .help("Number of lamports to assign to the bootstrap validator's stake account"), ) + .arg( + Arg::with_name("internal_validator_lamports") + .long("internal-validator-lamports") + .value_name("LAMPORTS") + .takes_value(true) + .default_value(default_validator_lamports) + .help("Number of lamports to assign to the internal (non bootstrap) validators"), + ) + .arg( + Arg::with_name("internal_validator_stake_lamports") + .long("internal-validator-stake-lamports") + .value_name("LAMPORTS") + .takes_value(true) + .default_value(default_validator_stake_lamports) + .help("Number of lamports to assign to the internal (non bootstrap) validators' stake account"), + ) .arg( Arg::with_name("target_lamports_per_signature") .long("target-lamports-per-signature") @@ -516,19 +610,8 @@ fn main() -> Result<(), Box> { } } - let bootstrap_validator_pubkeys = pubkeys_of(&matches, "bootstrap_validator").unwrap(); - assert_eq!(bootstrap_validator_pubkeys.len() % 3, 0); - - // Ensure there are no duplicated pubkeys in the --bootstrap-validator list - { - let mut v = bootstrap_validator_pubkeys.clone(); - v.sort(); - v.dedup(); - if v.len() != bootstrap_validator_pubkeys.len() { - eprintln!("Error: --bootstrap-validator pubkeys cannot be duplicated"); - process::exit(1); - } - } + let bootstrap_validator_pubkeys = validate_pubkeys(&matches, "bootstrap_validator"); + let internal_validator_pubkeys = validate_pubkeys(&matches, "internal_validator"); let bootstrap_validator_lamports = value_t_or_exit!(matches, "bootstrap_validator_lamports", u64); @@ -539,6 +622,13 @@ fn main() -> Result<(), Box> { rent.minimum_balance(StakeStateV2::size_of()), )?; + let internal_validator_lamports = value_t_or_exit!(matches, "internal_validator_lamports", u64); + let internal_validator_stake_lamports = rent_exempt_check( + &matches, + "internal_validator_stake_lamports", + rent.minimum_balance(StakeStateV2::size_of()), + )?; + let bootstrap_stake_authorized_pubkey = pubkey_of(&matches, "bootstrap_stake_authorized_pubkey"); let faucet_lamports = value_t!(matches, "faucet_lamports", u64).unwrap_or(0); @@ -627,43 +717,27 @@ fn main() -> Result<(), Box> { } let commission = value_t_or_exit!(matches, "vote_commission_percentage", u8); + let rent = genesis_config.rent.clone(); + + add_validator_accounts( + &mut genesis_config, + &mut bootstrap_validator_pubkeys.iter(), + bootstrap_validator_lamports, + bootstrap_validator_stake_lamports, + commission, + &rent, + bootstrap_stake_authorized_pubkey.as_ref(), + ); - let mut bootstrap_validator_pubkeys_iter = bootstrap_validator_pubkeys.iter(); - loop { - let Some(identity_pubkey) = bootstrap_validator_pubkeys_iter.next() else { - break; - }; - let vote_pubkey = bootstrap_validator_pubkeys_iter.next().unwrap(); - let stake_pubkey = bootstrap_validator_pubkeys_iter.next().unwrap(); - - genesis_config.add_account( - *identity_pubkey, - AccountSharedData::new(bootstrap_validator_lamports, 0, &system_program::id()), - ); - - let vote_account = vote_state::create_account_with_authorized( - identity_pubkey, - identity_pubkey, - identity_pubkey, - commission, - VoteState::get_rent_exempt_reserve(&genesis_config.rent).max(1), - ); - - genesis_config.add_account( - *stake_pubkey, - stake_state::create_account( - bootstrap_stake_authorized_pubkey - .as_ref() - .unwrap_or(identity_pubkey), - vote_pubkey, - &vote_account, - &genesis_config.rent, - bootstrap_validator_stake_lamports, - ), - ); - - genesis_config.add_account(*vote_pubkey, vote_account); - } + add_validator_accounts( + &mut genesis_config, + &mut internal_validator_pubkeys.iter(), + internal_validator_lamports, + internal_validator_stake_lamports, + commission, + &rent, + None, + ); if let Some(creation_time) = unix_timestamp_from_rfc3339_datetime(&matches, "creation_time") { genesis_config.creation_time = creation_time; From cae626f69e4cee703efe93e6ee1344c2f562361c Mon Sep 17 00:00:00 2001 From: greg Date: Wed, 2 Oct 2024 22:34:48 +0000 Subject: [PATCH 2/2] switch to validator-genesis-accounts file for baking in validator stakes into genesis --- genesis/src/lib.rs | 11 ++ genesis/src/main.rs | 279 +++++++++++++++++++++++++++++++++----------- 2 files changed, 219 insertions(+), 71 deletions(-) diff --git a/genesis/src/lib.rs b/genesis/src/lib.rs index 5faf788d0f4f8a..15785dca9245ea 100644 --- a/genesis/src/lib.rs +++ b/genesis/src/lib.rs @@ -14,3 +14,14 @@ pub struct Base64Account { pub data: String, pub executable: bool, } + +/// A validator account where the data is encoded as a Base64 string. +/// Includes the vote account and stake account. +#[derive(Serialize, Deserialize, Debug)] +pub struct Base64ValidatorAccount { + pub balance_lamports: u64, + pub stake_lamports: u64, + pub identity_account: String, + pub vote_account: String, + pub stake_account: String, +} diff --git a/genesis/src/main.rs b/genesis/src/main.rs index 66c4a4225c7f91..6a43c0cebe3761 100644 --- a/genesis/src/main.rs +++ b/genesis/src/main.rs @@ -16,7 +16,9 @@ use { }, }, solana_entry::poh::compute_hashes_per_tick, - solana_genesis::{genesis_accounts::add_genesis_accounts, Base64Account}, + solana_genesis::{ + genesis_accounts::add_genesis_accounts, Base64Account, Base64ValidatorAccount, + }, solana_ledger::{blockstore::create_new_ledger, blockstore_options::LedgerColumnOptions}, solana_rpc_client::rpc_client::RpcClient, solana_rpc_client_api::request::MAX_MULTIPLE_ACCOUNTS, @@ -113,6 +115,62 @@ pub fn load_genesis_accounts(file: &str, genesis_config: &mut GenesisConfig) -> Ok(lamports) } +pub fn load_validator_accounts( + file: &str, + commission: u8, + rent: &Rent, + genesis_config: &mut GenesisConfig, +) -> io::Result<()> { + let accounts_file = File::open(file)?; + let validator_genesis_accounts: Vec = + serde_yaml::from_reader(accounts_file) + .map_err(|err| io::Error::new(io::ErrorKind::Other, format!("{err:?}")))?; + + for account_details in validator_genesis_accounts { + let pubkeys = [ + pubkey_from_str(account_details.identity_account.as_str()).map_err(|err| { + io::Error::new( + io::ErrorKind::Other, + format!( + "Invalid pubkey/keypair {}: {:?}", + account_details.identity_account, err + ), + ) + })?, + pubkey_from_str(account_details.vote_account.as_str()).map_err(|err| { + io::Error::new( + io::ErrorKind::Other, + format!( + "Invalid pubkey/keypair {}: {:?}", + account_details.vote_account, err + ), + ) + })?, + pubkey_from_str(account_details.stake_account.as_str()).map_err(|err| { + io::Error::new( + io::ErrorKind::Other, + format!( + "Invalid pubkey/keypair {}: {:?}", + account_details.stake_account, err + ), + ) + })?, + ]; + + add_validator_accounts( + genesis_config, + &mut pubkeys.iter(), + account_details.balance_lamports, + account_details.stake_lamports, + commission, + rent, + None, + ); + } + + Ok(()) +} + fn check_rpc_genesis_hash( cluster_type: &ClusterType, rpc_client: &RpcClient, @@ -177,25 +235,6 @@ fn features_to_deactivate_for_cluster( Ok(features_to_deactivate) } -fn validate_pubkeys(matches: &ArgMatches, arg_name: &str) -> Vec { - // This method will only return an empty vector in the case where `--validator` is not provided - // Clap will enforce that `--bootstrap` flag is provided - let pubkeys = pubkeys_of(matches, arg_name).unwrap_or_default(); - if !pubkeys.is_empty() { - assert_eq!(pubkeys.len() % 3, 0); - - // Ensure there are no duplicated pubkeys in the list - let mut v = pubkeys.clone(); - v.sort(); - v.dedup(); - if v.len() != pubkeys.len() { - eprintln!("Error: --{arg_name} pubkeys cannot be duplicated"); - process::exit(1); - } - } - pubkeys -} - fn add_validator_accounts( genesis_config: &mut GenesisConfig, pubkeys_iter: &mut Iter, @@ -235,7 +274,6 @@ fn add_validator_accounts( stake_lamports, ), ); - genesis_config.add_account(*vote_pubkey, vote_account); } } @@ -278,9 +316,6 @@ fn main() -> Result<(), Box> { .max(rent.minimum_balance(StakeStateV2::size_of())) .to_string(); - let default_validator_lamports = default_bootstrap_validator_lamports; - let default_validator_stake_lamports = default_bootstrap_validator_stake_lamports; - let default_target_tick_duration = PohConfig::default().target_tick_duration; let default_ticks_per_slot = &clock::DEFAULT_TICKS_PER_SLOT.to_string(); let default_cluster_type = "mainnet-beta"; @@ -309,17 +344,6 @@ fn main() -> Result<(), Box> { .required(true) .help("The bootstrap validator's identity, vote and stake pubkeys"), ) - .arg( - Arg::with_name("internal_validator") - .short("v") - .long("internal-validator") - .value_name("IDENTITY_PUBKEY VOTE_PUBKEY STAKE_PUBKEY") - .takes_value(true) - .validator(is_pubkey_or_keypair) - .number_of_values(3) - .multiple(true) - .help("An internal (non bootstrap) validator's identity, vote, and stake pubkeys and its stake in lamports"), - ) .arg( Arg::with_name("ledger_path") .short("l") @@ -375,22 +399,6 @@ fn main() -> Result<(), Box> { .default_value(default_bootstrap_validator_stake_lamports) .help("Number of lamports to assign to the bootstrap validator's stake account"), ) - .arg( - Arg::with_name("internal_validator_lamports") - .long("internal-validator-lamports") - .value_name("LAMPORTS") - .takes_value(true) - .default_value(default_validator_lamports) - .help("Number of lamports to assign to the internal (non bootstrap) validators"), - ) - .arg( - Arg::with_name("internal_validator_stake_lamports") - .long("internal-validator-stake-lamports") - .value_name("LAMPORTS") - .takes_value(true) - .default_value(default_validator_stake_lamports) - .help("Number of lamports to assign to the internal (non bootstrap) validators' stake account"), - ) .arg( Arg::with_name("target_lamports_per_signature") .long("target-lamports-per-signature") @@ -516,6 +524,14 @@ fn main() -> Result<(), Box> { .multiple(true) .help("The location of pubkey for primordial accounts and balance"), ) + .arg( + Arg::with_name("validator_accounts_file") + .long("validator-accounts-file") + .value_name("FILENAME") + .takes_value(true) + .multiple(true) + .help("The location of identity, vote, and stake pubkeys and balances for validator accounts"), + ) .arg( Arg::with_name("cluster_type") .long("cluster-type") @@ -610,8 +626,19 @@ fn main() -> Result<(), Box> { } } - let bootstrap_validator_pubkeys = validate_pubkeys(&matches, "bootstrap_validator"); - let internal_validator_pubkeys = validate_pubkeys(&matches, "internal_validator"); + let bootstrap_validator_pubkeys = pubkeys_of(&matches, "bootstrap_validator").unwrap(); + assert_eq!(bootstrap_validator_pubkeys.len() % 3, 0); + + // Ensure there are no duplicated pubkeys in the --bootstrap-validator list + { + let mut v = bootstrap_validator_pubkeys.clone(); + v.sort(); + v.dedup(); + if v.len() != bootstrap_validator_pubkeys.len() { + eprintln!("Error: --bootstrap-validator pubkeys cannot be duplicated"); + process::exit(1); + } + } let bootstrap_validator_lamports = value_t_or_exit!(matches, "bootstrap_validator_lamports", u64); @@ -622,13 +649,6 @@ fn main() -> Result<(), Box> { rent.minimum_balance(StakeStateV2::size_of()), )?; - let internal_validator_lamports = value_t_or_exit!(matches, "internal_validator_lamports", u64); - let internal_validator_stake_lamports = rent_exempt_check( - &matches, - "internal_validator_stake_lamports", - rent.minimum_balance(StakeStateV2::size_of()), - )?; - let bootstrap_stake_authorized_pubkey = pubkey_of(&matches, "bootstrap_stake_authorized_pubkey"); let faucet_lamports = value_t!(matches, "faucet_lamports", u64).unwrap_or(0); @@ -729,16 +749,6 @@ fn main() -> Result<(), Box> { bootstrap_stake_authorized_pubkey.as_ref(), ); - add_validator_accounts( - &mut genesis_config, - &mut internal_validator_pubkeys.iter(), - internal_validator_lamports, - internal_validator_stake_lamports, - commission, - &rent, - None, - ); - if let Some(creation_time) = unix_timestamp_from_rfc3339_datetime(&matches, "creation_time") { genesis_config.creation_time = creation_time; } @@ -765,6 +775,12 @@ fn main() -> Result<(), Box> { } } + if let Some(files) = matches.values_of("validator_accounts_file") { + for file in files { + load_validator_accounts(file, commission, &rent, &mut genesis_config)?; + } + } + let max_genesis_archive_unpacked_size = value_t_or_exit!(matches, "max_genesis_archive_unpacked_size", u64); @@ -884,7 +900,7 @@ fn main() -> Result<(), Box> { mod tests { use { super::*, - solana_sdk::genesis_config::GenesisConfig, + solana_sdk::{borsh1, genesis_config::GenesisConfig, stake}, std::{collections::HashMap, fs::remove_file, io::Write, path::Path}, }; @@ -1224,4 +1240,125 @@ mod tests { assert_eq!(genesis_config.accounts.len(), 3); } + + #[test] + fn test_append_validator_accounts_to_genesis() { + // Test invalid file returns error + assert!(load_validator_accounts( + "unknownfile", + 100, + &Rent::default(), + &mut GenesisConfig::default() + ) + .is_err()); + + let mut genesis_config = GenesisConfig::default(); + + let validator_accounts = vec![ + Base64ValidatorAccount { + identity_account: solana_sdk::pubkey::new_rand().to_string(), + vote_account: solana_sdk::pubkey::new_rand().to_string(), + stake_account: solana_sdk::pubkey::new_rand().to_string(), + balance_lamports: 100000000000, + stake_lamports: 10000000000, + }, + Base64ValidatorAccount { + identity_account: solana_sdk::pubkey::new_rand().to_string(), + vote_account: solana_sdk::pubkey::new_rand().to_string(), + stake_account: solana_sdk::pubkey::new_rand().to_string(), + balance_lamports: 200000000000, + stake_lamports: 20000000000, + }, + Base64ValidatorAccount { + identity_account: solana_sdk::pubkey::new_rand().to_string(), + vote_account: solana_sdk::pubkey::new_rand().to_string(), + stake_account: solana_sdk::pubkey::new_rand().to_string(), + balance_lamports: 300000000000, + stake_lamports: 30000000000, + }, + ]; + + let serialized = serde_yaml::to_string(&validator_accounts).unwrap(); + + // Write the YAML string to a file + let path = Path::new("test_append_validator_accounts_to_genesis.yml"); + let mut file = File::create(path).unwrap(); + file.write_all(b"---\n").unwrap(); + file.write_all(serialized.as_bytes()).unwrap(); + + // Load the validator accounts from the file + load_validator_accounts( + "test_append_validator_accounts_to_genesis.yml", + 100, + &Rent::default(), + &mut genesis_config, + ) + .expect("Failed to load validator accounts"); + + remove_file(path).unwrap(); + + let accounts_per_validator = 3; + let expected_accounts_len = validator_accounts.len() * accounts_per_validator; + { + assert_eq!(genesis_config.accounts.len(), expected_accounts_len); + + // Test account data matches + for b64_account in validator_accounts.iter() { + // Check Identity + let identity_pk = b64_account.identity_account.parse().unwrap(); + assert_eq!( + system_program::id(), + genesis_config.accounts[&identity_pk].owner + ); + assert_eq!( + b64_account.balance_lamports, + genesis_config.accounts[&identity_pk].lamports + ); + + // Check vote account + let vote_pk = b64_account.vote_account.parse().unwrap(); + let vote_data = genesis_config.accounts[&vote_pk].data.clone(); + let vote_state = VoteState::deserialize(&vote_data).unwrap(); + assert_eq!(vote_state.node_pubkey, identity_pk); + assert_eq!(vote_state.authorized_withdrawer, identity_pk); + let authorized_voters = vote_state.authorized_voters(); + assert_eq!(authorized_voters.first().unwrap().1, &identity_pk); + + // check stake account + let stake_pk = b64_account.stake_account.parse().unwrap(); + assert_eq!( + b64_account.stake_lamports, + genesis_config.accounts[&stake_pk].lamports + ); + + let stake_data = genesis_config.accounts[&stake_pk].data.clone(); + let stake_state = + borsh1::try_from_slice_unchecked::(&stake_data).unwrap(); + assert!( + matches!(stake_state, StakeStateV2::Stake(_, _, _)), + "Expected StakeStateV2::Stake variant" + ); + + if let StakeStateV2::Stake(meta, stake, stake_flags) = stake_state { + assert_eq!(meta.authorized.staker, identity_pk); + assert_eq!(meta.authorized.withdrawer, identity_pk); + + assert_eq!(stake.delegation.voter_pubkey, vote_pk); + let stake_account = AccountSharedData::new( + b64_account.stake_lamports, + StakeStateV2::size_of(), + &solana_stake_program::id(), + ); + let rent_exempt_reserve = + &Rent::default().minimum_balance(stake_account.data().len()); + assert_eq!( + stake.delegation.stake, + b64_account.stake_lamports - rent_exempt_reserve + ); + + assert_eq!(stake_flags, stake::stake_flags::StakeFlags::empty()); + } + } + } + } }