diff --git a/Cargo.lock b/Cargo.lock index 1bcbf12..31611ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4287,9 +4287,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.197" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" dependencies = [ "serde_derive", ] @@ -4315,9 +4315,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.197" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", @@ -7155,6 +7155,8 @@ dependencies = [ "reqwest", "rustc_version", "rustls", + "serde", + "serde_yaml", "solana-accounts-db", "solana-clap-v3-utils", "solana-core", diff --git a/Cargo.toml b/Cargo.toml index 9843f97..a36e78d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,8 @@ openssl = "0.10.66" rand = "0.8.5" reqwest = { version = "0.11.23", features = ["blocking", "brotli", "deflate", "gzip", "rustls-tls", "json"] } rustls = { version = "0.21.11", default-features = false, features = ["quic"] } +serde = "1.0.208" +serde_yaml = "0.9.34" solana-accounts-db = "1.18.20" solana-clap-v3-utils = "1.18.20" solana-core = "1.18.20" diff --git a/README.md b/README.md index 6e03013..74a40a1 100644 --- a/README.md +++ b/README.md @@ -90,9 +90,10 @@ cargo run --bin cluster -- --base-image # e.g. ubuntu:20.04 --image-name # e.g. cluster-image # validator config + --skip-primordial-accounts --full-rpc - --internal-node-sol - --internal-node-stake-sol + --internal-node-sol + --internal-node-stake-sol # kubernetes config --cpu-requests --memory-requests @@ -119,8 +120,39 @@ For client Version >= 2.0.0 ``` ## Baking Validator Stakes into Genesis -- By default, validator stakes are baked into genesis on genesis creation. That way when the cluster boots up, all validators will consistently be in the leader schedule. -- If you do not want this and instead want the stake to warm up after deplyoyment, pass in the flag `--skip-primordial-stakes`. +- You can bake validator stakes into gensis on genesis creation by passing in `--validator-balances-file `. This way when the cluster boots up, all validators will consistently be in the leader schedule. In the validator balances file, you can set specific validator balances and stake amounts. +The validator balances file has the following yaml format: +``` +--- +v0: + balances_lamports: + stake_lamports: +v1: + balances_lamports: + stake_lamports: +... +vN: + balances_lamports: + stake_lamports: +``` +^ Note, the file must have the `v0`, `v1`, ..., `vN` format. The number of validators in this file must match `--num-validators ` + +For example, we could create: `validator-balances.yml` and have it look like: +``` +--- +v0: + balance_lamports: 400000000000 + stake_lamports: 40000000000 +v1: + balance_lamports: 200000000000 + stake_lamports: 20000000000 +v2: + balance_lamports: 300000000000 + stake_lamports: 30000000000 +``` + +- If you do not want to bake stakes into genesis and instead want the stake to warm up after deplyoyment, pass in the flag `--skip-primordial-stakes` and leave out `--validator-balances` +- `--internal-node-sol`, `--internal-node-stake-sol`, are `--comission` are only valid with `--skip-primordial-stakes` ## Metrics 1) Setup metrics database: diff --git a/src/genesis.rs b/src/genesis.rs index b91624b..5ec7d42 100644 --- a/src/genesis.rs +++ b/src/genesis.rs @@ -2,12 +2,14 @@ use { crate::{fetch_spl, new_spinner_progress_bar, NodeType, SOLANA_RELEASE, SUN, WRITING}, log::*, rand::Rng, + serde::{Deserialize, Serialize}, solana_core::gen_keys::GenKeys, solana_sdk::{ native_token::sol_to_lamports, - signature::{write_keypair_file, Keypair}, + signature::{write_keypair_file, Keypair, Signer}, }, std::{ + collections::HashMap, error::Error, fs::{File, OpenOptions}, io::{self, BufRead, BufWriter, Read, Write}, @@ -24,6 +26,35 @@ pub const DEFAULT_INTERNAL_NODE_SOL: f64 = 100.0; pub const DEFAULT_BOOTSTRAP_NODE_STAKE_SOL: f64 = 10.0; pub const DEFAULT_BOOTSTRAP_NODE_SOL: f64 = 100.0; pub const DEFAULT_CLIENT_LAMPORTS_PER_SIGNATURE: u64 = 42; +const VALIDATOR_ACCOUNTS_KEYPAIR_COUNT: usize = 3; +const RPC_ACCOUNTS_KEYPAIR_COUNT: usize = 1; + +#[derive(Debug, Deserialize)] +struct ValidatorStakes { + balance_lamports: u64, + stake_lamports: u64, +} + +fn generate_filename(node_type: &NodeType, account_type: &str, index: usize) -> String { + match node_type { + NodeType::Bootstrap => format!("{node_type}/{account_type}.json"), + NodeType::Standard | NodeType::RPC => { + format!("{node_type}-{account_type}-{index}.json") + } + NodeType::Client(_, _) => panic!("Client type not supported"), + } +} + +/// A validator account where the data is encoded as a Base64 string. +/// Includes the vote account and stake account. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ValidatorAccounts { + pub balance_lamports: u64, + pub stake_lamports: u64, + pub identity_account: String, + pub vote_account: String, + pub stake_account: String, +} fn parse_spl_genesis_file( spl_file: &PathBuf, @@ -68,6 +99,7 @@ pub struct GenesisFlags { pub internal_node_sol: f64, pub internal_node_stake_sol: f64, pub skip_primordial_stakes: bool, + pub validator_accounts_file: Option, } fn append_client_accounts_to_file( @@ -97,11 +129,18 @@ fn append_client_accounts_to_file( pub struct Genesis { config_dir: PathBuf, key_generator: GenKeys, + pub validator_stakes_file: Option, + validator_accounts: HashMap, pub flags: GenesisFlags, } impl Genesis { - pub fn new(config_dir: PathBuf, flags: GenesisFlags, retain_previous_genesis: bool) -> Self { + pub fn new( + config_dir: PathBuf, + validator_stakes_file: Option, + flags: GenesisFlags, + retain_previous_genesis: bool, + ) -> Self { // if we are deploying a heterogeneous cluster // all deployments after the first must retain the original genesis directory if !retain_previous_genesis { @@ -116,6 +155,8 @@ impl Genesis { Self { config_dir, key_generator: GenKeys::new(seed), + validator_stakes_file, + validator_accounts: HashMap::default(), flags, } } @@ -149,16 +190,12 @@ impl Genesis { } }; - let account_types: Vec = if let Some(tag) = deployment_tag { - account_types - .into_iter() - .map(|acct| format!("{}-{}", acct, tag)) - .collect() - } else { - account_types + let account_types: Vec = match deployment_tag { + Some(tag) => account_types .into_iter() - .map(|acct| acct.to_string()) - .collect() + .map(|acct| format!("{acct}-{tag}")) + .collect(), + None => account_types.into_iter().map(String::from).collect(), }; let total_accounts_to_generate = number_of_accounts * account_types.len(); @@ -166,35 +203,87 @@ impl Genesis { .key_generator .gen_n_keypairs(total_accounts_to_generate as u64); + if node_type == NodeType::Standard { + self.initialize_validator_accounts(&node_type, &keypairs); + } + self.write_accounts_to_file(&node_type, &account_types, &keypairs)?; + // self.initialize_validator_accounts(&keypairs); Ok(()) } fn write_accounts_to_file( - &self, + &mut self, node_type: &NodeType, account_types: &[String], keypairs: &[Keypair], ) -> Result<(), Box> { - for (i, keypair) in keypairs.iter().enumerate() { - let account_index = i / account_types.len(); - let account = &account_types[i % account_types.len()]; - info!("Account: {account}, node_type: {node_type}"); - let filename = match node_type { - NodeType::Bootstrap => { - format!("{node_type}/{account}.json") + let chunk_size = match node_type { + NodeType::Bootstrap | NodeType::Standard => VALIDATOR_ACCOUNTS_KEYPAIR_COUNT, + NodeType::RPC => RPC_ACCOUNTS_KEYPAIR_COUNT, + NodeType::Client(_, _) => return Err("Client type not supported".into()), + }; + for (i, account_type_keypair) in keypairs.chunks_exact(chunk_size).enumerate() { + match node_type { + NodeType::Bootstrap | NodeType::Standard => { + // Create a filename for each type of account based on node type and index + let identity_filename = + generate_filename(node_type, account_types[0].as_str(), i); + let stake_filename = generate_filename(node_type, account_types[1].as_str(), i); + let vote_filename = generate_filename(node_type, account_types[2].as_str(), i); + + write_keypair_file( + &account_type_keypair[0], + self.config_dir.join(identity_filename), + )?; + write_keypair_file( + &account_type_keypair[1], + self.config_dir.join(vote_filename), + )?; + write_keypair_file( + &account_type_keypair[2], + self.config_dir.join(stake_filename), + )?; } - NodeType::Standard | NodeType::RPC => { - format!("{node_type}-{account}-{account_index}.json") + NodeType::RPC => { + let identity_filename = + generate_filename(node_type, account_types[0].as_str(), i); + write_keypair_file( + &account_type_keypair[0], + self.config_dir.join(identity_filename), + )?; } - NodeType::Client(_, _) => panic!("Client type not supported"), + NodeType::Client(_, _) => return Err("Client type not supported".into()), + } + } + + Ok(()) + } + + fn initialize_validator_accounts(&mut self, node_type: &NodeType, keypairs: &[Keypair]) { + if node_type != &NodeType::Standard { + return; + } + for (i, account_type_keypair) in keypairs + .chunks_exact(VALIDATOR_ACCOUNTS_KEYPAIR_COUNT) + .enumerate() + { + let identity_account = account_type_keypair[0].pubkey().to_string(); + let vote_account = account_type_keypair[1].pubkey().to_string(); + let stake_account = account_type_keypair[2].pubkey().to_string(); + + let validator_account = ValidatorAccounts { + balance_lamports: 0, + stake_lamports: 0, + identity_account, + vote_account, + stake_account, }; - let outfile = self.config_dir.join(&filename); - write_keypair_file(keypair, outfile)?; + let key = format!("v{i}"); + self.validator_accounts.insert(key, validator_account); } - Ok(()) } pub fn create_client_accounts( @@ -284,11 +373,7 @@ impl Genesis { Ok(child) } - fn setup_genesis_flags( - &self, - num_validators: usize, - image_tag: &str, - ) -> Result, Box> { + fn setup_genesis_flags(&self) -> Result, Box> { let mut args = vec![ "--bootstrap-validator-lamports".to_string(), sol_to_lamports(self.flags.bootstrap_validator_sol).to_string(), @@ -349,28 +434,20 @@ impl Genesis { args.push(path); } - if !self.flags.skip_primordial_stakes { - for i in 0..num_validators { - args.push("--internal-validator".to_string()); - for account_type in ["identity", "vote-account", "stake-account"].iter() { - let path = self - .config_dir - .join(format!("validator-{account_type}-{image_tag}-{i}.json")) - .into_os_string() - .into_string() - .map_err(|_| "Failed to convert path to string")?; - args.push(path); - } - } - - // stake delegated from internal_node_sol - let internal_node_lamports = - self.flags.internal_node_sol - self.flags.internal_node_stake_sol; - args.push("--internal-validator-lamports".to_string()); - args.push(sol_to_lamports(internal_node_lamports).to_string()); - - args.push("--internal-validator-stake-lamports".to_string()); - args.push(sol_to_lamports(self.flags.internal_node_stake_sol).to_string()); + if let Some(validator_accounts_file) = &self.flags.validator_accounts_file { + args.push("--validator-accounts-file".to_string()); + args.push( + validator_accounts_file + .clone() + .into_os_string() + .into_string() + .map_err(|err| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("Invalid Unicode data in path: {:?}", err), + ) + })?, + ); } if let Some(slots_per_epoch) = self.flags.slots_per_epoch { @@ -400,10 +477,8 @@ impl Genesis { &mut self, solana_root_path: &Path, exec_path: &Path, - num_validators: usize, - image_tag: &str, ) -> Result<(), Box> { - let mut args = self.setup_genesis_flags(num_validators, image_tag)?; + let mut args = self.setup_genesis_flags()?; let mut spl_args = self.setup_spl_args(solana_root_path).await?; args.append(&mut spl_args); @@ -505,4 +580,59 @@ impl Genesis { Ok(bank_hash) } + + pub fn load_validator_genesis_stakes_from_file(&mut self) -> io::Result<()> { + let validator_stakes_file = match &self.validator_stakes_file { + Some(file) => file, + None => { + warn!("validator_stakes_file is None"); + return Ok(()); + } + }; + let file = File::open(validator_stakes_file)?; + let validator_stakes: HashMap = serde_yaml::from_reader(file) + .map_err(|err| io::Error::new(io::ErrorKind::Other, format!("{err:?}")))?; + + if validator_stakes.len() != self.validator_accounts.len() { + return Err(io::Error::new( + io::ErrorKind::Other, + format!( + "Number of validator stakes ({}) does not match number of validator accounts ({})", + validator_stakes.len(), self.validator_accounts.len() + ), + )); + } + + // match `validator_stakes` with corresponding `ValidatorAccounts` and update balance and stake + for (key, stake) in validator_stakes { + if let Some(validator_account) = self.validator_accounts.get_mut(&key) { + validator_account.balance_lamports = stake.balance_lamports; + validator_account.stake_lamports = stake.stake_lamports; + } else { + return Err(io::Error::new( + io::ErrorKind::Other, + format!("Validator account for key '{key}' not found"), + )); + } + } + + self.write_validator_genesis_accouts_to_file()?; + Ok(()) + } + + fn write_validator_genesis_accouts_to_file(&mut self) -> std::io::Result<()> { + // get ValidatorAccounts vec to write to file for solana-genesis + let validator_accounts_vec: Vec = + self.validator_accounts.values().cloned().collect(); + let output_file = self.config_dir.join("validator-genesis-accounts.yml"); + self.flags.validator_accounts_file = Some(output_file.clone()); + + // write ValidatorAccouns to yaml file for solana-genesis + let file = File::create(&output_file)?; + serde_yaml::to_writer(file, &validator_accounts_vec) + .map_err(|err| io::Error::new(io::ErrorKind::Other, format!("{err:?}")))?; + + info!("Validator genesis accounts successfully written to {output_file:?}"); + Ok(()) + } } diff --git a/src/kubernetes.rs b/src/kubernetes.rs index 2262a3a..32b4c80 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -569,14 +569,16 @@ impl<'a> Kubernetes<'a> { Self::generate_full_rpc_flags(&mut flags); } - flags.push("--internal-node-stake-sol".to_string()); - flags.push(self.validator_config.internal_node_stake_sol.to_string()); + if self.validator_config.skip_primordial_stakes { + flags.push("--internal-node-stake-sol".to_string()); + flags.push(self.validator_config.internal_node_stake_sol.to_string()); - flags.push("--commission".to_string()); - flags.push(self.validator_config.commission.to_string()); + flags.push("--internal-node-sol".to_string()); + flags.push(self.validator_config.internal_node_sol.to_string()); - flags.push("--internal-node-sol".to_string()); - flags.push(self.validator_config.internal_node_sol.to_string()); + flags.push("--commission".to_string()); + flags.push(self.validator_config.commission.to_string()); + } if let Some(shred_version) = self.validator_config.shred_version { flags.push("--expected-shred-version".to_string()); diff --git a/src/main.rs b/src/main.rs index 2bdcd49..ac69078 100644 --- a/src/main.rs +++ b/src/main.rs @@ -238,6 +238,7 @@ fn parse_matches() -> clap::ArgMatches { .long("internal-node-sol") .takes_value(true) .default_value(&DEFAULT_INTERNAL_NODE_SOL.to_string()) + .conflicts_with("validator_balances_file") .help("Amount to fund internal nodes in genesis"), ) .arg( @@ -245,27 +246,42 @@ fn parse_matches() -> clap::ArgMatches { .long("internal-node-stake-sol") .takes_value(true) .default_value(&DEFAULT_INTERNAL_NODE_STAKE_SOL.to_string()) + .conflicts_with("validator_balances_file") .help("Amount to stake internal nodes (Sol) in genesis"), ) - .arg( - Arg::with_name("skip_primordial_stakes") - .long("skip-primordial-stakes") - .help("Do not bake validator stake accounts into genesis. \ - Validators will be funded and staked after the cluster boots. \ - This will result in several epochs for all of the stake to warm up"), - ) .arg( Arg::with_name("commission") .long("commission") .value_name("PERCENTAGE") .takes_value(true) .default_value("100") + .conflicts_with("validator_balances_file") .help("The commission taken by nodes on staking rewards (0-100)") ) + .arg( + Arg::with_name("skip_primordial_stakes") + .long("skip-primordial-stakes") + .help("Do not bake validator stake accounts into genesis. + Validators will be funded and staked after the cluster boots. + This will result in several epochs for all of the stake to warm up"), + ) + .arg( + Arg::with_name("validator_balances_file") + .long("validator-balances-file") + .value_name("FILENAME") + .takes_value(true) + .help("The location of validator balances and stake balances for validator accounts"), + ) + .group( + ArgGroup::with_name("validation_stake_config") + .args(&["skip_primordial_stakes", "validator_balances_file"]) + .required(true) // Only one of these args must be present + .multiple(false), // Passing both in will result in an error + ) .arg( Arg::with_name("no_restart") .long("no-restart") - .help("Validator config. If set, validators will not restart after \ + .help("Validator config. If set, validators will not restart after exiting for any reason."), ) //RPC config @@ -604,6 +620,7 @@ async fn main() -> Result<(), Box> { internal_node_sol, internal_node_stake_sol, skip_primordial_stakes, + validator_accounts_file: None, }; let limit_ledger_size = value_t_or_exit!(matches, "limit_ledger_size", u64); @@ -677,6 +694,9 @@ async fn main() -> Result<(), Box> { let retain_previous_genesis = !deploy_bootstrap_validator; let mut genesis = Genesis::new( config_directory.clone(), + matches + .value_of("validator_balances_file") + .map(PathBuf::from), genesis_flags, retain_previous_genesis, ); @@ -695,14 +715,13 @@ async fn main() -> Result<(), Box> { genesis.generate_accounts(NodeType::Bootstrap, 1, None)?; info!("Generated bootstrap account"); + if genesis.validator_stakes_file.is_some() { + genesis.load_validator_genesis_stakes_from_file()?; + } + // creates genesis and writes to binary file genesis - .generate( - cluster_data_root.get_root_path(), - &exec_path, - num_validators, - &image_tag, - ) + .generate(cluster_data_root.get_root_path(), &exec_path) .await?; info!("Genesis created"); diff --git a/src/startup_scripts.rs b/src/startup_scripts.rs index 134dfb2..4e9f7ef 100644 --- a/src/startup_scripts.rs +++ b/src/startup_scripts.rs @@ -597,7 +597,7 @@ run_delegate_stake() { fi fi echo "created stake account" - + if [ "$stake_account_already_exists" != true ]; then echo "stake account does not exist. so lets deligate" if ! run_solana_command "solana delegate-stake validator-accounts/stake.json validator-accounts/vote.json --force -k $IDENTITY_FILE" "Delegate Stake"; then