From bdf9300bf06e7963e857fa0a7477ad45fe837eb8 Mon Sep 17 00:00:00 2001 From: okjodom Date: Sun, 5 Nov 2023 06:32:21 -0800 Subject: [PATCH] sim-cli: source simulation file from data directory Adds a simln data-dir cli param, but maintains the default value to cwd Sources the simulation file from this data dir Prompts the user to select a simulatin file if the default/configured file is not found --- .gitignore | 2 +- Cargo.lock | 79 +++++++++++++++++++++++++++++++++++++++++-- sim-cli/Cargo.toml | 1 + sim-cli/src/main.rs | 81 ++++++++++++++++++++++++++++++++++++++++++--- sim-lib/src/lib.rs | 57 ++++++++++++++++--------------- 5 files changed, 183 insertions(+), 37 deletions(-) diff --git a/.gitignore b/.gitignore index e99d25e7..5f974d65 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,4 @@ sim.json package-lock.json activity-generator/releases/* .DS_Store -*simulation_*.csv \ No newline at end of file +/results \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 9901124a..21aa1a41 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -344,6 +344,19 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "console" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "unicode-width", + "windows-sys 0.45.0", +] + [[package]] name = "constant_time_eq" version = "0.1.5" @@ -438,6 +451,19 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "dialoguer" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "658bce805d770f407bc62102fca7c2c64ceef2fbcb2b8bd19d2765ce093980de" +dependencies = [ + "console", + "shell-words", + "tempfile", + "thiserror", + "zeroize", +] + [[package]] name = "dirs" version = "1.0.5" @@ -455,6 +481,12 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "equivalent" version = "1.0.1" @@ -1065,7 +1097,7 @@ dependencies = [ "libc", "redox_syscall 0.4.1", "smallvec", - "windows-targets", + "windows-targets 0.48.5", ] [[package]] @@ -1664,6 +1696,12 @@ dependencies = [ "serde", ] +[[package]] +name = "shell-words" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" + [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -1681,6 +1719,7 @@ dependencies = [ "bitcoin 0.30.1", "clap", "ctrlc", + "dialoguer", "log", "serde", "serde_json", @@ -2167,6 +2206,12 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "unicode-width" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" + [[package]] name = "untrusted" version = "0.7.1" @@ -2329,13 +2374,37 @@ dependencies = [ "windows_x86_64_msvc 0.42.2", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", ] [[package]] @@ -2445,3 +2514,9 @@ checksum = "a3b801d0e0a6726477cc207f60162da452f3a95adb368399bef20a946e06f65c" dependencies = [ "memchr", ] + +[[package]] +name = "zeroize" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" diff --git a/sim-cli/Cargo.toml b/sim-cli/Cargo.toml index 6ee311ae..67a6f41b 100644 --- a/sim-cli/Cargo.toml +++ b/sim-cli/Cargo.toml @@ -11,6 +11,7 @@ Instantly simulate real-world Lightning network activity [dependencies] anyhow = { version = "1.0.69", features = ["backtrace"] } clap = { version = "4.1.6", features = ["derive", "env", "std", "help", "usage", "error-context", "suggestions"], default-features = false } +dialoguer = "0.11.0" log = "0.4.20" serde = "1.0.183" serde_json = "1.0.104" diff --git a/sim-cli/src/main.rs b/sim-cli/src/main.rs index 03ec6db0..27de80cc 100644 --- a/sim-cli/src/main.rs +++ b/sim-cli/src/main.rs @@ -10,10 +10,16 @@ use clap::Parser; use log::LevelFilter; use sim_lib::{ cln::ClnNode, lnd::LndNode, ActivityDefinition, LightningError, LightningNode, NodeConnection, - NodeId, SimParams, Simulation, + NodeId, SimParams, Simulation, WriteResults, }; use simple_logger::SimpleLogger; +/// The default directory where the simulation files are stored and where the results will be written to. +pub const DEFAULT_DATA_DIR: &str = "."; + +/// The default simulation file to be used by the simulator. +pub const DEFAULT_SIM_FILE: &str = "sim.json"; + /// The default expected payment amount for the simulation, around ~$10 at the time of writing. pub const EXPECTED_PAYMENT_AMOUNT: u64 = 3_800_000; @@ -42,8 +48,12 @@ fn deserialize_f64_greater_than_zero(x: String) -> Result { #[derive(Parser)] #[command(version, about)] struct Cli { + /// Path to a directory containing simulation files, and where simulation results will be stored + #[clap(long, short, env = "SIM_LN_DATA_DIR", default_value = DEFAULT_DATA_DIR)] + data_dir: PathBuf, /// Path to the simulation file to be used by the simulator - #[clap(index = 1)] + /// This can either be an absolute path, or relative path with respect to data_dir + #[clap(long, short, default_value = DEFAULT_SIM_FILE)] sim_file: PathBuf, /// Total time the simulator will be running #[clap(long, short)] @@ -77,8 +87,9 @@ async fn main() -> anyhow::Result<()> { .init() .unwrap(); + let sim_path = read_sim_path(cli.data_dir.clone(), cli.sim_file).await?; let SimParams { nodes, activity } = - serde_json::from_str(&std::fs::read_to_string(cli.sim_file)?) + serde_json::from_str(&std::fs::read_to_string(sim_path)?) .map_err(|e| anyhow!("Could not deserialize node connection data or activity description from simulation file (line {}, col {}).", e.line(), e.column()))?; let mut clients: HashMap>> = HashMap::new(); @@ -179,14 +190,22 @@ async fn main() -> anyhow::Result<()> { }); } + let write_results = if !cli.no_results { + Some(WriteResults { + results_dir: mkdir(cli.data_dir.join("results")).await?, + batch_size: cli.print_batch_size, + }) + } else { + None + }; + let sim = Simulation::new( clients, validated_activities, cli.total_time, - cli.print_batch_size, cli.expected_pmt_amt, cli.capacity_multiplier, - cli.no_results, + write_results, ); let sim2 = sim.clone(); @@ -199,3 +218,55 @@ async fn main() -> anyhow::Result<()> { Ok(()) } + +async fn read_sim_path(data_dir: PathBuf, sim_file: PathBuf) -> anyhow::Result { + let sim_path = if sim_file.is_relative() { + data_dir.join(sim_file) + } else { + sim_file + }; + + if sim_path.exists() { + Ok(sim_path) + } else { + log::info!("Simulation file '{}' does not exist", sim_path.display()); + select_sim_file(data_dir).await + } +} + +async fn select_sim_file(data_dir: PathBuf) -> anyhow::Result { + let sim_files = std::fs::read_dir(data_dir.clone())? + .filter_map(|f| { + f.ok().and_then(|f| { + if f.path().extension()?.to_str()? == "json" { + f.file_name().into_string().ok() + } else { + None + } + }) + }) + .collect::>(); + + if sim_files.is_empty() { + anyhow::bail!( + "no simulation files found in {}", + data_dir.canonicalize()?.display() + ); + } + + let selection = dialoguer::Select::new() + .with_prompt(format!( + "Select a simulation file. Found these in {}", + data_dir.canonicalize()?.display() + )) + .items(&sim_files) + .default(0) + .interact()?; + + Ok(data_dir.join(sim_files[selection].clone())) +} + +async fn mkdir(dir: PathBuf) -> anyhow::Result { + tokio::fs::create_dir_all(&dir).await?; + Ok(dir) +} diff --git a/sim-lib/src/lib.rs b/sim-lib/src/lib.rs index d18e7cd7..51f9e893 100644 --- a/sim-lib/src/lib.rs +++ b/sim-lib/src/lib.rs @@ -9,6 +9,7 @@ use serde::{Deserialize, Serialize}; use std::collections::HashSet; use std::fmt::{Display, Formatter}; use std::marker::Send; +use std::path::PathBuf; use std::time::UNIX_EPOCH; use std::{collections::HashMap, sync::Arc, time::SystemTime}; use thiserror::Error; @@ -326,15 +327,21 @@ pub struct Simulation { shutdown_listener: Listener, // Total simulation time. The simulation will run forever if undefined. total_time: Option, - /// The number of activity results to batch before printing in CSV. - print_batch_size: u32, /// The expected payment size for the network. expected_payment_msat: u64, /// The number of times that the network sends its total capacity in a month of operation when generating random /// activity. activity_multiplier: f64, - /// Whether we want the simulation not to produce and result file. Useful for developing, defaults to false. - no_results: bool, + /// Configurations for printing results to CSV. Results are not written if this option is None. + write_results: Option, +} + +#[derive(Clone)] +pub struct WriteResults { + /// Data directory where CSV result files are written. + pub results_dir: PathBuf, + /// The number of activity results to batch before printing in CSV. + pub batch_size: u32, } impl Simulation { @@ -342,10 +349,9 @@ impl Simulation { nodes: HashMap>>, activity: Vec, total_time: Option, - print_batch_size: u32, expected_payment_msat: u64, activity_multiplier: f64, - no_results: bool, + write_results: Option, ) -> Self { let (shutdown_trigger, shutdown_listener) = triggered::trigger(); Self { @@ -354,10 +360,9 @@ impl Simulation { shutdown_trigger, shutdown_listener, total_time: total_time.map(|x| Duration::from_secs(x as u64)), - print_batch_size, expected_payment_msat, activity_multiplier, - no_results, + write_results, } } @@ -561,8 +566,7 @@ impl Simulation { result_logger, results_receiver, listener, - self.print_batch_size, - self.no_results, + self.write_results.clone(), )); log::debug!("Simulator data collection set up."); } @@ -886,14 +890,11 @@ async fn consume_simulation_results( logger: Arc>, receiver: Receiver<(Payment, PaymentResult)>, listener: Listener, - print_batch_size: u32, - no_results: bool, + write_results: Option, ) { log::debug!("Simulation results consumer started."); - if let Err(e) = - write_payment_results(logger, receiver, listener, print_batch_size, no_results).await - { + if let Err(e) = write_payment_results(logger, receiver, listener, write_results).await { log::error!("Error while reporting payment results: {:?}.", e); } @@ -904,20 +905,19 @@ async fn write_payment_results( logger: Arc>, mut receiver: Receiver<(Payment, PaymentResult)>, listener: Listener, - print_batch_size: u32, - no_results: bool, + write_results: Option, ) -> Result<(), SimulationError> { - let mut writer = if !no_results { - Some(WriterBuilder::new().from_path(format!( + let mut writer = write_results.and_then(|write_result| { + let file = write_result.results_dir.join(format!( "simulation_{:?}.csv", SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .unwrap() .as_secs() - ))?) - } else { - None - }; + )); + let writer = WriterBuilder::new().from_path(file).ok()?; + Some((writer, write_result.batch_size)) + }); let mut counter = 1; loop { @@ -925,7 +925,7 @@ async fn write_payment_results( biased; _ = listener.clone() => { log::debug!("Simulation results consumer received shutdown signal."); - return writer.map_or(Ok(()), |ref mut w| w.flush().map_err(|_| SimulationError::FileError)) + return writer.map_or(Ok(()), |(ref mut w, _)| w.flush().map_err(|_| SimulationError::FileError)) }, payment_report = receiver.recv() => { match payment_report { @@ -933,19 +933,18 @@ async fn write_payment_results( logger.lock().await.report_result(&details, &result); log::trace!("Resolved dispatched payment: {} with: {}.", details, result); - if let Some(ref mut w) = writer { + if let Some((ref mut w, batch_size)) = writer { w.serialize((details, result)).map_err(|e| { let _ = w.flush(); SimulationError::CsvError(e) })?; - - counter = counter % print_batch_size + 1; - if print_batch_size == counter { + counter = counter % batch_size + 1; + if batch_size == counter { w.flush().map_err(|_| SimulationError::FileError)?; } } }, - None => return writer.map(|ref mut w| w.flush().map_err(|_| SimulationError::FileError)).unwrap_or(Ok(())), + None => return writer.map_or(Ok(()), |(ref mut w, _)| w.flush().map_err(|_| SimulationError::FileError)), } } }