From 720710cdd2388f699bee8954ee606bfe3ebf492c Mon Sep 17 00:00:00 2001 From: Kould <2435992353@qq.com> Date: Thu, 14 Nov 2024 02:50:41 +0800 Subject: [PATCH] fix: test proportion of each scenario of `TPCC` and add more indicators (#244) --- README.md | 12 ---- tpcc/src/main.rs | 173 +++++++++++++++++++++++++++++++++++----------- tpcc/src/utils.rs | 68 ++++++++++++++++++ 3 files changed, 201 insertions(+), 52 deletions(-) create mode 100644 tpcc/src/utils.rs diff --git a/README.md b/README.md index 0e492d09..4f7577fe 100755 --- a/README.md +++ b/README.md @@ -57,18 +57,6 @@ let tuples = fnck_sql.run("select * from t1")?; ### TPCC run `cargo run -p tpcc --release` to run tpcc -- CPU: i9-13900HX -- Memory: 32.0 GB -- SSD: YMTC PC411-1024GB-B -```shell -<90th Percentile RT (MaxRT)> - New-Order : 0.882 (0.947) - Payment : 0.080 (0.095) -Order-Status : 0.235 (0.255) - Delivery : 5.386 (5.658) - Stock-Level : 0.001 (0.002) -``` - #### PG Wire Service run `cargo run --features="net"` to start server ![start](./static/images/start.gif) diff --git a/tpcc/src/main.rs b/tpcc/src/main.rs index 03881bdf..2d9091e1 100644 --- a/tpcc/src/main.rs +++ b/tpcc/src/main.rs @@ -5,6 +5,7 @@ use crate::order_stat::OrderStatTest; use crate::payment::PaymentTest; use crate::rt_hist::RtHist; use crate::slev::SlevTest; +use crate::utils::SeqGen; use clap::Parser; use fnck_sql::db::{DBTransaction, DataBaseBuilder}; use fnck_sql::errors::DatabaseError; @@ -20,8 +21,16 @@ mod order_stat; mod payment; mod rt_hist; mod slev; +mod utils; pub(crate) const ALLOW_MULTI_WAREHOUSE_TX: bool = true; +pub(crate) const RT_LIMITS: [Duration; 5] = [ + Duration::from_millis(500), + Duration::from_millis(500), + Duration::from_millis(500), + Duration::from_secs(8), + Duration::from_secs(2), +]; pub(crate) trait TpccTransaction { type Args; @@ -54,7 +63,7 @@ struct Args { path: String, #[clap(long, default_value = "5")] max_retry: usize, - #[clap(long, default_value = "1080")] + #[clap(long, default_value = "720")] measure_time: u64, #[clap(long, default_value = "1")] num_ware: usize, @@ -72,6 +81,9 @@ fn main() -> Result<(), TpccError> { Load::load_ord(&mut rng, &database, args.num_ware)?; let mut rt_hist = RtHist::new(); + let mut success = [0usize; 5]; + let mut late = [0usize; 5]; + let mut failure = [0usize; 5]; let tests = vec![ Box::new(NewOrdTest) as Box>, Box::new(PaymentTest), @@ -81,64 +93,145 @@ fn main() -> Result<(), TpccError> { ]; let tpcc_args = TpccArgs { joins: args.joins }; - let tpcc_start = Instant::now(); let duration = Duration::new(args.measure_time, 0); let mut round_count = 0; + let mut seq_gen = SeqGen::new(10, 10, 1, 1, 1); + let tpcc_start = Instant::now(); while tpcc_start.elapsed() < duration { - for (i, tpcc_test) in tests.iter().enumerate() { - let mut is_succeed = false; - for j in 0..args.max_retry + 1 { - let transaction_start = Instant::now(); - let mut tx = database.new_transaction()?; - - if let Err(err) = - tpcc_test.do_transaction(&mut rng, &mut tx, args.num_ware, &tpcc_args) - { - eprintln!( - "[{}] Error while doing transaction: {}", - tpcc_test.name(), - err - ); + let i = seq_gen.get(); + let tpcc_test = &tests[i]; + + let mut is_succeed = false; + for j in 0..args.max_retry + 1 { + let transaction_start = Instant::now(); + let mut tx = database.new_transaction()?; + + if let Err(err) = tpcc_test.do_transaction(&mut rng, &mut tx, args.num_ware, &tpcc_args) + { + failure[i] += 1; + eprintln!( + "[{}] Error while doing transaction: {}", + tpcc_test.name(), + err + ); + } else { + let rt = transaction_start.elapsed(); + rt_hist.hist_inc(i, rt); + is_succeed = true; + + if rt <= RT_LIMITS[i] { + success[i] += 1; } else { - rt_hist.hist_inc(i, transaction_start.elapsed()); - is_succeed = true; - break; - } - if j < args.max_retry { - println!("[{}] Retry for the {}th time", tpcc_test.name(), j + 1); + late[i] += 1; } + break; } - if !is_succeed { - return Err(TpccError::MaxRetry); + if j < args.max_retry { + println!("[{}] Retry for the {}th time", tpcc_test.name(), j + 1); } } - if round_count != 0 && round_count % 4 == 0 { + if !is_succeed { + return Err(TpccError::MaxRetry); + } + if round_count != 0 && round_count % 8 == 0 { println!( - "============ TPCC CheckPoint {} on round {round_count}: ===============", - round_count / 4 + "[TPCC CheckPoint {} on round {round_count}][{}]: 90th Percentile RT: {:.3}", + round_count / 4, + tpcc_test.name(), + rt_hist.hist_ckp(i) ); - for (i, name) in vec![ - "New-Order", - "Payment", - "Order-Status", - "Delivery", - "Stock-Level", - ] - .into_iter() - .enumerate() - { - println!("{name} 90th Percentile RT: {:.3}", rt_hist.hist_ckp(i)); - } - println!("=========================================================="); } round_count += 1; } + let actual_tpcc_time = tpcc_start.elapsed(); + println!("---------------------------------------------------"); + // Raw Results + print_transaction(&success, &late, &failure, |name, success, late, failure| { + println!("|{}| sc: {} lt: {} fl: {}", name, success, late, failure) + }); + println!("in {} sec.", actual_tpcc_time.as_secs()); + println!(" (all must be [OK])"); + println!("[transaction percentage]"); + + let mut j = 0.0; + for i in 0..5 { + j += (success[i] + late[i]) as f64; + } + // Payment + let f = ((success[1] + late[1]) as f64 / j) * 100.0; + print!(" Payment: {:.1}% (>=43.0%)", f); + if f >= 43.0 { + println!(" [Ok]"); + } else { + println!(" [NG]"); + } + // Order-Status + let f = ((success[2] + late[2]) as f64 / j) * 100.0; + print!(" Order-Status: {:.1}% (>=4.0%)", f); + if f >= 4.0 { + println!(" [Ok]"); + } else { + println!(" [NG]"); + } + // Delivery + let f = ((success[3] + late[3]) as f64 / j) * 100.0; + print!(" Order-Status: {:.1}% (>=4.0%)", f); + if f >= 4.0 { + println!(" [Ok]"); + } else { + println!(" [NG]"); + } + // Stock-Level + let f = ((success[4] + late[4]) as f64 / j) * 100.0; + print!(" Order-Status: {:.1}% (>=4.0%)", f); + if f >= 4.0 { + println!(" [Ok]"); + } else { + println!(" [NG]"); + } + println!("[response time (at least 90%% passed)]"); + print_transaction(&success, &late, &failure, |name, success, late, _| { + let f = (success as f64 / (success + late) as f64) * 100.0; + print!(" {}: {:.1}", name, f); + if f >= 90.0 { + println!(" [OK]"); + } else { + println!(" [NG]"); + } + }); + print_transaction(&success, &late, &failure, |name, success, late, _| { + println!(" {} Total: {}", name, success + late) + }); + println!(); rt_hist.hist_report(); + println!(""); + let tpmc = (success[0] + late[0]) as f64 / (actual_tpcc_time.as_secs_f64() / 60.0); + println!("{} Tpmc", tpmc); Ok(()) } +fn print_transaction( + success: &[usize], + late: &[usize], + failure: &[usize], + fn_print: F, +) { + for (i, name) in vec![ + "New-Order", + "Payment", + "Order-Status", + "Delivery", + "Stock-Level", + ] + .into_iter() + .enumerate() + { + fn_print(name, success[i], late[i], failure[i]); + } +} + fn other_ware(rng: &mut ThreadRng, home_ware: usize, num_ware: usize) -> usize { if num_ware == 1 { return home_ware; diff --git a/tpcc/src/utils.rs b/tpcc/src/utils.rs new file mode 100644 index 00000000..6b065713 --- /dev/null +++ b/tpcc/src/utils.rs @@ -0,0 +1,68 @@ +use rand::rngs::ThreadRng; +use rand::Rng; + +pub(crate) struct SeqGen { + no: usize, + py: usize, + os: usize, + dl: usize, + sl: usize, + total: usize, + seq: Vec, + next_num: usize, + rng: ThreadRng, +} + +impl SeqGen { + pub(crate) fn new(n: usize, p: usize, o: usize, d: usize, s: usize) -> Self { + let total = n + p + o + d + s; + + Self { + no: n, + py: p, + os: o, + dl: d, + sl: s, + total, + seq: vec![0; total], + next_num: 0, + rng: Default::default(), + } + } + + pub(crate) fn get(&mut self) -> usize { + if self.next_num >= self.total { + self.shuffle(); + self.next_num = 0; + } + let pos = self.next_num; + self.next_num += 1; + self.seq[pos] + } + + pub(crate) fn shuffle(&mut self) { + let mut pos = 0; + + let mut fn_init = |len, tx| { + for i in 0..len { + self.seq[pos + i] = tx; + } + pos += len; + }; + fn_init(self.no, 0); + fn_init(self.py, 1); + fn_init(self.os, 2); + fn_init(self.dl, 3); + fn_init(self.sl, 4); + + let mut i = 0; + for j in (0..self.total).rev() { + let rmd = self.rng.gen::() % (j + 1); + let tmp = self.seq[rmd + i]; + self.seq[rmd + i] = self.seq[i]; + self.seq[i] = tmp; + + i += 1; + } + } +}