Mg/setup measurement error (#11)

* Batch setups and save in ram to reduce measurement error on benches with a setup, fix lints

* Modernize deny

* Rename measurement_time, remove risk of mem explosion

deny.toml

@@ -1,7 +1,4 @@
 [advisories]
-vulnerability = "deny"
-unmaintained = "warn"
-yanked = "deny"
 ignore = []
 
 [bans]
@@ -23,11 +20,8 @@ github = []
 
 [licenses]
 private = { ignore = true, registries = ["embark"] }
-unlicensed = "deny"
-allow-osi-fsf-free = "neither"
 # We want really high confidence when inferring licenses from text
 confidence-threshold = 0.92
-copyleft = "deny"
 allow = [
     "Apache-2.0",                     # https://tldrlegal.com/license/apache-license-2.0-(apache-2.0)
     "MIT",                            # https://tldrlegal.com/license/mit-license

tiny-bench/benches/benchmark.rs

@@ -12,6 +12,8 @@ fn main() {
     bench_test_three();
     #[cfg(feature = "bench")]
     bench_test_four();
+    #[cfg(feature = "bench")]
+    bench_test_five();
 }
 
 #[cfg(feature = "bench")]
@@ -67,3 +69,8 @@ fn bench_test_four() {
         || {},
     );
 }
+
+#[cfg(feature = "bench")]
+fn bench_test_five() {
+    tiny_bench::bench_with_setup_labeled("test five, setup noop", || 5, |_i| {});
+}

tiny-bench/src/benching.rs

@@ -86,13 +86,16 @@ pub fn bench_with_configuration_labeled<T, F: FnMut() -> T>(
 
 fn calculate_iters_and_total_iters(
     cfg: &BenchmarkConfig,
-    mean_execution_time: f64,
+    mut mean_execution_time_ns: f64,
     sample_size: u64,
 ) -> (Vec<u64>, u128) {
     if let Some(max_it) = cfg.max_iterations {
         (vec![max_it], u128::from(max_it))
     } else {
-        let iters = calculate_iterations(mean_execution_time, sample_size, cfg.measurement_time);
+        // This can be arbitrarily small, causing an absurd amount of iterations.
+        // Raise it to 1 nano -> max 5B iterations
+        mean_execution_time_ns = mean_execution_time_ns.max(1.0);
+        let iters = calculate_iterations(mean_execution_time_ns, sample_size, cfg.measurement_time);
         let mut total_iters = 0u128;
         for count in iters.iter().copied() {
             total_iters = total_iters.saturating_add(u128::from(count));
@@ -108,7 +111,7 @@ fn run<T, F: FnMut() -> T>(sample_sizes: Vec<u64>, mut closure: F) -> SamplingDa
         .map(|it_count| {
             let start = Instant::now();
             for _ in 0..it_count {
-                black_box((closure)());
+                black_box(closure());
             }
             start.elapsed().as_nanos()
         })
@@ -218,18 +221,40 @@ fn run_with_setup<T, R, F: FnMut(R) -> T, S: FnMut() -> R>(
     mut setup: S,
     mut closure: F,
 ) -> SamplingData {
+    const BATCH_SIZE: usize = 10_000;
     let times = sample_sizes
         .iter()
         .copied()
         .map(|it_count| {
-            let mut elapsed = Duration::ZERO;
-            for _ in 0..it_count {
-                let input = (setup)();
+            if it_count < BATCH_SIZE as u64 {
+                let inputs = (0..it_count).map(|_| setup()).collect::<Vec<_>>();
                 let start = Instant::now();
-                black_box((closure)(input));
-                elapsed += Instant::now().duration_since(start);
+                for i in inputs {
+                    black_box(closure(i));
+                }
+                start.elapsed().as_nanos()
+            } else {
+                let mut elapsed = Duration::ZERO;
+                let mut batch = Vec::with_capacity(BATCH_SIZE);
+                for _ in 0..it_count {
+                    batch.push(setup());
+                    if batch.len() >= BATCH_SIZE {
+                        let start = Instant::now();
+                        for i in batch.drain(..) {
+                            black_box(closure(i));
+                        }
+                        elapsed += start.elapsed();
+                    }
+                }
+                if !batch.is_empty() {
+                    let start = Instant::now();
+                    for i in batch {
+                        black_box(closure(i));
+                    }
+                    elapsed += start.elapsed();
+                }
+                elapsed.as_nanos()
             }
-            elapsed.as_nanos()
         })
         .collect();
     SamplingData {

tiny-bench/src/output/analysis/criterion.rs

@@ -121,7 +121,7 @@ pub(crate) fn calculate_p_value(total_t: f64, distribution: &[f64]) -> f64 {
 }
 
 #[inline]
-pub(crate) fn calculate_median(sample: &mut Vec<f64>) -> f64 {
+pub(crate) fn calculate_median(sample: &mut [f64]) -> f64 {
     sample.sort_by(f64::total_cmp);
     sample.get(sample.len() / 2).copied().unwrap_or_default()
 }