From d006f3e66bd06a033851299d7101fc8215ac35bb Mon Sep 17 00:00:00 2001 From: aner-starkware <147302140+aner-starkware@users.noreply.github.com> Date: Thu, 20 Jun 2024 20:28:43 +0300 Subject: [PATCH] test: add full committer flow benchmarking (#236) * test: add full committer flow benchmarking * test: add full committer flow benchmarking test --- .github/workflows/ci.yml | 3 +- .../committer_cli/benches/committer_bench.rs | 33 ++++++++++++++++--- .../benches/committer_flow_inputs.json | 1 + .../src/tests/benchmark_tests.rs | 32 ++++++++++++++---- 4 files changed, 57 insertions(+), 12 deletions(-) create mode 100644 crates/committer_cli/benches/committer_flow_inputs.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4cacbcf5..435b57db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,7 +92,7 @@ jobs: - uses: Swatinem/rust-cache@v2 - run: cargo test - run-bench-test: + run-bench-tests: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -105,6 +105,7 @@ jobs: credentials_json: ${{ secrets.COMMITER_PRODUCTS_EXT_WRITER_JSON }} - uses: 'google-github-actions/setup-gcloud@v2' - run: gcloud storage cp gs://committer-products-external/tree_flow_inputs.json ./crates/committer_cli/benches/tree_flow_inputs.json + - run: gcloud storage cp gs://committer-products-external/committer_flow_inputs.json ./crates/committer_cli/benches/committer_flow_inputs.json - run: cargo test --release -- --include-ignored test_benchmark gcs-push: diff --git a/crates/committer_cli/benches/committer_bench.rs b/crates/committer_cli/benches/committer_bench.rs index a2b4351f..e61f65e5 100644 --- a/crates/committer_cli/benches/committer_bench.rs +++ b/crates/committer_cli/benches/committer_bench.rs @@ -9,15 +9,19 @@ use committer::{ types::NodeIndex, }, }; -use committer_cli::tests::utils::parse_from_python::parse_input_single_storage_tree_flow_test; +use committer_cli::{ + commands::commit, tests::utils::parse_from_python::parse_input_single_storage_tree_flow_test, +}; use criterion::{criterion_group, criterion_main, Criterion}; const CONCURRENCY_MODE: bool = true; -const INPUT: &str = include_str!("tree_flow_inputs.json"); +const SINGLE_TREE_FLOW_INPUT: &str = include_str!("tree_flow_inputs.json"); +const FLOW_TEST_INPUT: &str = include_str!("committer_flow_inputs.json"); pub fn single_tree_flow_benchmark(criterion: &mut Criterion) { - let (leaf_modifications, storage, root_hash) = - parse_input_single_storage_tree_flow_test(&serde_json::from_str(INPUT).unwrap()); + let (leaf_modifications, storage, root_hash) = parse_input_single_storage_tree_flow_test( + &serde_json::from_str(SINGLE_TREE_FLOW_INPUT).unwrap(), + ); let runtime = match CONCURRENCY_MODE { true => tokio::runtime::Builder::new_multi_thread().build().unwrap(), @@ -47,5 +51,24 @@ pub fn single_tree_flow_benchmark(criterion: &mut Criterion) { }); } -criterion_group!(benches, single_tree_flow_benchmark); +pub fn full_committer_flow_benchmark(criterion: &mut Criterion) { + let runtime = match CONCURRENCY_MODE { + true => tokio::runtime::Builder::new_multi_thread().build().unwrap(), + false => tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(), + }; + + criterion.bench_function("full_committer_flow", |benchmark| { + benchmark.iter(|| { + runtime.block_on(commit(FLOW_TEST_INPUT)); + }) + }); +} + +criterion_group!( + benches, + single_tree_flow_benchmark, + full_committer_flow_benchmark +); criterion_main!(benches); diff --git a/crates/committer_cli/benches/committer_flow_inputs.json b/crates/committer_cli/benches/committer_flow_inputs.json new file mode 100644 index 00000000..04326e03 --- /dev/null +++ b/crates/committer_cli/benches/committer_flow_inputs.json @@ -0,0 +1 @@ +This file is a placeholder for inputs to single_tree_flow regression test and benchamrk. diff --git a/crates/committer_cli/src/tests/benchmark_tests.rs b/crates/committer_cli/src/tests/benchmark_tests.rs index 885577cc..c7261e37 100644 --- a/crates/committer_cli/src/tests/benchmark_tests.rs +++ b/crates/committer_cli/src/tests/benchmark_tests.rs @@ -4,18 +4,24 @@ use committer::patricia_merkle_tree::external_test_utils::single_tree_flow_test; use pretty_assertions::assert_eq; use serde_json::Value; -use crate::tests::utils::parse_from_python::parse_input_single_storage_tree_flow_test; +use crate::{ + commands::commit, tests::utils::parse_from_python::parse_input_single_storage_tree_flow_test, +}; //TODO(Aner, 20/06/2024): this test needs to be fixed to be run correctly in the CI: //1. Fix the test to measure cpu_time and not wall_time. //2. Fix the max time threshold to be the expected time for the benchmark test. -const MAX_TIME_FOR_BECHMARK_TEST: f64 = 5.0; -const INPUT: &str = include_str!("../../benches/tree_flow_inputs.json"); +const MAX_TIME_FOR_SINGLE_TREE_BECHMARK_TEST: f64 = 5.0; +const MAX_TIME_FOR_COMMITTER_FLOW_BECHMARK_TEST: f64 = 5.0; +const SINGLE_TREE_FLOW_INPUT: &str = include_str!("../../benches/tree_flow_inputs.json"); +//TODO(Aner, 20/06/2024): modify the committer_flow_inputs.json file to be from pseudo-real data +// and to include the expected output. +const FLOW_TEST_INPUT: &str = include_str!("../../benches/committer_flow_inputs.json"); #[ignore = "To avoid running the benchmark test in Coverage or without the --release flag."] #[tokio::test(flavor = "multi_thread")] -pub async fn test_benchmark() { - let input: HashMap = serde_json::from_str(INPUT).unwrap(); +pub async fn test_benchmark_single_tree() { + let input: HashMap = serde_json::from_str(SINGLE_TREE_FLOW_INPUT).unwrap(); let (leaf_modifications, storage, root_hash) = parse_input_single_storage_tree_flow_test(&input); let expected_hash = input.get("expected_hash").unwrap(); @@ -30,5 +36,19 @@ pub async fn test_benchmark() { assert_eq!(output_hash.as_str().unwrap(), expected_hash); // 4. Assert the execution time does not exceed the threshold. - assert!(execution_time.as_secs_f64() < MAX_TIME_FOR_BECHMARK_TEST); + assert!(execution_time.as_secs_f64() < MAX_TIME_FOR_SINGLE_TREE_BECHMARK_TEST); +} + +#[ignore = "To avoid running the benchmark test in Coverage or without the --release flag."] +#[tokio::test(flavor = "multi_thread")] +pub async fn test_benchmark_committer_flow() { + let start = std::time::Instant::now(); + // Benchmark the committer flow test. + commit(FLOW_TEST_INPUT).await; + let execution_time = std::time::Instant::now() - start; + + // TODO(Aner, 20/06/2024): add assert for the output of the committer flow test. + + // Assert the execution time does not exceed the threshold. + assert!(execution_time.as_secs_f64() < MAX_TIME_FOR_COMMITTER_FLOW_BECHMARK_TEST); }