From bf2c7b4a1b323cf3342959d5d3b6bf00e161f898 Mon Sep 17 00:00:00 2001 From: arnaucube Date: Fri, 22 Nov 2024 15:18:23 +0100 Subject: [PATCH 1/4] add Nova benchmarks logic --- benches/README.md | 5 ++ benches/nova.rs | 117 ++++++++++++++++++++++++++ folding-schemes/Cargo.toml | 9 ++ folding-schemes/src/frontend/utils.rs | 18 ++-- 4 files changed, 140 insertions(+), 9 deletions(-) create mode 100644 benches/README.md create mode 100644 benches/nova.rs diff --git a/benches/README.md b/benches/README.md new file mode 100644 index 00000000..09091f26 --- /dev/null +++ b/benches/README.md @@ -0,0 +1,5 @@ +# benchmarks + +Run: `cargo bench` + +To run a specific benchmark, for example Nova benchmark, run: `cargo bench nova` diff --git a/benches/nova.rs b/benches/nova.rs new file mode 100644 index 00000000..11200512 --- /dev/null +++ b/benches/nova.rs @@ -0,0 +1,117 @@ +use ark_ec::CurveGroup; +use ark_ff::PrimeField; +use criterion::*; + +use ark_bn254::{constraints::GVar as bn_GVar, Fr as bn_Fr, G1Projective as bn_G}; +use ark_grumpkin::{constraints::GVar as grumpkin_GVar, Projective as grumpkin_G}; +use ark_pallas::{constraints::GVar as pallas_GVar, Fr as pallas_Fr, Projective as pallas_G}; +use ark_vesta::{constraints::GVar as vesta_GVar, Projective as vesta_G}; + +use folding_schemes::{ + commitment::pedersen::Pedersen, + folding::nova::{Nova, PreprocessorParam}, + frontend::{utils::CustomFCircuit, FCircuit}, + transcript::poseidon::poseidon_canonical_config, + Error, FoldingScheme, +}; + +fn bench_nova_ivc(c: &mut Criterion) { + bench_nova_ivc_opt::< + pallas_G, + vesta_G, + Nova< + pallas_G, + pallas_GVar, + vesta_G, + vesta_GVar, + CustomFCircuit, + Pedersen, + Pedersen, + false, + >, + >(c, "Nova - Pallas-Vesta curves".to_string()) + .unwrap(); + + bench_nova_ivc_opt::< + bn_G, + grumpkin_G, + Nova< + bn_G, + bn_GVar, + grumpkin_G, + grumpkin_GVar, + CustomFCircuit, + Pedersen, + Pedersen, + false, + >, + >(c, "Nova - BN254-Grumpkin curves".to_string()) + .unwrap(); +} +fn bench_nova_ivc_opt< + C1: CurveGroup, + C2: CurveGroup, + FS: FoldingScheme< + C1, + C2, + CustomFCircuit, + PreprocessorParam = PreprocessorParam< + C1, + C2, + CustomFCircuit, + Pedersen, + Pedersen, + false, + >, + >, +>( + c: &mut Criterion, + name: String, +) -> Result<(), Error> +where + C1: CurveGroup, + C2::BaseField: PrimeField, +{ + // iterate over the powers of n + for n in [10, 14, 15, 16, 17, 18, 19, 20].iter() { + let fcircuit_size = 1 << n; // 2^n + + let f_circuit = CustomFCircuit::::new(fcircuit_size)?; + + let poseidon_config = poseidon_canonical_config::(); + let mut rng = rand::rngs::OsRng; + + // prepare the Nova prover & verifier params + let prep_param: FS::PreprocessorParam = + PreprocessorParam::new(poseidon_config.clone(), f_circuit); + let fs_params = FS::preprocess(&mut rng, &prep_param)?; + + let z_0 = vec![C1::ScalarField::from(3_u32)]; + let mut fs = FS::init(&fs_params, f_circuit, z_0)?; + + // warmup steps + for _ in 0..5 { + fs.prove_step(rng, vec![], None)?; + } + + let mut group = c.benchmark_group(format!( + "{} - FCircuit: {} (2^{}) constraints", + name, fcircuit_size, n + )); + group.significance_level(0.1).sample_size(10); + group.bench_function("prove_step", |b| { + b.iter(|| fs.prove_step(rng, vec![], None).unwrap()) + }); + + // verify the IVCProof + let ivc_proof = fs.ivc_proof(); + group.bench_function("verify", |b| { + b.iter(|| FS::verify(fs_params.1.clone(), ivc_proof.clone()).unwrap()) + }); + group.finish(); + } + Ok(()) +} + +criterion_group!(benches, bench_nova_ivc); +criterion_main!(benches); diff --git a/folding-schemes/Cargo.toml b/folding-schemes/Cargo.toml index 22b074de..b0b4e41d 100644 --- a/folding-schemes/Cargo.toml +++ b/folding-schemes/Cargo.toml @@ -42,6 +42,10 @@ num-bigint = {version = "0.4", features = ["rand"]} tracing = { version = "0.1", default-features = false, features = [ "attributes" ] } tracing-subscriber = { version = "0.2" } +# for benchmarks +criterion = "0.5" +pprof = { version = "0.13", features = ["criterion", "flamegraph"] } + # This allows the crate to be built when targeting WASM. # See more at: https://docs.rs/getrandom/#webassembly-support [target.'cfg(all(target_arch = "wasm32", target_os = "unknown"))'.dependencies] @@ -53,6 +57,11 @@ parallel = [] light-test = [] +[[bench]] +name = "nova" +path = "../benches/nova.rs" +harness = false + [[example]] name = "sha256" path = "../examples/sha256.rs" diff --git a/folding-schemes/src/frontend/utils.rs b/folding-schemes/src/frontend/utils.rs index 27d4ec98..24b0cdd7 100644 --- a/folding-schemes/src/frontend/utils.rs +++ b/folding-schemes/src/frontend/utils.rs @@ -1,7 +1,9 @@ use ark_ff::PrimeField; -use ark_r1cs_std::{alloc::AllocVar, fields::fp::FpVar}; +use ark_r1cs_std::{ + alloc::AllocVar, + fields::{fp::FpVar, FieldVar}, +}; use ark_relations::r1cs::{ConstraintSystemRef, SynthesisError}; -#[cfg(test)] use ark_std::marker::PhantomData; use ark_std::{fmt::Debug, Zero}; @@ -96,14 +98,12 @@ impl FCircuit for CubicFCircuit { /// CustomFCircuit is a circuit that has the number of constraints specified in the /// `n_constraints` parameter. Note that the generated circuit will have very sparse matrices. -#[cfg(test)] #[derive(Clone, Copy, Debug)] pub struct CustomFCircuit { _f: PhantomData, pub n_constraints: usize, } -#[cfg(test)] impl FCircuit for CustomFCircuit { type Params = usize; @@ -125,22 +125,22 @@ impl FCircuit for CustomFCircuit { z_i: Vec, _external_inputs: Vec, ) -> Result, Error> { - let mut z_i1 = F::one(); + let mut z_i1 = z_i[0]; for _ in 0..self.n_constraints - 1 { - z_i1 *= z_i[0]; + z_i1 = z_i1.square(); } Ok(vec![z_i1]) } fn generate_step_constraints( &self, - cs: ConstraintSystemRef, + _cs: ConstraintSystemRef, _i: usize, z_i: Vec>, _external_inputs: Vec>, ) -> Result>, SynthesisError> { - let mut z_i1 = FpVar::::new_witness(cs.clone(), || Ok(F::one()))?; + let mut z_i1 = z_i[0].clone(); for _ in 0..self.n_constraints - 1 { - z_i1 *= z_i[0].clone(); + z_i1 = z_i1.square()?; } Ok(vec![z_i1]) From d31c4419b85cb7c330681ad838cd22eddd1b63e7 Mon Sep 17 00:00:00 2001 From: arnaucube Date: Fri, 22 Nov 2024 16:35:16 +0100 Subject: [PATCH 2/4] abstract benchmarks, add HyperNova & ProtoGalaxy's IVC benchmarks --- benches/README.md | 2 +- benches/common.rs | 57 +++++++++++++++ benches/hypernova.rs | 83 ++++++++++++++++++++++ benches/nova.rs | 139 +++++++++++++------------------------ benches/protogalaxy.rs | 77 ++++++++++++++++++++ folding-schemes/Cargo.toml | 10 +++ 6 files changed, 276 insertions(+), 92 deletions(-) create mode 100644 benches/common.rs create mode 100644 benches/hypernova.rs create mode 100644 benches/protogalaxy.rs diff --git a/benches/README.md b/benches/README.md index 09091f26..d479bb56 100644 --- a/benches/README.md +++ b/benches/README.md @@ -2,4 +2,4 @@ Run: `cargo bench` -To run a specific benchmark, for example Nova benchmark, run: `cargo bench nova` +To run a specific benchmark, for example Nova benchmark, run: `cargo bench --bench=nova` diff --git a/benches/common.rs b/benches/common.rs new file mode 100644 index 00000000..c36866a2 --- /dev/null +++ b/benches/common.rs @@ -0,0 +1,57 @@ +use ark_ec::CurveGroup; +use ark_ff::PrimeField; +use criterion::*; + +use folding_schemes::{ + frontend::{utils::CustomFCircuit, FCircuit}, + Error, FoldingScheme, +}; + +pub(crate) fn bench_ivc_opt< + C1: CurveGroup, + C2: CurveGroup, + FS: FoldingScheme>, +>( + c: &mut Criterion, + name: String, + n: usize, + prep_param: FS::PreprocessorParam, +) -> Result<(), Error> +where + C1: CurveGroup, + C2::BaseField: PrimeField, +{ + let fcircuit_size = 1 << n; // 2^n + + let f_circuit = CustomFCircuit::::new(fcircuit_size)?; + + let mut rng = rand::rngs::OsRng; + + // prepare the FS prover & verifier params + let fs_params = FS::preprocess(&mut rng, &prep_param)?; + + let z_0 = vec![C1::ScalarField::from(3_u32)]; + let mut fs = FS::init(&fs_params, f_circuit, z_0)?; + + // warmup steps + for _ in 0..5 { + fs.prove_step(rng, vec![], None)?; + } + + let mut group = c.benchmark_group(format!( + "{} - FCircuit: {} (2^{}) constraints", + name, fcircuit_size, n + )); + group.significance_level(0.1).sample_size(10); + group.bench_function("prove_step", |b| { + b.iter(|| black_box(fs.clone()).prove_step(rng, vec![], None).unwrap()) + }); + + // verify the IVCProof + let ivc_proof = fs.ivc_proof(); + group.bench_function("verify", |b| { + b.iter(|| FS::verify(black_box(fs_params.1.clone()), black_box(ivc_proof.clone())).unwrap()) + }); + group.finish(); + Ok(()) +} diff --git a/benches/hypernova.rs b/benches/hypernova.rs new file mode 100644 index 00000000..e32b9b45 --- /dev/null +++ b/benches/hypernova.rs @@ -0,0 +1,83 @@ +use criterion::*; + +use ark_bn254::{constraints::GVar as bn_GVar, Fr as bn_Fr, G1Projective as bn_G}; +use ark_grumpkin::{constraints::GVar as grumpkin_GVar, Projective as grumpkin_G}; +use ark_pallas::{constraints::GVar as pallas_GVar, Fr as pallas_Fr, Projective as pallas_G}; +use ark_vesta::{constraints::GVar as vesta_GVar, Projective as vesta_G}; + +use folding_schemes::{ + commitment::pedersen::Pedersen, + folding::{hypernova::HyperNova, nova::PreprocessorParam}, + frontend::{utils::CustomFCircuit, FCircuit}, + transcript::poseidon::poseidon_canonical_config, +}; + +mod common; +use common::bench_ivc_opt; + +fn bench_hypernova_ivc(c: &mut Criterion) { + let poseidon_config = poseidon_canonical_config::(); + + // iterate over the powers of n + for n in [0_usize, 14, 16, 18, 19, 20, 21, 22].iter() { + let fcircuit_size = 1 << n; // 2^n + let fcircuit = CustomFCircuit::::new(fcircuit_size).unwrap(); + let prep_param = PreprocessorParam::new(poseidon_config.clone(), fcircuit); + + bench_ivc_opt::< + pallas_G, + vesta_G, + HyperNova< + pallas_G, + pallas_GVar, + vesta_G, + vesta_GVar, + CustomFCircuit, + Pedersen, + Pedersen, + 1, + 1, + false, + >, + >( + c, + "HyperNova - Pallas-Vesta curves".to_string(), + *n, + prep_param, + ) + .unwrap(); + } + + let poseidon_config = poseidon_canonical_config::(); + for n in [0_usize, 14, 16, 18, 19, 20, 21, 22].iter() { + let fcircuit_size = 1 << n; // 2^n + let fcircuit = CustomFCircuit::::new(fcircuit_size).unwrap(); + let prep_param = PreprocessorParam::new(poseidon_config.clone(), fcircuit); + + bench_ivc_opt::< + bn_G, + grumpkin_G, + HyperNova< + bn_G, + bn_GVar, + grumpkin_G, + grumpkin_GVar, + CustomFCircuit, + Pedersen, + Pedersen, + 1, + 1, + false, + >, + >( + c, + "HyperNova - BN254-Grumpkin curves".to_string(), + *n, + prep_param, + ) + .unwrap(); + } +} + +criterion_group!(benches, bench_hypernova_ivc); +criterion_main!(benches); diff --git a/benches/nova.rs b/benches/nova.rs index 11200512..d34d2ad9 100644 --- a/benches/nova.rs +++ b/benches/nova.rs @@ -1,5 +1,3 @@ -use ark_ec::CurveGroup; -use ark_ff::PrimeField; use criterion::*; use ark_bn254::{constraints::GVar as bn_GVar, Fr as bn_Fr, G1Projective as bn_G}; @@ -12,105 +10,64 @@ use folding_schemes::{ folding::nova::{Nova, PreprocessorParam}, frontend::{utils::CustomFCircuit, FCircuit}, transcript::poseidon::poseidon_canonical_config, - Error, FoldingScheme, }; +mod common; +use common::bench_ivc_opt; + fn bench_nova_ivc(c: &mut Criterion) { - bench_nova_ivc_opt::< - pallas_G, - vesta_G, - Nova< - pallas_G, - pallas_GVar, - vesta_G, - vesta_GVar, - CustomFCircuit, - Pedersen, - Pedersen, - false, - >, - >(c, "Nova - Pallas-Vesta curves".to_string()) - .unwrap(); + let poseidon_config = poseidon_canonical_config::(); - bench_nova_ivc_opt::< - bn_G, - grumpkin_G, - Nova< - bn_G, - bn_GVar, - grumpkin_G, - grumpkin_GVar, - CustomFCircuit, - Pedersen, - Pedersen, - false, - >, - >(c, "Nova - BN254-Grumpkin curves".to_string()) - .unwrap(); -} -fn bench_nova_ivc_opt< - C1: CurveGroup, - C2: CurveGroup, - FS: FoldingScheme< - C1, - C2, - CustomFCircuit, - PreprocessorParam = PreprocessorParam< - C1, - C2, - CustomFCircuit, - Pedersen, - Pedersen, - false, - >, - >, ->( - c: &mut Criterion, - name: String, -) -> Result<(), Error> -where - C1: CurveGroup, - C2::BaseField: PrimeField, -{ // iterate over the powers of n - for n in [10, 14, 15, 16, 17, 18, 19, 20].iter() { + for n in [0_usize, 14, 16, 18, 19, 20, 21, 22].iter() { let fcircuit_size = 1 << n; // 2^n + let fcircuit = CustomFCircuit::::new(fcircuit_size).unwrap(); + let prep_param = PreprocessorParam::new(poseidon_config.clone(), fcircuit); - let f_circuit = CustomFCircuit::::new(fcircuit_size)?; - - let poseidon_config = poseidon_canonical_config::(); - let mut rng = rand::rngs::OsRng; - - // prepare the Nova prover & verifier params - let prep_param: FS::PreprocessorParam = - PreprocessorParam::new(poseidon_config.clone(), f_circuit); - let fs_params = FS::preprocess(&mut rng, &prep_param)?; - - let z_0 = vec![C1::ScalarField::from(3_u32)]; - let mut fs = FS::init(&fs_params, f_circuit, z_0)?; - - // warmup steps - for _ in 0..5 { - fs.prove_step(rng, vec![], None)?; - } + bench_ivc_opt::< + pallas_G, + vesta_G, + Nova< + pallas_G, + pallas_GVar, + vesta_G, + vesta_GVar, + CustomFCircuit, + Pedersen, + Pedersen, + false, + >, + >(c, "Nova - Pallas-Vesta curves".to_string(), *n, prep_param) + .unwrap(); + } - let mut group = c.benchmark_group(format!( - "{} - FCircuit: {} (2^{}) constraints", - name, fcircuit_size, n - )); - group.significance_level(0.1).sample_size(10); - group.bench_function("prove_step", |b| { - b.iter(|| fs.prove_step(rng, vec![], None).unwrap()) - }); + let poseidon_config = poseidon_canonical_config::(); + for n in [0_usize, 14, 16, 18, 19, 20, 21, 22].iter() { + let fcircuit_size = 1 << n; // 2^n + let fcircuit = CustomFCircuit::::new(fcircuit_size).unwrap(); + let prep_param = PreprocessorParam::new(poseidon_config.clone(), fcircuit); - // verify the IVCProof - let ivc_proof = fs.ivc_proof(); - group.bench_function("verify", |b| { - b.iter(|| FS::verify(fs_params.1.clone(), ivc_proof.clone()).unwrap()) - }); - group.finish(); + bench_ivc_opt::< + bn_G, + grumpkin_G, + Nova< + bn_G, + bn_GVar, + grumpkin_G, + grumpkin_GVar, + CustomFCircuit, + Pedersen, + Pedersen, + false, + >, + >( + c, + "Nova - BN254-Grumpkin curves".to_string(), + *n, + prep_param, + ) + .unwrap(); } - Ok(()) } criterion_group!(benches, bench_nova_ivc); diff --git a/benches/protogalaxy.rs b/benches/protogalaxy.rs new file mode 100644 index 00000000..3d0f8977 --- /dev/null +++ b/benches/protogalaxy.rs @@ -0,0 +1,77 @@ +use criterion::*; + +use ark_bn254::{constraints::GVar as bn_GVar, Fr as bn_Fr, G1Projective as bn_G}; +use ark_grumpkin::{constraints::GVar as grumpkin_GVar, Projective as grumpkin_G}; +use ark_pallas::{constraints::GVar as pallas_GVar, Fr as pallas_Fr, Projective as pallas_G}; +use ark_vesta::{constraints::GVar as vesta_GVar, Projective as vesta_G}; + +use folding_schemes::{ + commitment::pedersen::Pedersen, + folding::protogalaxy::ProtoGalaxy, + frontend::{utils::CustomFCircuit, FCircuit}, + transcript::poseidon::poseidon_canonical_config, +}; + +mod common; +use common::bench_ivc_opt; + +fn bench_protogalaxy_ivc(c: &mut Criterion) { + let poseidon_config = poseidon_canonical_config::(); + + // iterate over the powers of n + for n in [0_usize, 14, 16, 18, 19, 20, 21, 22].iter() { + let fcircuit_size = 1 << n; // 2^n + let fcircuit = CustomFCircuit::::new(fcircuit_size).unwrap(); + let prep_param = (poseidon_config.clone(), fcircuit); + + bench_ivc_opt::< + pallas_G, + vesta_G, + ProtoGalaxy< + pallas_G, + pallas_GVar, + vesta_G, + vesta_GVar, + CustomFCircuit, + Pedersen, + Pedersen, + >, + >( + c, + "ProtoGalaxy - Pallas-Vesta curves".to_string(), + *n, + prep_param, + ) + .unwrap(); + } + + let poseidon_config = poseidon_canonical_config::(); + for n in [0_usize, 14, 16, 18, 19, 20, 21, 22].iter() { + let fcircuit_size = 1 << n; // 2^n + let fcircuit = CustomFCircuit::::new(fcircuit_size).unwrap(); + let prep_param = (poseidon_config.clone(), fcircuit); + + bench_ivc_opt::< + bn_G, + grumpkin_G, + ProtoGalaxy< + bn_G, + bn_GVar, + grumpkin_G, + grumpkin_GVar, + CustomFCircuit, + Pedersen, + Pedersen, + >, + >( + c, + "ProtoGalaxy - BN254-Grumpkin curves".to_string(), + *n, + prep_param, + ) + .unwrap(); + } +} + +criterion_group!(benches, bench_protogalaxy_ivc); +criterion_main!(benches); diff --git a/folding-schemes/Cargo.toml b/folding-schemes/Cargo.toml index b0b4e41d..44b5f56f 100644 --- a/folding-schemes/Cargo.toml +++ b/folding-schemes/Cargo.toml @@ -62,6 +62,16 @@ name = "nova" path = "../benches/nova.rs" harness = false +[[bench]] +name = "hypernova" +path = "../benches/hypernova.rs" +harness = false + +[[bench]] +name = "protogalaxy" +path = "../benches/protogalaxy.rs" +harness = false + [[example]] name = "sha256" path = "../examples/sha256.rs" From 252a760b8947dc5a85f317f3b4dc3b7dfcdaa70e Mon Sep 17 00:00:00 2001 From: arnaucube Date: Mon, 25 Nov 2024 16:26:35 +0100 Subject: [PATCH 3/4] add profiler & flamegraph at benches --- benches/README.md | 9 +++++++-- benches/nova.rs | 10 +++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/benches/README.md b/benches/README.md index d479bb56..6f998097 100644 --- a/benches/README.md +++ b/benches/README.md @@ -1,5 +1,10 @@ # benchmarks +*Note: we're starting to benchmark & profile Sonobe, current results are pre-optimizations.* + +- Benchmark + - Run: `cargo bench` + - To run a specific benchmark, for example Nova's benchmark, run: `cargo bench --bench=nova` +- Profiling + - eg. `cargo bench --bench=nova -- --profile-time 3` -Run: `cargo bench` -To run a specific benchmark, for example Nova benchmark, run: `cargo bench --bench=nova` diff --git a/benches/nova.rs b/benches/nova.rs index d34d2ad9..3706e192 100644 --- a/benches/nova.rs +++ b/benches/nova.rs @@ -1,4 +1,8 @@ use criterion::*; +use pprof::{ + criterion::{Output, PProfProfiler}, + flamegraph::Options, +}; use ark_bn254::{constraints::GVar as bn_GVar, Fr as bn_Fr, G1Projective as bn_G}; use ark_grumpkin::{constraints::GVar as grumpkin_GVar, Projective as grumpkin_G}; @@ -70,5 +74,9 @@ fn bench_nova_ivc(c: &mut Criterion) { } } -criterion_group!(benches, bench_nova_ivc); +criterion_group! { + name = benches; + config = Criterion::default().with_profiler(PProfProfiler::new(100, Output::Flamegraph(None))); + targets = bench_nova_ivc +} criterion_main!(benches); From fa29af67f983f591731bcbd087c64705884a649a Mon Sep 17 00:00:00 2001 From: arnaucube Date: Mon, 25 Nov 2024 16:35:51 +0100 Subject: [PATCH 4/4] add bench compile to github CI --- .github/workflows/ci.yml | 16 ++++++++++++++++ benches/nova.rs | 5 +---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 80ca07de..d6a9edfd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,6 +140,22 @@ jobs: - name: Run examples run: cargo run --release --example 2>&1 | grep -E '^ ' | xargs -n1 cargo run --release --example + # run the benchmarks with the flag `--no-run` to ensure that they compile, + # but without executing them. + bench: + if: github.event.pull_request.draft == false + name: Bench compile + timeout-minutes: 30 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + - uses: Swatinem/rust-cache@v2 + - uses: actions-rs/cargo@v1 + with: + command: bench + args: --no-run + fmt: if: github.event.pull_request.draft == false name: Rustfmt diff --git a/benches/nova.rs b/benches/nova.rs index 3706e192..469e2358 100644 --- a/benches/nova.rs +++ b/benches/nova.rs @@ -1,8 +1,5 @@ use criterion::*; -use pprof::{ - criterion::{Output, PProfProfiler}, - flamegraph::Options, -}; +use pprof::criterion::{Output, PProfProfiler}; use ark_bn254::{constraints::GVar as bn_GVar, Fr as bn_Fr, G1Projective as bn_G}; use ark_grumpkin::{constraints::GVar as grumpkin_GVar, Projective as grumpkin_G};