feat: update DB creation circuits for generic extraction (Part 2)

mp2-common/src/poseidon.rs

@@ -35,6 +35,9 @@ pub type H = <C as GenericConfig<D>>::Hasher;
 pub type P = <H as AlgebraicHasher<GoldilocksField>>::AlgebraicPermutation;
 pub type HashPermutation = <H as Hasher<F>>::Permutation;
 
+/// The result of hash to integer has 4 Uint32 (128 bits).
+pub const HASH_TO_INT_LEN: usize = 4;
+
 /// The flattened length of Poseidon hash, each original field is splitted from an
 /// Uint64 into two Uint32.
 pub const FLATTEN_POSEIDON_LEN: usize = NUM_HASH_OUT_ELTS * 2;

mp2-common/src/utils.rs

@@ -12,6 +12,7 @@ use plonky2::plonk::circuit_builder::CircuitBuilder;
 use plonky2::plonk::circuit_data::VerifierCircuitData;
 use plonky2::plonk::config::{GenericConfig, GenericHashOut, Hasher};
 use plonky2_crypto::u32::arithmetic_u32::U32Target;
+use plonky2_ecdsa::gadgets::biguint::BigUintTarget;
 
 use plonky2_ecgfp5::gadgets::{base_field::QuinticExtensionTarget, curve::CurveTarget};
 use sha3::Digest;
@@ -372,6 +373,7 @@ impl<F: RichField> ToFields<F> for HashOut<F> {
         self.elements.to_vec()
     }
 }
+
 pub trait Fieldable<F: RichField> {
     fn to_field(&self) -> F;
 }
@@ -439,6 +441,12 @@ impl ToTargets for &[Target] {
     }
 }
 
+impl ToTargets for BigUintTarget {
+    fn to_targets(&self) -> Vec<Target> {
+        self.limbs.iter().map(|u| u.0).collect()
+    }
+}
+
 pub trait TargetsConnector {
     fn connect_targets<T: ToTargets>(&mut self, e1: T, e2: T);
     fn is_equal_targets<T: ToTargets>(&mut self, e1: T, e2: T) -> BoolTarget;

mp2-v1/tests/common/celltree.rs

@@ -10,7 +10,7 @@ use mp2_v1::{
         row::{CellCollection, Row, RowPayload, RowTreeKey},
     },
 };
-use plonky2::plonk::config::GenericHashOut;
+use plonky2::{field::types::Sample, hash::hash_types::HashOut, plonk::config::GenericHashOut};
 use ryhope::storage::{
     updatetree::{Next, UpdateTree},
     RoEpochKvStorage,
@@ -69,6 +69,8 @@ impl TestContext {
                         cell.identifier(),
                         cell.value(),
                         column.multiplier,
+                        // TODO: Check mpt_metadata = cell.hash?
+                        HashOut::rand(),
                     ),
                 );
                 self.b.bench("indexing::cell_tree::leaf", || {
@@ -94,6 +96,8 @@ impl TestContext {
                         cell.identifier(),
                         cell.value(),
                         column.multiplier,
+                        // TODO: Check mpt_metadata = cell.hash?
+                        HashOut::rand(),
                         left_proof.clone(),
                     ),
                 );
@@ -149,6 +153,8 @@ impl TestContext {
                         cell.identifier(),
                         cell.value(),
                         column.multiplier,
+                        // TODO: Check mpt_metadata = cell.hash?
+                        HashOut::rand(),
                         [left_proof, right_proof],
                     ),
                 );
@@ -171,7 +177,7 @@ impl TestContext {
                 "[+] [+] Merkle SLOT identifier {:?} -> value {} value.digest() = {:?}",
                 cell.identifier(),
                 cell.value(),
-                pi.individual_digest_point()
+                pi.individual_values_digest_point()
             );
 
             self.storage

mp2-v1/tests/common/index_tree.rs

@@ -86,7 +86,7 @@ impl TestContext {
                 // TODO: Fix the rows digest in rows tree according to values extraction update.
                 // <https://github.com/Lagrange-Labs/mapreduce-plonky2/pull/385>
                 assert_eq!(
-                    row_pi.rows_digest_field(),
+                    row_pi.individual_digest_point(),
                     ext_pi.value_point(),
                     "values extracted vs value in db don't match (left row, right mpt (block {})",
                     node.value.0.to::<u64>()

mp2-v1/tests/common/mod.rs

@@ -51,7 +51,7 @@ fn cell_tree_proof_to_hash(proof: &[u8]) -> HashOutput {
         .proof
         .public_inputs;
     verifiable_db::cells_tree::PublicInputs::from_slice(&root_pi)
-        .root_hash_hashout()
+        .node_hash()
         .to_bytes()
         .try_into()
         .unwrap()
@@ -63,7 +63,7 @@ fn row_tree_proof_to_hash(proof: &[u8]) -> HashOutput {
         .proof
         .public_inputs;
     verifiable_db::row_tree::PublicInputs::from_slice(&root_pi)
-        .root_hash_hashout()
+        .root_hash()
         .to_bytes()
         .try_into()
         .unwrap()

mp2-v1/tests/common/rowtree.rs

@@ -11,7 +11,7 @@ use mp2_v1::{
         row::{RowPayload, RowTree, RowTreeKey, ToNonce},
     },
 };
-use plonky2::plonk::config::GenericHashOut;
+use plonky2::{field::types::Sample, hash::hash_types::HashOut, plonk::config::GenericHashOut};
 use ryhope::{
     storage::{
         pgsql::PgsqlStorage,
@@ -132,8 +132,8 @@ impl TestContext {
                 let pis = cells_tree::PublicInputs::from_slice(&pvk.proof().public_inputs);
                 debug!(
                     " Cell Root SPLIT digest: multiplier {:?}, individual {:?}",
-                    pis.multiplier_digest_point(),
-                    pis.individual_digest_point()
+                    pis.multiplier_values_digest_point(),
+                    pis.individual_values_digest_point()
                 );
             }
 
@@ -151,6 +151,10 @@ impl TestContext {
                         id,
                         value,
                         multiplier,
+                        // TODO: mpt_metadata
+                        HashOut::rand(),
+                        // TODO: row_unique_data
+                        HashOut::rand(),
                         cell_tree_proof,
                     )
                     .unwrap(),
@@ -187,6 +191,10 @@ impl TestContext {
                         value,
                         multiplier,
                         context.left.is_some(),
+                        // TODO: mpt_metadata
+                        HashOut::rand(),
+                        // TODO: row_unique_data
+                        HashOut::rand(),
                         child_proof,
                         cell_tree_proof,
                     )
@@ -229,6 +237,10 @@ impl TestContext {
                         id,
                         value,
                         multiplier,
+                        // TODO: mpt_metadata
+                        HashOut::rand(),
+                        // TODO: row_unique_data
+                        HashOut::rand(),
                         left_proof,
                         right_proof,
                         cell_tree_proof,
@@ -277,7 +289,7 @@ impl TestContext {
         let pi = verifiable_db::row_tree::PublicInputs::from_slice(&pproof.proof().public_inputs);
         debug!(
             "[--] FINAL MERKLE DIGEST VALUE --> {:?} ",
-            pi.rows_digest_field()
+            pi.individual_digest_point()
         );
         if root_proof_key.primary != primary {
             debug!("[--] NO UPDATES on row this turn? row.root().primary = {} vs new primary proving step {}",root_proof_key.primary,primary);

parsil/src/tests.rs

@@ -149,6 +149,7 @@ fn test_serde_circuit_pis() {
 }
 
 #[test]
+#[ignore = "wait for non-aggregation SELECT to come back"]
 fn isolation() {
     fn isolated_to_string(q: &str, lo_sec: bool, hi_sec: bool) -> String {
         let settings = ParsilSettings {

verifiable-db/Cargo.toml

@@ -5,6 +5,7 @@ edition = "2021"
 
 [dependencies]
 mp2_common = { path = "../mp2-common" }
+num.workspace = true
 plonky2_crypto.workspace = true
 recursion_framework = { path = "../recursion-framework" }
 ryhope = { path = "../ryhope" }
@@ -29,4 +30,5 @@ serial_test.workspace = true
 tokio.workspace = true
 
 [features]
-original_poseidon = ["mp2_common/original_poseidon"]
+original_poseidon = ["mp2_common/original_poseidon"]
+

verifiable-db/src/block_tree/api.rs

@@ -274,7 +274,10 @@ mod tests {
         *,
     };
     use crate::{
-        block_tree::leaf::tests::{compute_expected_hash, compute_expected_set_digest},
+        block_tree::{
+            compute_final_digest,
+            leaf::tests::{compute_expected_hash, compute_expected_set_digest},
+        },
         extraction, row_tree,
     };
     use mp2_common::{
@@ -288,13 +291,12 @@ mod tests {
         iop::target::Target,
         plonk::config::Hasher,
     };
-    use plonky2_ecgfp5::curve::curve::Point;
     use rand::{rngs::ThreadRng, thread_rng, Rng};
     use recursion_framework::framework_testing::TestingRecursiveCircuits;
     use std::iter;
 
     const EXTRACTION_IO_LEN: usize = extraction::test::PublicInputs::<F>::TOTAL_LEN;
-    const ROWS_TREE_IO_LEN: usize = row_tree::PublicInputs::<F>::TOTAL_LEN;
+    const ROWS_TREE_IO_LEN: usize = row_tree::PublicInputs::<F>::total_len();
 
     struct TestBuilder<E>
     where
@@ -346,10 +348,9 @@ mod tests {
         fn generate_rows_tree_proof(
             &self,
             rng: &mut ThreadRng,
-            row_digest: &[F],
             is_merge_case: bool,
         ) -> Result<ProofWithVK> {
-            let pi = random_rows_tree_pi(rng, row_digest, is_merge_case);
+            let pi = random_rows_tree_pi(rng, is_merge_case);
 
             let proof = self
                 .rows_tree_set
@@ -358,20 +359,23 @@ mod tests {
 
             Ok(ProofWithVK::from((proof[0].clone(), vk)))
         }
+
         fn generate_leaf_proof(
             &self,
             rng: &mut ThreadRng,
             block_id: F,
             block_number: U256,
         ) -> Result<ProofWithVK> {
-            let row_digest = Point::sample(rng).to_weierstrass().to_fields();
+            let rows_tree_proof = self.generate_rows_tree_proof(rng, true)?;
+            let rows_tree_pi =
+                row_tree::PublicInputs::from_slice(&rows_tree_proof.proof.public_inputs);
+            let final_digest = compute_final_digest(true, &rows_tree_pi)
+                .to_weierstrass()
+                .to_fields();
             let extraction_proof =
-                self.generate_extraction_proof(rng, block_number, &row_digest, true)?;
-            let rows_tree_proof = self.generate_rows_tree_proof(rng, &row_digest, true)?;
+                self.generate_extraction_proof(rng, block_number, &final_digest, true)?;
             let extraction_pi =
                 extraction::test::PublicInputs::from_slice(&extraction_proof.proof.public_inputs);
-            let rows_tree_pi =
-                row_tree::PublicInputs::from_slice(&rows_tree_proof.proof.public_inputs);
 
             let input = CircuitInput::new_leaf(
                 block_id.to_canonical_u64(),
@@ -438,8 +442,12 @@ mod tests {
             }
             // Check new node digest
             {
-                let exp_digest =
-                    compute_expected_set_digest(block_id, block_number.to_vec(), rows_tree_pi);
+                let exp_digest = compute_expected_set_digest(
+                    true,
+                    block_id,
+                    block_number.to_vec(),
+                    rows_tree_pi,
+                );
                 assert_eq!(pi.new_value_set_digest_point(), exp_digest.to_weierstrass());
             }
 
@@ -458,14 +466,16 @@ mod tests {
             left_child: HashOut<F>,
             right_child: HashOut<F>,
         ) -> Result<ProofWithVK> {
-            let row_digest = Point::sample(rng).to_weierstrass().to_fields();
+            let rows_tree_proof = self.generate_rows_tree_proof(rng, false)?;
+            let rows_tree_pi =
+                row_tree::PublicInputs::from_slice(&rows_tree_proof.proof.public_inputs);
+            let final_digest = compute_final_digest(false, &rows_tree_pi)
+                .to_weierstrass()
+                .to_fields();
             let extraction_proof =
-                self.generate_extraction_proof(rng, block_number, &row_digest, false)?;
-            let rows_tree_proof = self.generate_rows_tree_proof(rng, &row_digest, false)?;
+                self.generate_extraction_proof(rng, block_number, &final_digest, false)?;
             let extraction_pi =
                 extraction::test::PublicInputs::from_slice(&extraction_proof.proof.public_inputs);
-            let rows_tree_pi =
-                row_tree::PublicInputs::from_slice(&rows_tree_proof.proof.public_inputs);
 
             let old_rows_tree_hash =
                 HashOut::from_vec(random_vector::<u32>(NUM_HASH_OUT_ELTS).to_fields());
@@ -553,8 +563,12 @@ mod tests {
             }
             // Check new node digest
             {
-                let exp_digest =
-                    compute_expected_set_digest(block_id, block_number.to_vec(), rows_tree_pi);
+                let exp_digest = compute_expected_set_digest(
+                    false,
+                    block_id,
+                    block_number.to_vec(),
+                    rows_tree_pi,
+                );
                 assert_eq!(pi.new_value_set_digest_point(), exp_digest.to_weierstrass());
             }