From 32e2991c4d03bb8de1ce0d161492e157487aa098 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 3 Apr 2023 13:53:43 -0600 Subject: [PATCH 01/27] zcash_client_backend: Add note commitment tree sizes to `CompactBlock` serialization. --- zcash_client_backend/proto/compact_formats.proto | 2 ++ zcash_client_backend/src/proto/compact_formats.rs | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/zcash_client_backend/proto/compact_formats.proto b/zcash_client_backend/proto/compact_formats.proto index 077537c60..eac2b2f2f 100644 --- a/zcash_client_backend/proto/compact_formats.proto +++ b/zcash_client_backend/proto/compact_formats.proto @@ -22,6 +22,8 @@ message CompactBlock { uint32 time = 5; // Unix epoch time when the block was mined bytes header = 6; // (hash, prevHash, and time) OR (full header) repeated CompactTx vtx = 7; // zero or more compact transactions from this block + uint32 saplingCommitmentTreeSize = 8; // the size of the Sapling note commitment tree as of the end of this block + uint32 orchardCommitmentTreeSize = 9; // the size of the Orchard note commitment tree as of the end of this block } // CompactTx contains the minimum information for a wallet to know if this transaction diff --git a/zcash_client_backend/src/proto/compact_formats.rs b/zcash_client_backend/src/proto/compact_formats.rs index 056764b78..c8d45173c 100644 --- a/zcash_client_backend/src/proto/compact_formats.rs +++ b/zcash_client_backend/src/proto/compact_formats.rs @@ -26,6 +26,12 @@ pub struct CompactBlock { /// zero or more compact transactions from this block #[prost(message, repeated, tag = "7")] pub vtx: ::prost::alloc::vec::Vec, + /// the size of the Sapling note commitment tree as of the end of this block + #[prost(uint32, tag = "8")] + pub sapling_commitment_tree_size: u32, + /// the size of the Orchard note commitment tree as of the end of this block + #[prost(uint32, tag = "9")] + pub orchard_commitment_tree_size: u32, } /// CompactTx contains the minimum information for a wallet to know if this transaction /// is relevant to it (either pays to it or spends from it) via shielded elements From 3e358bc1c95d7b0f4ce8f662ac1668865661aed1 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 3 Apr 2023 13:53:43 -0600 Subject: [PATCH 02/27] zcash_client_backend: Use `shardtree` for note commitments in block scanning. Also adds a skeleton `zcash_client_sqlite` implementation of `shardtree::ShardStore` and a skeleton migration for related database changes. --- Cargo.toml | 5 + zcash_client_backend/Cargo.toml | 1 + zcash_client_backend/src/data_api.rs | 155 ++++++++---- zcash_client_backend/src/data_api/chain.rs | 160 ++++-------- .../src/data_api/chain/error.rs | 12 + zcash_client_backend/src/data_api/error.rs | 33 ++- zcash_client_backend/src/data_api/wallet.rs | 146 +++++++---- .../src/data_api/wallet/input_selection.rs | 53 ++-- zcash_client_backend/src/wallet.rs | 16 +- zcash_client_backend/src/welding_rig.rs | 235 ++++++++++-------- zcash_client_sqlite/Cargo.toml | 1 + zcash_client_sqlite/src/chain.rs | 45 +++- zcash_client_sqlite/src/error.rs | 11 + zcash_client_sqlite/src/lib.rs | 234 +++++++++-------- zcash_client_sqlite/src/wallet.rs | 79 ++++-- zcash_client_sqlite/src/wallet/init.rs | 17 +- .../src/wallet/init/migrations.rs | 2 + .../init/migrations/shardtree_support.rs | 56 +++++ zcash_client_sqlite/src/wallet/sapling.rs | 222 +++++++---------- .../src/wallet/sapling/commitment_tree.rs | 123 +++++++++ zcash_primitives/src/merkle_tree.rs | 2 +- 21 files changed, 1017 insertions(+), 591 deletions(-) create mode 100644 zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs create mode 100644 zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs diff --git a/Cargo.toml b/Cargo.toml index 044d879e9..073970f92 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,3 +17,8 @@ members = [ lto = true panic = 'abort' codegen-units = 1 + +[patch.crates-io] +incrementalmerkletree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "082109deacf8611ee7917732e19b56158bda96d5" } +shardtree = { git = "https://github.com/zcash/incrementalmerkletree.git", rev = "082109deacf8611ee7917732e19b56158bda96d5" } +orchard = { git = "https://github.com/zcash/orchard.git", rev = "5da41a6bbb44290e353ee4b38bcafe37ffe79ce8" } diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index eb6eb2c1d..7d0e46382 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -21,6 +21,7 @@ development = ["zcash_proofs"] [dependencies] incrementalmerkletree = { version = "0.4", features = ["legacy-api"] } +shardtree = "0.0" zcash_address = { version = "0.3", path = "../components/zcash_address" } zcash_encoding = { version = "0.2", path = "../components/zcash_encoding" } zcash_note_encryption = "0.4" diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 1b3dff2a7..80ad24f55 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -1,10 +1,12 @@ //! Interfaces for wallet data persistence & low-level wallet utilities. -use std::cmp; use std::collections::HashMap; use std::fmt::Debug; +use std::{cmp, ops::Range}; +use incrementalmerkletree::Retention; use secrecy::SecretVec; +use shardtree::{ShardStore, ShardTree, ShardTreeError}; use zcash_primitives::{ block::BlockHash, consensus::BlockHeight, @@ -29,6 +31,8 @@ pub mod chain; pub mod error; pub mod wallet; +pub const SAPLING_SHARD_HEIGHT: u8 = sapling::NOTE_COMMITMENT_TREE_DEPTH / 2; + pub enum NullifierQuery { Unspent, All, @@ -61,6 +65,30 @@ pub trait WalletRead { /// This will return `Ok(None)` if no block data is present in the database. fn block_height_extrema(&self) -> Result, Self::Error>; + /// Returns the height to which the wallet has been fully scanned. + /// + /// This is the height for which the wallet has fully trial-decrypted this and all preceding + /// blocks above the wallet's birthday height. Along with this height, this method returns + /// metadata describing the state of the wallet's note commitment trees as of the end of that + /// block. + fn fully_scanned_height( + &self, + ) -> Result, Self::Error>; + + /// Returns a vector of suggested scan ranges based upon the current wallet state. + /// + /// This method should only be used in cases where the [`CompactBlock`] data that will be made + /// available to `scan_cached_blocks` for the requested block ranges includes note commitment + /// tree size information for each block; or else the scan is likely to fail if notes belonging + /// to the wallet are detected. + /// + /// [`CompactBlock`]: crate::proto::compact_formats::CompactBlock + fn suggest_scan_ranges( + &self, + batch_size: usize, + limit: usize, + ) -> Result>, Self::Error>; + /// Returns the default target height (for the block in which a new /// transaction would be mined) and anchor height (to use for a new /// transaction), given the range of block heights that the backend @@ -165,19 +193,6 @@ pub trait WalletRead { /// Returns a transaction. fn get_transaction(&self, id_tx: Self::TxRef) -> Result; - /// Returns the note commitment tree at the specified block height. - fn get_commitment_tree( - &self, - block_height: BlockHeight, - ) -> Result, Self::Error>; - - /// Returns the incremental witnesses as of the specified block height. - #[allow(clippy::type_complexity)] - fn get_witnesses( - &self, - block_height: BlockHeight, - ) -> Result, Self::Error>; - /// Returns the nullifiers for notes that the wallet is tracking, along with their associated /// account IDs, that are either unspent or have not yet been confirmed as spent (in that a /// spending transaction known to the wallet has not yet been included in a block). @@ -236,12 +251,13 @@ pub trait WalletRead { /// decrypted and extracted from a [`CompactBlock`]. /// /// [`CompactBlock`]: crate::proto::compact_formats::CompactBlock -pub struct PrunedBlock<'a> { +pub struct PrunedBlock { pub block_height: BlockHeight, pub block_hash: BlockHash, pub block_time: u32, - pub commitment_tree: &'a sapling::CommitmentTree, - pub transactions: &'a Vec>, + pub transactions: Vec>, + pub sapling_commitment_tree_size: Option, + pub sapling_commitments: Vec<(sapling::Node, Retention)>, } /// A transaction that was detected during scanning of the blockchain, @@ -381,16 +397,14 @@ pub trait WalletWrite: WalletRead { account: AccountId, ) -> Result, Self::Error>; - /// Updates the state of the wallet database by persisting the provided - /// block information, along with the updated witness data that was - /// produced when scanning the block for transactions pertaining to - /// this wallet. + /// Updates the state of the wallet database by persisting the provided block information, + /// along with the note commitments that were detected when scanning the block for transactions + /// pertaining to this wallet. #[allow(clippy::type_complexity)] fn advance_by_block( &mut self, - block: &PrunedBlock, - updated_witnesses: &[(Self::NoteRef, sapling::IncrementalWitness)], - ) -> Result, Self::Error>; + block: PrunedBlock, + ) -> Result, Self::Error>; /// Caches a decrypted transaction in the persistent wallet store. fn store_decrypted_tx( @@ -424,10 +438,31 @@ pub trait WalletWrite: WalletRead { ) -> Result; } +pub trait WalletCommitmentTrees { + type Error; + type SaplingShardStore<'a>: ShardStore< + H = sapling::Node, + CheckpointId = BlockHeight, + Error = Self::Error, + >; + + fn with_sapling_tree_mut(&mut self, callback: F) -> Result + where + for<'a> F: FnMut( + &'a mut ShardTree< + Self::SaplingShardStore<'a>, + { sapling::NOTE_COMMITMENT_TREE_DEPTH }, + SAPLING_SHARD_HEIGHT, + >, + ) -> Result, + E: From>; +} + #[cfg(feature = "test-dependencies")] pub mod testing { use secrecy::{ExposeSecret, SecretVec}; - use std::collections::HashMap; + use shardtree::{MemoryShardStore, ShardTree, ShardTreeError}; + use std::{collections::HashMap, convert::Infallible, ops::Range}; use zcash_primitives::{ block::BlockHash, @@ -449,11 +484,26 @@ pub mod testing { }; use super::{ - DecryptedTransaction, NullifierQuery, PrunedBlock, SentTransaction, WalletRead, WalletWrite, + chain, DecryptedTransaction, NullifierQuery, PrunedBlock, SentTransaction, + WalletCommitmentTrees, WalletRead, WalletWrite, SAPLING_SHARD_HEIGHT, }; pub struct MockWalletDb { pub network: Network, + pub sapling_tree: ShardTree< + MemoryShardStore, + { SAPLING_SHARD_HEIGHT * 2 }, + SAPLING_SHARD_HEIGHT, + >, + } + + impl MockWalletDb { + pub fn new(network: Network) -> Self { + Self { + network, + sapling_tree: ShardTree::new(MemoryShardStore::empty(), 100), + } + } } impl WalletRead for MockWalletDb { @@ -465,6 +515,20 @@ pub mod testing { Ok(None) } + fn fully_scanned_height( + &self, + ) -> Result, Self::Error> { + Ok(None) + } + + fn suggest_scan_ranges( + &self, + _batch_size: usize, + _limit: usize, + ) -> Result>, Self::Error> { + Ok(vec![]) + } + fn get_min_unspent_height(&self) -> Result, Self::Error> { Ok(None) } @@ -524,21 +588,6 @@ pub mod testing { Err(()) } - fn get_commitment_tree( - &self, - _block_height: BlockHeight, - ) -> Result, Self::Error> { - Ok(None) - } - - #[allow(clippy::type_complexity)] - fn get_witnesses( - &self, - _block_height: BlockHeight, - ) -> Result, Self::Error> { - Ok(Vec::new()) - } - fn get_sapling_nullifiers( &self, _query: NullifierQuery, @@ -613,9 +662,8 @@ pub mod testing { #[allow(clippy::type_complexity)] fn advance_by_block( &mut self, - _block: &PrunedBlock, - _updated_witnesses: &[(Self::NoteRef, sapling::IncrementalWitness)], - ) -> Result, Self::Error> { + _block: PrunedBlock, + ) -> Result, Self::Error> { Ok(vec![]) } @@ -645,4 +693,23 @@ pub mod testing { Ok(0) } } + + impl WalletCommitmentTrees for MockWalletDb { + type Error = Infallible; + type SaplingShardStore<'a> = MemoryShardStore; + + fn with_sapling_tree_mut(&mut self, mut callback: F) -> Result + where + for<'a> F: FnMut( + &'a mut ShardTree< + Self::SaplingShardStore<'a>, + { sapling::NOTE_COMMITMENT_TREE_DEPTH }, + SAPLING_SHARD_HEIGHT, + >, + ) -> Result, + E: From>, + { + callback(&mut self.sapling_tree) + } + } } diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index 44736228d..ce0eb2a81 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -33,9 +33,7 @@ //! # fn test() -> Result<(), Error<(), Infallible, u32>> { //! let network = Network::TestNetwork; //! let block_source = chain_testing::MockBlockSource; -//! let mut db_data = testing::MockWalletDb { -//! network: Network::TestNetwork -//! }; +//! let mut db_data = testing::MockWalletDb::new(Network::TestNetwork); //! //! // 1) Download new CompactBlocks into block_source. //! @@ -79,7 +77,7 @@ //! // At this point, the cache and scanned data are locally consistent (though not //! // necessarily consistent with the latest chain tip - this would be discovered the //! // next time this codepath is executed after new blocks are received). -//! scan_cached_blocks(&network, &block_source, &mut db_data, None) +//! scan_cached_blocks(&network, &block_source, &mut db_data, None, None) //! # } //! # } //! ``` @@ -89,22 +87,34 @@ use std::convert::Infallible; use zcash_primitives::{ block::BlockHash, consensus::{self, BlockHeight}, - sapling::{self, note_encryption::PreparedIncomingViewingKey, Nullifier}, + sapling::{self, note_encryption::PreparedIncomingViewingKey}, zip32::Scope, }; use crate::{ - data_api::{PrunedBlock, WalletWrite}, + data_api::{NullifierQuery, WalletWrite}, proto::compact_formats::CompactBlock, scan::BatchRunner, - wallet::WalletTx, welding_rig::{add_block_to_runner, scan_block_with_runner}, }; pub mod error; use error::{ChainError, Error}; -use super::NullifierQuery; +pub struct CommitmentTreeMeta { + sapling_tree_size: u64, + //TODO: orchard_tree_size: u64 +} + +impl CommitmentTreeMeta { + pub fn from_parts(sapling_tree_size: u64) -> Self { + Self { sapling_tree_size } + } + + pub fn sapling_tree_size(&self) -> u64 { + self.sapling_tree_size + } +} /// This trait provides sequential access to raw blockchain data via a callback-oriented /// API. @@ -212,6 +222,7 @@ pub fn scan_cached_blocks( params: &ParamsT, block_source: &BlockSourceT, data_db: &mut DbT, + from_height: Option, limit: Option, ) -> Result<(), Error> where @@ -219,12 +230,6 @@ where BlockSourceT: BlockSource, DbT: WalletWrite, { - // Recall where we synced up to previously. - let mut last_height = data_db - .block_height_extrema() - .map_err(Error::Wallet)? - .map(|(_, max)| max); - // Fetch the UnifiedFullViewingKeys we are tracking let ufvks = data_db .get_unified_full_viewing_keys() @@ -236,25 +241,8 @@ where .filter_map(|(account, ufvk)| ufvk.sapling().map(move |k| (account, k))) .collect(); - // Get the most recent CommitmentTree - let mut tree = last_height.map_or_else( - || Ok(sapling::CommitmentTree::empty()), - |h| { - data_db - .get_commitment_tree(h) - .map(|t| t.unwrap_or_else(sapling::CommitmentTree::empty)) - .map_err(Error::Wallet) - }, - )?; - - // Get most recent incremental witnesses for the notes we are tracking - let mut witnesses = last_height.map_or_else( - || Ok(vec![]), - |h| data_db.get_witnesses(h).map_err(Error::Wallet), - )?; - - // Get the nullifiers for the notes we are tracking - let mut nullifiers = data_db + // Get the nullifiers for the unspent notes we are tracking + let mut sapling_nullifiers = data_db .get_sapling_nullifiers(NullifierQuery::Unspent) .map_err(Error::Wallet)?; @@ -271,8 +259,19 @@ where .map(|(tag, ivk)| (tag, PreparedIncomingViewingKey::new(&ivk))), ); + // Start at either the provided height, or where we synced up to previously. + let (last_scanned_height, commitment_tree_meta) = from_height.map_or_else( + || { + data_db.fully_scanned_height().map_or_else( + |e| Err(Error::Wallet(e)), + |next| Ok(next.map_or_else(|| (None, None), |(h, m)| (Some(h), Some(m)))), + ) + }, + |h| Ok((Some(h), None)), + )?; + block_source.with_blocks::<_, DbT::Error, DbT::NoteRef>( - last_height, + last_scanned_height, limit, |block: CompactBlock| { add_block_to_runner(params, block, &mut batch_runner); @@ -283,90 +282,35 @@ where batch_runner.flush(); block_source.with_blocks::<_, DbT::Error, DbT::NoteRef>( - last_height, + last_scanned_height, limit, |block: CompactBlock| { - let current_height = block.height(); - - // Scanned blocks MUST be height-sequential. - if let Some(h) = last_height { - if current_height != (h + 1) { - return Err( - ChainError::block_height_discontinuity(h + 1, current_height).into(), - ); - } - } - - let block_hash = BlockHash::from_slice(&block.hash); - let block_time = block.time; - - let txs: Vec> = { - let mut witness_refs: Vec<_> = witnesses.iter_mut().map(|w| &mut w.1).collect(); - - scan_block_with_runner( - params, - block, - &dfvks, - &nullifiers, - &mut tree, - &mut witness_refs[..], - Some(&mut batch_runner), - ) - }; - - // Enforce that all roots match. This is slow, so only include in debug builds. - #[cfg(debug_assertions)] - { - let cur_root = tree.root(); - for row in &witnesses { - if row.1.root() != cur_root { - return Err( - ChainError::invalid_witness_anchor(current_height, row.0).into() - ); - } - } - for tx in &txs { - for output in tx.sapling_outputs.iter() { - if output.witness().root() != cur_root { - return Err(ChainError::invalid_new_witness_anchor( - current_height, - tx.txid, - output.index(), - output.witness().root(), - ) - .into()); - } - } - } - } - - let new_witnesses = data_db - .advance_by_block( - &(PrunedBlock { - block_height: current_height, - block_hash, - block_time, - commitment_tree: &tree, - transactions: &txs, - }), - &witnesses, - ) - .map_err(Error::Wallet)?; - - let spent_nf: Vec<&Nullifier> = txs + let pruned_block = scan_block_with_runner( + params, + block, + &dfvks, + &sapling_nullifiers, + commitment_tree_meta.as_ref(), + Some(&mut batch_runner), + ) + .map_err(Error::Sync)?; + + let spent_nf: Vec<&sapling::Nullifier> = pruned_block + .transactions .iter() .flat_map(|tx| tx.sapling_spends.iter().map(|spend| spend.nf())) .collect(); - nullifiers.retain(|(_, nf)| !spent_nf.contains(&nf)); - nullifiers.extend(txs.iter().flat_map(|tx| { + + sapling_nullifiers.retain(|(_, nf)| !spent_nf.contains(&nf)); + sapling_nullifiers.extend(pruned_block.transactions.iter().flat_map(|tx| { tx.sapling_outputs .iter() .map(|out| (out.account(), *out.nf())) })); - witnesses.extend(new_witnesses); - - last_height = Some(current_height); + data_db + .advance_by_block(pruned_block) + .map_err(Error::Wallet)?; Ok(()) }, diff --git a/zcash_client_backend/src/data_api/chain/error.rs b/zcash_client_backend/src/data_api/chain/error.rs index b35334c6a..b28380d13 100644 --- a/zcash_client_backend/src/data_api/chain/error.rs +++ b/zcash_client_backend/src/data_api/chain/error.rs @@ -5,6 +5,8 @@ use std::fmt::{self, Debug, Display}; use zcash_primitives::{consensus::BlockHeight, sapling, transaction::TxId}; +use crate::welding_rig::SyncError; + /// The underlying cause of a [`ChainError`]. #[derive(Copy, Clone, Debug)] pub enum Cause { @@ -142,6 +144,9 @@ pub enum Error { /// commitments that could not be reconciled with the note commitment tree(s) maintained by the /// wallet. Chain(ChainError), + + /// An error occorred in block scanning. + Sync(SyncError), } impl fmt::Display for Error { @@ -164,6 +169,13 @@ impl fmt::Display for Error { write!(f, "{}", err) } + Error::Sync(SyncError::SaplingTreeSizeUnknown(h)) => { + write!( + f, + "Sync failed due to missing Sapling note commitment tree size at height {}", + h + ) + } } } } diff --git a/zcash_client_backend/src/data_api/error.rs b/zcash_client_backend/src/data_api/error.rs index 0614a612d..db4ffb984 100644 --- a/zcash_client_backend/src/data_api/error.rs +++ b/zcash_client_backend/src/data_api/error.rs @@ -1,5 +1,6 @@ //! Types for wallet error handling. +use shardtree::ShardTreeError; use std::error; use std::fmt::{self, Debug, Display}; use zcash_primitives::{ @@ -20,10 +21,13 @@ use zcash_primitives::{legacy::TransparentAddress, zip32::DiversifierIndex}; /// Errors that can occur as a consequence of wallet operations. #[derive(Debug)] -pub enum Error { +pub enum Error { /// An error occurred retrieving data from the underlying data source DataSource(DataSourceError), + /// An error in computations involving the note commitment trees. + CommitmentTree(ShardTreeError), + /// An error in note selection NoteSelection(SelectionError), @@ -60,9 +64,10 @@ pub enum Error { ChildIndexOutOfRange(DiversifierIndex), } -impl fmt::Display for Error +impl fmt::Display for Error where DE: fmt::Display, + CE: fmt::Display, SE: fmt::Display, FE: fmt::Display, N: fmt::Display, @@ -76,6 +81,9 @@ where e ) } + Error::CommitmentTree(e) => { + write!(f, "An error occurred in querying or updating a note commitment tree: {}", e) + } Error::NoteSelection(e) => { write!(f, "Note selection encountered the following error: {}", e) } @@ -120,9 +128,10 @@ where } } -impl error::Error for Error +impl error::Error for Error where DE: Debug + Display + error::Error + 'static, + CE: Debug + Display + error::Error + 'static, SE: Debug + Display + error::Error + 'static, FE: Debug + Display + 'static, N: Debug + Display, @@ -130,6 +139,7 @@ where fn source(&self) -> Option<&(dyn error::Error + 'static)> { match &self { Error::DataSource(e) => Some(e), + Error::CommitmentTree(e) => Some(e), Error::NoteSelection(e) => Some(e), Error::Builder(e) => Some(e), _ => None, @@ -137,19 +147,19 @@ where } } -impl From> for Error { +impl From> for Error { fn from(e: builder::Error) -> Self { Error::Builder(e) } } -impl From for Error { +impl From for Error { fn from(e: BalanceError) -> Self { Error::BalanceError(e) } } -impl From> for Error { +impl From> for Error { fn from(e: InputSelectorError) -> Self { match e { InputSelectorError::DataSource(e) => Error::DataSource(e), @@ -161,18 +171,25 @@ impl From> for Error { available, required, }, + InputSelectorError::SyncRequired => Error::ScanRequired, } } } -impl From for Error { +impl From for Error { fn from(e: sapling::builder::Error) -> Self { Error::Builder(builder::Error::SaplingBuild(e)) } } -impl From for Error { +impl From for Error { fn from(e: transparent::builder::Error) -> Self { Error::Builder(builder::Error::TransparentBuild(e)) } } + +impl From> for Error { + fn from(e: ShardTreeError) -> Self { + Error::CommitmentTree(e) + } +} diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index 160529dc6..b0930d966 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -1,8 +1,9 @@ use std::convert::Infallible; use std::fmt::Debug; +use shardtree::{ShardStore, ShardTree, ShardTreeError}; use zcash_primitives::{ - consensus::{self, NetworkUpgrade}, + consensus::{self, BlockHeight, NetworkUpgrade}, memo::MemoBytes, sapling::{ self, @@ -23,7 +24,8 @@ use crate::{ address::RecipientAddress, data_api::{ error::Error, wallet::input_selection::Proposal, DecryptedTransaction, PoolType, Recipient, - SentTransaction, SentTransactionOutput, WalletWrite, + SentTransaction, SentTransactionOutput, WalletCommitmentTrees, WalletRead, WalletWrite, + SAPLING_SHARD_HEIGHT, }, decrypt_transaction, fees::{self, ChangeValue, DustOutputPolicy}, @@ -122,7 +124,7 @@ where /// # Examples /// /// ``` -/// # #[cfg(feature = "test-dependencies")] +/// # #[cfg(all(feature = "test-dependencies", feature = "local-prover"))] /// # { /// use tempfile::NamedTempFile; /// use zcash_primitives::{ @@ -200,7 +202,8 @@ pub fn create_spend_to_address( ) -> Result< DbT::TxRef, Error< - DbT::Error, + ::Error, + ::Error, GreedyInputSelectorError, Infallible, DbT::NoteRef, @@ -208,7 +211,7 @@ pub fn create_spend_to_address( > where ParamsT: consensus::Parameters + Clone, - DbT: WalletWrite, + DbT: WalletWrite + WalletCommitmentTrees, DbT::NoteRef: Copy + Eq + Ord, { let req = zip321::TransactionRequest::new(vec![Payment { @@ -300,10 +303,16 @@ pub fn spend( min_confirmations: u32, ) -> Result< DbT::TxRef, - Error::Error, DbT::NoteRef>, + Error< + ::Error, + ::Error, + InputsT::Error, + ::Error, + DbT::NoteRef, + >, > where - DbT: WalletWrite, + DbT: WalletWrite + WalletCommitmentTrees, DbT::TxRef: Copy + Debug, DbT::NoteRef: Copy + Eq + Ord, ParamsT: consensus::Parameters + Clone, @@ -323,7 +332,16 @@ where min_confirmations, )?; - create_proposed_transaction(wallet_db, params, prover, usk, ovk_policy, proposal, None) + create_proposed_transaction( + wallet_db, + params, + prover, + usk, + ovk_policy, + proposal, + min_confirmations, + None, + ) } /// Select transaction inputs, compute fees, and construct a proposal for a transaction @@ -331,7 +349,7 @@ where /// [`create_proposed_transaction`]. #[allow(clippy::too_many_arguments)] #[allow(clippy::type_complexity)] -pub fn propose_transfer( +pub fn propose_transfer( wallet_db: &mut DbT, params: &ParamsT, spend_from_account: AccountId, @@ -340,7 +358,13 @@ pub fn propose_transfer( min_confirmations: u32, ) -> Result< Proposal, - Error::Error, DbT::NoteRef>, + Error< + DbT::Error, + CommitmentTreeErrT, + InputsT::Error, + ::Error, + DbT::NoteRef, + >, > where DbT: WalletWrite, @@ -348,20 +372,13 @@ where ParamsT: consensus::Parameters + Clone, InputsT: InputSelector, { - // Target the next block, assuming we are up-to-date. - let (target_height, anchor_height) = wallet_db - .get_target_and_anchor_heights(min_confirmations) - .map_err(Error::DataSource) - .and_then(|x| x.ok_or(Error::ScanRequired))?; - input_selector .propose_transaction( params, wallet_db, spend_from_account, - anchor_height, - target_height, request, + min_confirmations, ) .map_err(Error::from) } @@ -369,7 +386,7 @@ where #[cfg(feature = "transparent-inputs")] #[allow(clippy::too_many_arguments)] #[allow(clippy::type_complexity)] -pub fn propose_shielding( +pub fn propose_shielding( wallet_db: &mut DbT, params: &ParamsT, input_selector: &InputsT, @@ -378,7 +395,13 @@ pub fn propose_shielding( min_confirmations: u32, ) -> Result< Proposal, - Error::Error, DbT::NoteRef>, + Error< + DbT::Error, + CommitmentTreeErrT, + InputsT::Error, + ::Error, + DbT::NoteRef, + >, > where ParamsT: consensus::Parameters, @@ -386,19 +409,13 @@ where DbT::NoteRef: Copy + Eq + Ord, InputsT: InputSelector, { - let (target_height, latest_anchor) = wallet_db - .get_target_and_anchor_heights(min_confirmations) - .map_err(Error::DataSource) - .and_then(|x| x.ok_or(Error::ScanRequired))?; - input_selector .propose_shielding( params, wallet_db, shielding_threshold, from_addrs, - latest_anchor, - target_height, + min_confirmations, ) .map_err(Error::from) } @@ -417,10 +434,20 @@ pub fn create_proposed_transaction( usk: &UnifiedSpendingKey, ovk_policy: OvkPolicy, proposal: Proposal, + min_confirmations: u32, change_memo: Option, -) -> Result> +) -> Result< + DbT::TxRef, + Error< + ::Error, + ::Error, + InputsErrT, + FeeRuleT::Error, + DbT::NoteRef, + >, +> where - DbT: WalletWrite, + DbT: WalletWrite + WalletCommitmentTrees, DbT::TxRef: Copy + Debug, DbT::NoteRef: Copy + Eq + Ord, ParamsT: consensus::Parameters + Clone, @@ -459,14 +486,25 @@ where // Create the transaction. The type of the proposal ensures that there // are no possible transparent inputs, so we ignore those - let mut builder = Builder::new(params.clone(), proposal.target_height(), None); - - for selected in proposal.sapling_inputs() { - let (note, key, merkle_path) = select_key_for_note(selected, usk.sapling(), &dfvk) + let mut builder = Builder::new(params.clone(), proposal.min_target_height(), None); + + wallet_db.with_sapling_tree_mut::<_, _, Error<_, _, _, _, _>>(|sapling_tree| { + for selected in proposal.sapling_inputs() { + let (note, key, merkle_path) = select_key_for_note( + sapling_tree, + selected, + usk.sapling(), + &dfvk, + min_confirmations + .try_into() + .expect("min_confirmations should never be anywhere close to usize::MAX"), + )? .ok_or(Error::NoteMismatch(selected.note_id))?; - builder.add_sapling_spend(key, selected.diversifier, note, merkle_path)?; - } + builder.add_sapling_spend(key, selected.diversifier, note, merkle_path)?; + } + Ok(()) + })?; #[cfg(feature = "transparent-inputs")] let utxos = { @@ -577,7 +615,7 @@ where tx.sapling_bundle().and_then(|bundle| { try_sapling_note_decryption( params, - proposal.target_height(), + proposal.min_target_height(), &internal_ivk, &bundle.shielded_outputs()[output_index], ) @@ -672,11 +710,17 @@ pub fn shield_transparent_funds( min_confirmations: u32, ) -> Result< DbT::TxRef, - Error::Error, DbT::NoteRef>, + Error< + ::Error, + ::Error, + InputsT::Error, + ::Error, + DbT::NoteRef, + >, > where ParamsT: consensus::Parameters, - DbT: WalletWrite, + DbT: WalletWrite + WalletCommitmentTrees, DbT::NoteRef: Copy + Eq + Ord, InputsT: InputSelector, { @@ -696,17 +740,26 @@ where usk, OvkPolicy::Sender, proposal, + min_confirmations, Some(memo.clone()), ) } -fn select_key_for_note( +#[allow(clippy::type_complexity)] +fn select_key_for_note>( + commitment_tree: &mut ShardTree< + S, + { sapling::NOTE_COMMITMENT_TREE_DEPTH }, + SAPLING_SHARD_HEIGHT, + >, selected: &ReceivedSaplingNote, extsk: &ExtendedSpendingKey, dfvk: &DiversifiableFullViewingKey, -) -> Option<(sapling::Note, ExtendedSpendingKey, sapling::MerklePath)> { - let merkle_path = selected.witness.path().expect("the tree is not empty"); - + checkpoint_depth: usize, +) -> Result< + Option<(sapling::Note, ExtendedSpendingKey, sapling::MerklePath)>, + ShardTreeError, +> { // Attempt to reconstruct the note being spent using both the internal and external dfvks // corresponding to the unified spending key, checking against the witness we are using // to spend the note that we've used the correct key. @@ -717,13 +770,16 @@ fn select_key_for_note( .diversified_change_address(selected.diversifier) .map(|addr| addr.create_note(selected.note_value.into(), selected.rseed)); - let expected_root = selected.witness.root(); - external_note + let expected_root = commitment_tree.root_at_checkpoint(checkpoint_depth)?; + let merkle_path = commitment_tree + .witness_caching(selected.note_commitment_tree_position, checkpoint_depth)?; + + Ok(external_note .filter(|n| expected_root == merkle_path.root(Node::from_cmu(&n.cmu()))) .map(|n| (n, extsk.clone(), merkle_path.clone())) .or_else(|| { internal_note .filter(|n| expected_root == merkle_path.root(Node::from_cmu(&n.cmu()))) .map(|n| (n, extsk.derive_internal(), merkle_path)) - }) + })) } diff --git a/zcash_client_backend/src/data_api/wallet/input_selection.rs b/zcash_client_backend/src/data_api/wallet/input_selection.rs index 798b83450..403497c0d 100644 --- a/zcash_client_backend/src/data_api/wallet/input_selection.rs +++ b/zcash_client_backend/src/data_api/wallet/input_selection.rs @@ -35,6 +35,9 @@ pub enum InputSelectorError { /// Insufficient funds were available to satisfy the payment request that inputs were being /// selected to attempt to satisfy. InsufficientFunds { available: Amount, required: Amount }, + /// The data source does not have enough information to choose an expiry height + /// for the transaction. + SyncRequired, } impl fmt::Display for InputSelectorError { @@ -59,6 +62,7 @@ impl fmt::Display for InputSelectorError write!(f, "No chain data is available."), } } } @@ -71,7 +75,8 @@ pub struct Proposal { sapling_inputs: Vec>, balance: TransactionBalance, fee_rule: FeeRuleT, - target_height: BlockHeight, + min_target_height: BlockHeight, + min_anchor_height: BlockHeight, is_shielding: bool, } @@ -97,8 +102,19 @@ impl Proposal { &self.fee_rule } /// Returns the target height for which the proposal was prepared. - pub fn target_height(&self) -> BlockHeight { - self.target_height + /// + /// The chain must contain at least this many blocks in order for the proposal to + /// be executed. + pub fn min_target_height(&self) -> BlockHeight { + self.min_target_height + } + /// Returns the anchor height used in preparing the proposal. + /// + /// If, at the time that the proposal is executed, the anchor height required to satisfy + /// the minimum confirmation depth is less than this height, the proposal execution + /// API should return an error. + pub fn min_anchor_height(&self) -> BlockHeight { + self.min_anchor_height } /// Returns a flag indicating whether or not the proposed transaction /// is exclusively wallet-internal (if it does not involve any external @@ -146,9 +162,8 @@ pub trait InputSelector { params: &ParamsT, wallet_db: &Self::DataSource, account: AccountId, - anchor_height: BlockHeight, - target_height: BlockHeight, transaction_request: TransactionRequest, + min_confirmations: u32, ) -> Result< Proposal::DataSource as WalletRead>::NoteRef>, InputSelectorError<<::DataSource as WalletRead>::Error, Self::Error>, @@ -172,8 +187,7 @@ pub trait InputSelector { wallet_db: &Self::DataSource, shielding_threshold: NonNegativeAmount, source_addrs: &[TransparentAddress], - confirmed_height: BlockHeight, - target_height: BlockHeight, + min_confirmations: u32, ) -> Result< Proposal::DataSource as WalletRead>::NoteRef>, InputSelectorError<<::DataSource as WalletRead>::Error, Self::Error>, @@ -292,13 +306,18 @@ where params: &ParamsT, wallet_db: &Self::DataSource, account: AccountId, - anchor_height: BlockHeight, - target_height: BlockHeight, transaction_request: TransactionRequest, + min_confirmations: u32, ) -> Result, InputSelectorError> where ParamsT: consensus::Parameters, { + // Target the next block, assuming we are up-to-date. + let (target_height, anchor_height) = wallet_db + .get_target_and_anchor_heights(min_confirmations) + .map_err(InputSelectorError::DataSource) + .and_then(|x| x.ok_or(InputSelectorError::SyncRequired))?; + let mut transparent_outputs = vec![]; let mut sapling_outputs = vec![]; let mut output_total = Amount::zero(); @@ -362,7 +381,8 @@ where sapling_inputs, balance, fee_rule: (*self.change_strategy.fee_rule()).clone(), - target_height, + min_target_height: target_height, + min_anchor_height: anchor_height, is_shielding: false, }); } @@ -405,15 +425,19 @@ where wallet_db: &Self::DataSource, shielding_threshold: NonNegativeAmount, source_addrs: &[TransparentAddress], - confirmed_height: BlockHeight, - target_height: BlockHeight, + min_confirmations: u32, ) -> Result, InputSelectorError> where ParamsT: consensus::Parameters, { + let (target_height, latest_anchor) = wallet_db + .get_target_and_anchor_heights(min_confirmations) + .map_err(InputSelectorError::DataSource) + .and_then(|x| x.ok_or(InputSelectorError::SyncRequired))?; + let mut transparent_inputs: Vec = source_addrs .iter() - .map(|taddr| wallet_db.get_unspent_transparent_outputs(taddr, confirmed_height, &[])) + .map(|taddr| wallet_db.get_unspent_transparent_outputs(taddr, latest_anchor, &[])) .collect::>, _>>() .map_err(InputSelectorError::DataSource)? .into_iter() @@ -458,7 +482,8 @@ where sapling_inputs: vec![], balance, fee_rule: (*self.change_strategy.fee_rule()).clone(), - target_height, + min_target_height: target_height, + min_anchor_height: latest_anchor, is_shielding: true, }) } else { diff --git a/zcash_client_backend/src/wallet.rs b/zcash_client_backend/src/wallet.rs index ba58340b3..c702cfa73 100644 --- a/zcash_client_backend/src/wallet.rs +++ b/zcash_client_backend/src/wallet.rs @@ -1,6 +1,7 @@ //! Structs representing transaction data scanned from the block chain by a wallet or //! light client. +use incrementalmerkletree::Position; use zcash_note_encryption::EphemeralKeyBytes; use zcash_primitives::{ consensus::BlockHeight, @@ -117,7 +118,7 @@ pub struct WalletSaplingOutput { account: AccountId, note: sapling::Note, is_change: bool, - witness: sapling::IncrementalWitness, + note_commitment_tree_position: Position, nf: N, } @@ -131,7 +132,7 @@ impl WalletSaplingOutput { account: AccountId, note: sapling::Note, is_change: bool, - witness: sapling::IncrementalWitness, + note_commitment_tree_position: Position, nf: N, ) -> Self { Self { @@ -141,7 +142,7 @@ impl WalletSaplingOutput { account, note, is_change, - witness, + note_commitment_tree_position, nf, } } @@ -164,11 +165,8 @@ impl WalletSaplingOutput { pub fn is_change(&self) -> bool { self.is_change } - pub fn witness(&self) -> &sapling::IncrementalWitness { - &self.witness - } - pub fn witness_mut(&mut self) -> &mut sapling::IncrementalWitness { - &mut self.witness + pub fn note_commitment_tree_position(&self) -> Position { + self.note_commitment_tree_position } pub fn nf(&self) -> &N { &self.nf @@ -182,7 +180,7 @@ pub struct ReceivedSaplingNote { pub diversifier: sapling::Diversifier, pub note_value: Amount, pub rseed: sapling::Rseed, - pub witness: sapling::IncrementalWitness, + pub note_commitment_tree_position: Position, } impl sapling_fees::InputView for ReceivedSaplingNote { diff --git a/zcash_client_backend/src/welding_rig.rs b/zcash_client_backend/src/welding_rig.rs index 1906c133e..3266d4059 100644 --- a/zcash_client_backend/src/welding_rig.rs +++ b/zcash_client_backend/src/welding_rig.rs @@ -1,21 +1,27 @@ //! Tools for scanning a compact representation of the Zcash block chain. +//! +//! TODO: rename this module to `block_scanner` use std::collections::{HashMap, HashSet}; use std::convert::TryFrom; +use incrementalmerkletree::{Position, Retention}; use subtle::{ConditionallySelectable, ConstantTimeEq, CtOption}; use zcash_note_encryption::batch; +use zcash_primitives::consensus::BlockHeight; use zcash_primitives::{ consensus, sapling::{ self, note_encryption::{PreparedIncomingViewingKey, SaplingDomain}, - Node, Note, Nullifier, NullifierDerivingKey, SaplingIvk, + SaplingIvk, }, transaction::components::sapling::CompactOutputDescription, zip32::{sapling::DiversifiableFullViewingKey, AccountId, Scope}, }; +use crate::data_api::chain::CommitmentTreeMeta; +use crate::data_api::PrunedBlock; use crate::{ proto::compact_formats::CompactBlock, scan::{Batch, BatchRunner, Tasks}, @@ -56,16 +62,13 @@ pub trait ScanningKey { /// IVK-based implementations of this trait cannot successfully derive /// nullifiers, in which case `Self::Nf` should be set to the unit type /// and this function is a no-op. - fn sapling_nf( - key: &Self::SaplingNk, - note: &Note, - witness: &sapling::IncrementalWitness, - ) -> Self::Nf; + fn sapling_nf(key: &Self::SaplingNk, note: &sapling::Note, note_position: Position) + -> Self::Nf; } impl ScanningKey for DiversifiableFullViewingKey { type Scope = Scope; - type SaplingNk = NullifierDerivingKey; + type SaplingNk = sapling::NullifierDerivingKey; type SaplingKeys = [(Self::Scope, SaplingIvk, Self::SaplingNk); 2]; type Nf = sapling::Nullifier; @@ -84,16 +87,8 @@ impl ScanningKey for DiversifiableFullViewingKey { ] } - fn sapling_nf( - key: &Self::SaplingNk, - note: &Note, - witness: &sapling::IncrementalWitness, - ) -> Self::Nf { - note.nf( - key, - u64::try_from(witness.position()) - .expect("Sapling note commitment tree position must fit into a u64"), - ) + fn sapling_nf(key: &Self::SaplingNk, note: &sapling::Note, position: Position) -> Self::Nf { + note.nf(key, position.into()) } } @@ -111,7 +106,15 @@ impl ScanningKey for SaplingIvk { [((), self.clone(), ())] } - fn sapling_nf(_key: &Self::SaplingNk, _note: &Note, _witness: &sapling::IncrementalWitness) {} + fn sapling_nf(_key: &Self::SaplingNk, _note: &sapling::Note, _position: Position) {} +} + +/// Errors that can occur in block scanning. +#[derive(Debug)] +pub enum SyncError { + /// The size of the Sapling note commitment tree was not provided as part of a [`CompactBlock`] + /// being scanned, making it impossible to construct the nullifier for a detected note. + SaplingTreeSizeUnknown(BlockHeight), } /// Scans a [`CompactBlock`] with a set of [`ScanningKey`]s. @@ -141,17 +144,15 @@ pub fn scan_block( params: &P, block: CompactBlock, vks: &[(&AccountId, &K)], - nullifiers: &[(AccountId, Nullifier)], - tree: &mut sapling::CommitmentTree, - existing_witnesses: &mut [&mut sapling::IncrementalWitness], -) -> Vec> { + sapling_nullifiers: &[(AccountId, sapling::Nullifier)], + initial_commitment_tree_meta: Option<&CommitmentTreeMeta>, +) -> Result, SyncError> { scan_block_with_runner::<_, _, ()>( params, block, vks, - nullifiers, - tree, - existing_witnesses, + sapling_nullifiers, + initial_commitment_tree_meta, None, ) } @@ -202,21 +203,41 @@ pub(crate) fn scan_block_with_runner< params: &P, block: CompactBlock, vks: &[(&AccountId, &K)], - nullifiers: &[(AccountId, Nullifier)], - tree: &mut sapling::CommitmentTree, - existing_witnesses: &mut [&mut sapling::IncrementalWitness], + nullifiers: &[(AccountId, sapling::Nullifier)], + initial_commitment_tree_meta: Option<&CommitmentTreeMeta>, mut batch_runner: Option<&mut TaggedBatchRunner>, -) -> Vec> { +) -> Result, SyncError> { let mut wtxs: Vec> = vec![]; + let mut sapling_note_commitments: Vec<(sapling::Node, Retention)> = vec![]; let block_height = block.height(); let block_hash = block.hash(); + // It's possible to make progress without a Sapling tree position if we don't have any Sapling + // notes in the block, since we only use the position for constructing nullifiers for our own + // received notes. Thus, we allow it to be optional here, and only produce an error if we try + // to use it. `block.sapling_commitment_tree_size` is expected to be correct as of the end of + // the block, and we can't have a note of ours in a block with no outputs so treating the zero + // default value from the protobuf as `None` is always correct. + let mut sapling_tree_position = if block.sapling_commitment_tree_size == 0 { + initial_commitment_tree_meta.map(|m| (m.sapling_tree_size() + 1).into()) + } else { + let end_position_exclusive = Position::from(u64::from(block.sapling_commitment_tree_size)); + let output_count = block + .vtx + .iter() + .map(|tx| u64::try_from(tx.outputs.len()).unwrap()) + .sum(); + Some(end_position_exclusive - output_count) + }; + for tx in block.vtx.into_iter() { let txid = tx.txid(); let index = tx.index as usize; - // Check for spent notes - // The only step that is not constant-time is the filter() at the end. + // Check for spent notes. The only step that is not constant-time is + // the filter() at the end. + // TODO: However, this is O(|nullifiers| * |notes|); does using + // constant-time operations here really make sense? let shielded_spends: Vec<_> = tx .spends .into_iter() @@ -248,19 +269,8 @@ pub(crate) fn scan_block_with_runner< // Check for incoming notes while incrementing tree and witnesses let mut shielded_outputs: Vec> = vec![]; + let tx_outputs_len = u64::try_from(tx.outputs.len()).unwrap(); { - // Grab mutable references to new witnesses from previous transactions - // in this block so that we can update them. Scoped so we don't hold - // mutable references to wtxs for too long. - let mut block_witnesses: Vec<_> = wtxs - .iter_mut() - .flat_map(|tx| { - tx.sapling_outputs - .iter_mut() - .map(|output| output.witness_mut()) - }) - .collect(); - let decoded = &tx .outputs .into_iter() @@ -292,7 +302,7 @@ pub(crate) fn scan_block_with_runner< "The batch runner and scan_block must use the same set of IVKs.", ); - ((d_note.note, d_note.recipient), a, (*nk).clone()) + (d_note.note, a, (*nk).clone()) }) }) .collect() @@ -312,40 +322,21 @@ pub(crate) fn scan_block_with_runner< .map(PreparedIncomingViewingKey::new) .collect::>(); - batch::try_compact_note_decryption(&ivks, decoded) + batch::try_compact_note_decryption(&ivks, &decoded[..]) .into_iter() .map(|v| { - v.map(|(note_data, ivk_idx)| { + v.map(|((note, _), ivk_idx)| { let (account, _, nk) = &vks[ivk_idx]; - (note_data, *account, (*nk).clone()) + (note, *account, (*nk).clone()) }) }) .collect() }; for (index, ((_, output), dec_output)) in decoded.iter().zip(decrypted).enumerate() { - // Grab mutable references to new witnesses from previous outputs - // in this transaction so that we can update them. Scoped so we - // don't hold mutable references to shielded_outputs for too long. - let new_witnesses: Vec<_> = shielded_outputs - .iter_mut() - .map(|out| out.witness_mut()) - .collect(); - - // Increment tree and witnesses - let node = Node::from_cmu(&output.cmu); - for witness in &mut *existing_witnesses { - witness.append(node).unwrap(); - } - for witness in &mut block_witnesses { - witness.append(node).unwrap(); - } - for witness in new_witnesses { - witness.append(node).unwrap(); - } - tree.append(node).unwrap(); - - if let Some(((note, _), account, nk)) = dec_output { + // Collect block note commitments + let node = sapling::Node::from_cmu(&output.cmu); + if let Some((note, account, nk)) = dec_output { // A note is marked as "change" if the account that received it // also spent notes in the same transaction. This will catch, // for instance: @@ -353,8 +344,10 @@ pub(crate) fn scan_block_with_runner< // - Notes created by consolidation transactions. // - Notes sent from one account to itself. let is_change = spent_from_accounts.contains(&account); - let witness = sapling::IncrementalWitness::from_tree(tree.clone()); - let nf = K::sapling_nf(&nk, ¬e, &witness); + let note_commitment_tree_position = sapling_tree_position + .ok_or(SyncError::SaplingTreeSizeUnknown(block_height))? + + index.try_into().unwrap(); + let nf = K::sapling_nf(&nk, ¬e, note_commitment_tree_position); shielded_outputs.push(WalletSaplingOutput::from_parts( index, @@ -363,9 +356,33 @@ pub(crate) fn scan_block_with_runner< account, note, is_change, - witness, + note_commitment_tree_position, nf, - )) + )); + + sapling_note_commitments.push(( + node, + if index == decoded.len() - 1 { + Retention::Checkpoint { + id: block_height, + is_marked: true, + } + } else { + Retention::Marked + }, + )); + } else { + sapling_note_commitments.push(( + node, + if index == decoded.len() - 1 { + Retention::Checkpoint { + id: block_height, + is_marked: false, + } + } else { + Retention::Ephemeral + }, + )); } } } @@ -378,9 +395,22 @@ pub(crate) fn scan_block_with_runner< sapling_outputs: shielded_outputs, }); } + + sapling_tree_position = sapling_tree_position.map(|pos| pos + tx_outputs_len); } - wtxs + Ok(PrunedBlock { + block_height, + block_hash, + block_time: block.time, + transactions: wtxs, + sapling_commitment_tree_size: if block.sapling_commitment_tree_size == 0 { + None + } else { + Some(block.sapling_commitment_tree_size) + }, + sapling_commitments: sapling_note_commitments, + }) } #[cfg(test)] @@ -396,16 +426,18 @@ mod tests { constants::SPENDING_KEY_GENERATOR, memo::MemoBytes, sapling::{ + self, note_encryption::{sapling_note_encryption, PreparedIncomingViewingKey, SaplingDomain}, util::generate_random_rseed, value::NoteValue, - CommitmentTree, Note, Nullifier, SaplingIvk, + Nullifier, SaplingIvk, }, transaction::components::Amount, zip32::{AccountId, DiversifiableFullViewingKey, ExtendedSpendingKey}, }; use crate::{ + data_api::chain::CommitmentTreeMeta, proto::compact_formats::{ CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, }, @@ -455,13 +487,14 @@ mod tests { dfvk: &DiversifiableFullViewingKey, value: Amount, tx_after: bool, + initial_sapling_tree_size: u32, ) -> CompactBlock { let to = dfvk.default_address().1; // Create a fake Note for the account let mut rng = OsRng; let rseed = generate_random_rseed(&Network::TestNetwork, height, &mut rng); - let note = Note::from_parts(to, NoteValue::from_raw(value.into()), rseed); + let note = sapling::Note::from_parts(to, NoteValue::from_raw(value.into()), rseed); let encryptor = sapling_note_encryption::<_, Network>( Some(dfvk.fvk().ovk), note.clone(), @@ -514,6 +547,9 @@ mod tests { cb.vtx.push(tx); } + cb.sapling_commitment_tree_size = initial_sapling_tree_size + + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(); + cb } @@ -530,10 +566,10 @@ mod tests { &dfvk, Amount::from_u64(5).unwrap(), false, + 0, ); assert_eq!(cb.vtx.len(), 2); - let mut tree = CommitmentTree::empty(); let mut batch_runner = if scan_multithreaded { let mut runner = BatchRunner::<_, _, _, ()>::new( 10, @@ -551,15 +587,16 @@ mod tests { None }; - let txs = scan_block_with_runner( + let pruned_block = scan_block_with_runner( &Network::TestNetwork, cb, &[(&account, &dfvk)], &[], - &mut tree, - &mut [], + Some(&CommitmentTreeMeta::from_parts(0)), batch_runner.as_mut(), - ); + ) + .unwrap(); + let txs = pruned_block.transactions; assert_eq!(txs.len(), 1); let tx = &txs[0]; @@ -569,9 +606,6 @@ mod tests { assert_eq!(tx.sapling_outputs[0].index(), 0); assert_eq!(tx.sapling_outputs[0].account(), account); assert_eq!(tx.sapling_outputs[0].note().value().inner(), 5); - - // Check that the witness root matches - assert_eq!(tx.sapling_outputs[0].witness().root(), tree.root()); } go(false); @@ -591,10 +625,10 @@ mod tests { &dfvk, Amount::from_u64(5).unwrap(), true, + 0, ); assert_eq!(cb.vtx.len(), 3); - let mut tree = CommitmentTree::empty(); let mut batch_runner = if scan_multithreaded { let mut runner = BatchRunner::<_, _, _, ()>::new( 10, @@ -612,15 +646,16 @@ mod tests { None }; - let txs = scan_block_with_runner( + let pruned_block = scan_block_with_runner( &Network::TestNetwork, cb, &[(&AccountId::from(0), &dfvk)], &[], - &mut tree, - &mut [], + Some(&CommitmentTreeMeta::from_parts(0)), batch_runner.as_mut(), - ); + ) + .unwrap(); + let txs = pruned_block.transactions; assert_eq!(txs.len(), 1); let tx = &txs[0]; @@ -630,9 +665,6 @@ mod tests { assert_eq!(tx.sapling_outputs[0].index(), 0); assert_eq!(tx.sapling_outputs[0].account(), AccountId::from(0)); assert_eq!(tx.sapling_outputs[0].note().value().inner(), 5); - - // Check that the witness root matches - assert_eq!(tx.sapling_outputs[0].witness().root(), tree.root()); } go(false); @@ -646,19 +678,26 @@ mod tests { let nf = Nullifier([7; 32]); let account = AccountId::from(12); - let cb = fake_compact_block(1u32.into(), nf, &dfvk, Amount::from_u64(5).unwrap(), false); + let cb = fake_compact_block( + 1u32.into(), + nf, + &dfvk, + Amount::from_u64(5).unwrap(), + false, + 0, + ); assert_eq!(cb.vtx.len(), 2); let vks: Vec<(&AccountId, &SaplingIvk)> = vec![]; - let mut tree = CommitmentTree::empty(); - let txs = scan_block( + let pruned_block = scan_block( &Network::TestNetwork, cb, &vks[..], &[(account, nf)], - &mut tree, - &mut [], - ); + Some(&CommitmentTreeMeta::from_parts(0)), + ) + .unwrap(); + let txs = pruned_block.transactions; assert_eq!(txs.len(), 1); let tx = &txs[0]; diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index de71727b3..88585e0d2 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -15,6 +15,7 @@ rust-version = "1.65" [dependencies] incrementalmerkletree = { version = "0.4", features = ["legacy-api"] } +shardtree = { version = "0.0", features = ["legacy-api"] } zcash_client_backend = { version = "0.9", path = "../zcash_client_backend" } zcash_primitives = { version = "0.12", path = "../zcash_primitives", default-features = false } diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index 81a0e028a..11e065f9e 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -314,6 +314,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(5).unwrap(), + 0, ); insert_into_cache(&db_cache, &cb); @@ -328,7 +329,7 @@ mod tests { assert_matches!(validate_chain_result, Ok(())); // Scan the cache - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Data-only chain should be valid validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None).unwrap(); @@ -340,6 +341,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(7).unwrap(), + 1, ); insert_into_cache(&db_cache, &cb2); @@ -347,7 +349,7 @@ mod tests { validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None).unwrap(); // Scan the cache again - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Data-only chain should be valid validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None).unwrap(); @@ -373,6 +375,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(5).unwrap(), + 0, ); let (cb2, _) = fake_compact_block( sapling_activation_height() + 1, @@ -380,12 +383,13 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(7).unwrap(), + 1, ); insert_into_cache(&db_cache, &cb); insert_into_cache(&db_cache, &cb2); // Scan the cache - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Data-only chain should be valid validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None).unwrap(); @@ -397,6 +401,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(8).unwrap(), + 2, ); let (cb4, _) = fake_compact_block( sapling_activation_height() + 3, @@ -404,6 +409,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(3).unwrap(), + 3, ); insert_into_cache(&db_cache, &cb3); insert_into_cache(&db_cache, &cb4); @@ -434,6 +440,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(5).unwrap(), + 0, ); let (cb2, _) = fake_compact_block( sapling_activation_height() + 1, @@ -441,12 +448,13 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(7).unwrap(), + 1, ); insert_into_cache(&db_cache, &cb); insert_into_cache(&db_cache, &cb2); // Scan the cache - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Data-only chain should be valid validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None).unwrap(); @@ -458,6 +466,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(8).unwrap(), + 2, ); let (cb4, _) = fake_compact_block( sapling_activation_height() + 3, @@ -465,6 +474,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(3).unwrap(), + 3, ); insert_into_cache(&db_cache, &cb3); insert_into_cache(&db_cache, &cb4); @@ -503,6 +513,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 0, ); let (cb2, _) = fake_compact_block( @@ -511,12 +522,13 @@ mod tests { &dfvk, AddressType::DefaultExternal, value2, + 1, ); insert_into_cache(&db_cache, &cb); insert_into_cache(&db_cache, &cb2); // Scan the cache - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Account balance should reflect both received notes assert_eq!( @@ -551,7 +563,7 @@ mod tests { ); // Scan the cache again - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Account balance should again reflect both received notes assert_eq!( @@ -581,9 +593,10 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 0, ); insert_into_cache(&db_cache, &cb1); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); assert_eq!( get_balance(&db_data.conn, AccountId::from(0)).unwrap(), value @@ -596,6 +609,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 1, ); let (cb3, _) = fake_compact_block( sapling_activation_height() + 2, @@ -603,9 +617,10 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 2, ); insert_into_cache(&db_cache, &cb3); - match scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None) { + match scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None) { Err(Error::Chain(e)) => { assert_matches!( e.cause(), @@ -618,7 +633,7 @@ mod tests { // If we add a block of height SAPLING_ACTIVATION_HEIGHT + 1, we can now scan both insert_into_cache(&db_cache, &cb2); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); assert_eq!( get_balance(&db_data.conn, AccountId::from(0)).unwrap(), Amount::from_u64(150_000).unwrap() @@ -652,11 +667,12 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 0, ); insert_into_cache(&db_cache, &cb); // Scan the cache - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Account balance should reflect the received note assert_eq!( @@ -672,11 +688,12 @@ mod tests { &dfvk, AddressType::DefaultExternal, value2, + 1, ); insert_into_cache(&db_cache, &cb2); // Scan the cache again - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Account balance should reflect both received notes assert_eq!( @@ -712,11 +729,12 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 0, ); insert_into_cache(&db_cache, &cb); // Scan the cache - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Account balance should reflect the received note assert_eq!( @@ -737,11 +755,12 @@ mod tests { &dfvk, to2, value2, + 1, ), ); // Scan the cache again - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Account balance should equal the change assert_eq!( diff --git a/zcash_client_sqlite/src/error.rs b/zcash_client_sqlite/src/error.rs index cfd326b9c..3f1832411 100644 --- a/zcash_client_sqlite/src/error.rs +++ b/zcash_client_sqlite/src/error.rs @@ -3,6 +3,7 @@ use std::error; use std::fmt; +use shardtree::ShardTreeError; use zcash_client_backend::encoding::{Bech32DecodeError, TransparentCodecError}; use zcash_primitives::{consensus::BlockHeight, zip32::AccountId}; @@ -74,6 +75,9 @@ pub enum SqliteClientError { /// belonging to the wallet #[cfg(feature = "transparent-inputs")] AddressNotRecognized(TransparentAddress), + + /// An error occurred in inserting data into one of the wallet's note commitment trees. + CommitmentTree(ShardTreeError), } impl error::Error for SqliteClientError { @@ -114,6 +118,7 @@ impl fmt::Display for SqliteClientError { SqliteClientError::AccountIdOutOfRange => write!(f, "Wallet account identifiers must be less than 0x7FFFFFFF."), #[cfg(feature = "transparent-inputs")] SqliteClientError::AddressNotRecognized(_) => write!(f, "The address associated with a received txo is not identifiable as belonging to the wallet."), + SqliteClientError::CommitmentTree(err) => write!(f, "An error occurred accessing or updating note commitment tree data: {}.", err), } } } @@ -160,3 +165,9 @@ impl From for SqliteClientError { SqliteClientError::InvalidMemo(e) } } + +impl From> for SqliteClientError { + fn from(e: ShardTreeError) -> Self { + SqliteClientError::CommitmentTree(e) + } +} diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 1835b27a9..e17cbab07 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -34,8 +34,10 @@ use rusqlite::{self, Connection}; use secrecy::{ExposeSecret, SecretVec}; -use std::{borrow::Borrow, collections::HashMap, convert::AsRef, fmt, path::Path}; +use std::{borrow::Borrow, collections::HashMap, convert::AsRef, fmt, ops::Range, path::Path}; +use incrementalmerkletree::Position; +use shardtree::{ShardTree, ShardTreeError}; use zcash_primitives::{ block::BlockHash, consensus::{self, BlockHeight}, @@ -52,8 +54,10 @@ use zcash_primitives::{ use zcash_client_backend::{ address::{AddressMetadata, UnifiedAddress}, data_api::{ - self, chain::BlockSource, DecryptedTransaction, NullifierQuery, PoolType, PrunedBlock, - Recipient, SentTransaction, WalletRead, WalletWrite, + self, + chain::{BlockSource, CommitmentTreeMeta}, + DecryptedTransaction, NullifierQuery, PoolType, PrunedBlock, Recipient, SentTransaction, + WalletCommitmentTrees, WalletRead, WalletWrite, SAPLING_SHARD_HEIGHT, }, keys::{UnifiedFullViewingKey, UnifiedSpendingKey}, proto::compact_formats::CompactBlock, @@ -61,7 +65,9 @@ use zcash_client_backend::{ DecryptedOutput, TransferType, }; -use crate::error::SqliteClientError; +use crate::{ + error::SqliteClientError, wallet::sapling::commitment_tree::WalletDbSaplingShardStore, +}; #[cfg(feature = "unstable")] use { @@ -125,15 +131,15 @@ impl WalletDb { }) } - pub fn transactionally(&mut self, f: F) -> Result + pub fn transactionally>(&mut self, f: F) -> Result where - F: FnOnce(&WalletDb, P>) -> Result, + F: FnOnce(&mut WalletDb, P>) -> Result, { - let wdb = WalletDb { + let mut wdb = WalletDb { conn: SqlTransaction(self.conn.transaction()?), params: self.params.clone(), }; - let result = f(&wdb)?; + let result = f(&mut wdb)?; wdb.conn.0.commit()?; Ok(result) } @@ -148,6 +154,20 @@ impl, P: consensus::Parameters> WalletRead for W wallet::block_height_extrema(self.conn.borrow()).map_err(SqliteClientError::from) } + fn fully_scanned_height( + &self, + ) -> Result, Self::Error> { + wallet::fully_scanned_height(self.conn.borrow()) + } + + fn suggest_scan_ranges( + &self, + _batch_size: usize, + _limit: usize, + ) -> Result>, Self::Error> { + todo!() + } + fn get_min_unspent_height(&self) -> Result, Self::Error> { wallet::get_min_unspent_height(self.conn.borrow()).map_err(SqliteClientError::from) } @@ -210,24 +230,9 @@ impl, P: consensus::Parameters> WalletRead for W } } - fn get_commitment_tree( - &self, - block_height: BlockHeight, - ) -> Result, Self::Error> { - wallet::sapling::get_sapling_commitment_tree(self.conn.borrow(), block_height) - } - - #[allow(clippy::type_complexity)] - fn get_witnesses( - &self, - block_height: BlockHeight, - ) -> Result, Self::Error> { - wallet::sapling::get_sapling_witnesses(self.conn.borrow(), block_height) - } - fn get_sapling_nullifiers( &self, - query: data_api::NullifierQuery, + query: NullifierQuery, ) -> Result, Self::Error> { match query { NullifierQuery::Unspent => wallet::sapling::get_sapling_nullifiers(self.conn.borrow()), @@ -386,21 +391,21 @@ impl WalletWrite for WalletDb #[allow(clippy::type_complexity)] fn advance_by_block( &mut self, - block: &PrunedBlock, - updated_witnesses: &[(Self::NoteRef, sapling::IncrementalWitness)], - ) -> Result, Self::Error> { + block: PrunedBlock, + ) -> Result, Self::Error> { self.transactionally(|wdb| { // Insert the block into the database. + let block_height = block.block_height; wallet::insert_block( &wdb.conn.0, - block.block_height, + block_height, block.block_hash, block.block_time, - block.commitment_tree, + block.sapling_commitment_tree_size.map(|s| s.into()), )?; - let mut new_witnesses = vec![]; - for tx in block.transactions { + let mut wallet_note_ids = vec![]; + for tx in &block.transactions { let tx_row = wallet::put_tx_meta(&wdb.conn.0, tx, block.block_height)?; // Mark notes as spent and remove them from the scanning cache @@ -413,32 +418,24 @@ impl WalletWrite for WalletDb wallet::sapling::put_received_note(&wdb.conn.0, output, tx_row)?; // Save witness for note. - new_witnesses.push((received_note_id, output.witness().clone())); + wallet_note_ids.push(received_note_id); } } - // Insert current new_witnesses into the database. - for (received_note_id, witness) in updated_witnesses.iter().chain(new_witnesses.iter()) - { - if let NoteId::ReceivedNoteId(rnid) = *received_note_id { - wallet::sapling::insert_witness( - &wdb.conn.0, - rnid, - witness, - block.block_height, - )?; - } else { - return Err(SqliteClientError::InvalidNoteId); + let mut sapling_commitments = block.sapling_commitments.into_iter(); + wdb.with_sapling_tree_mut::<_, _, SqliteClientError>(move |sapling_tree| { + if let Some(sapling_tree_size) = block.sapling_commitment_tree_size { + let start_position = Position::from(u64::from(sapling_tree_size)) + - u64::try_from(sapling_commitments.len()).unwrap(); + sapling_tree.batch_insert(start_position, &mut sapling_commitments)?; } - } - - // Prune the stored witnesses (we only expect rollbacks of at most PRUNING_HEIGHT blocks). - wallet::prune_witnesses(&wdb.conn.0, block.block_height - PRUNING_HEIGHT)?; + Ok(()) + })?; // Update now-expired transactions that didn't get mined. - wallet::update_expired_notes(&wdb.conn.0, block.block_height)?; + wallet::update_expired_notes(&wdb.conn.0, block_height)?; - Ok(new_witnesses) + Ok(wallet_note_ids) }) } @@ -493,55 +490,37 @@ impl WalletWrite for WalletDb wallet::sapling::put_received_note(&wdb.conn.0, output, tx_ref)?; } } - } - // If any of the utxos spent in the transaction are ours, mark them as spent. - #[cfg(feature = "transparent-inputs")] - for txin in d_tx - .tx - .transparent_bundle() - .iter() - .flat_map(|b| b.vin.iter()) - { - wallet::mark_transparent_utxo_spent(&wdb.conn.0, tx_ref, &txin.prevout)?; - } + // If any of the utxos spent in the transaction are ours, mark them as spent. + #[cfg(feature = "transparent-inputs")] + for txin in d_tx.tx.transparent_bundle().iter().flat_map(|b| b.vin.iter()) { + wallet::mark_transparent_utxo_spent(&wdb.conn.0, tx_ref, &txin.prevout)?; + } - // If we have some transparent outputs: - if !d_tx - .tx - .transparent_bundle() - .iter() - .any(|b| b.vout.is_empty()) - { - let nullifiers = wdb.get_sapling_nullifiers(data_api::NullifierQuery::All)?; - // If the transaction contains shielded spends from our wallet, we will store z->t - // transactions we observe in the same way they would be stored by - // create_spend_to_address. - if let Some((account_id, _)) = nullifiers.iter().find(|(_, nf)| { - d_tx.tx - .sapling_bundle() - .iter() - .flat_map(|b| b.shielded_spends().iter()) - .any(|input| nf == input.nullifier()) - }) { - for (output_index, txout) in d_tx - .tx - .transparent_bundle() - .iter() - .flat_map(|b| b.vout.iter()) - .enumerate() - { - if let Some(address) = txout.recipient_address() { - wallet::put_sent_output( - &wdb.conn.0, - &wdb.params, - *account_id, - tx_ref, - output_index, - &Recipient::Transparent(address), - txout.value, - None, - )?; + // If we have some transparent outputs: + if d_tx.tx.transparent_bundle().iter().any(|b| !b.vout.is_empty()) { + let nullifiers = wdb.get_sapling_nullifiers(NullifierQuery::All)?; + // If the transaction contains shielded spends from our wallet, we will store z->t + // transactions we observe in the same way they would be stored by + // create_spend_to_address. + if let Some((account_id, _)) = nullifiers.iter().find( + |(_, nf)| + d_tx.tx.sapling_bundle().iter().flat_map(|b| b.shielded_spends().iter()) + .any(|input| nf == input.nullifier()) + ) { + for (output_index, txout) in d_tx.tx.transparent_bundle().iter().flat_map(|b| b.vout.iter()).enumerate() { + if let Some(address) = txout.recipient_address() { + wallet::put_sent_output( + &wdb.conn.0, + &wdb.params, + *account_id, + tx_ref, + output_index, + &Recipient::Transparent(address), + txout.value, + None + )?; + } } } } @@ -633,6 +612,59 @@ impl WalletWrite for WalletDb } } +impl WalletCommitmentTrees for WalletDb { + type Error = rusqlite::Error; + type SaplingShardStore<'a> = WalletDbSaplingShardStore<'a, 'a>; + + fn with_sapling_tree_mut(&mut self, mut callback: F) -> Result + where + for<'a> F: FnMut( + &'a mut ShardTree< + Self::SaplingShardStore<'a>, + { sapling::NOTE_COMMITMENT_TREE_DEPTH }, + SAPLING_SHARD_HEIGHT, + >, + ) -> Result, + E: From>, + { + let tx = self.conn.transaction().map_err(ShardTreeError::Storage)?; + let shard_store = + WalletDbSaplingShardStore::from_connection(&tx).map_err(ShardTreeError::Storage)?; + let result = { + let mut shardtree = ShardTree::new(shard_store, 100); + callback(&mut shardtree)? + }; + tx.commit().map_err(ShardTreeError::Storage)?; + Ok(result) + } +} + +impl<'conn, P: consensus::Parameters> WalletCommitmentTrees for WalletDb, P> { + type Error = rusqlite::Error; + type SaplingShardStore<'a> = WalletDbSaplingShardStore<'a, 'a>; + + fn with_sapling_tree_mut(&mut self, mut callback: F) -> Result + where + for<'a> F: FnMut( + &'a mut ShardTree< + Self::SaplingShardStore<'a>, + { sapling::NOTE_COMMITMENT_TREE_DEPTH }, + SAPLING_SHARD_HEIGHT, + >, + ) -> Result, + E: From>, + { + let mut shardtree = ShardTree::new( + WalletDbSaplingShardStore::from_connection(&self.conn.0) + .map_err(ShardTreeError::Storage)?, + 100, + ); + let result = callback(&mut shardtree)?; + + Ok(result) + } +} + /// A handle for the SQLite block source. pub struct BlockDb(Connection); @@ -1024,6 +1056,7 @@ mod tests { dfvk: &DiversifiableFullViewingKey, req: AddressType, value: Amount, + initial_sapling_tree_size: u32, ) -> (CompactBlock, Nullifier) { let to = match req { AddressType::DefaultExternal => dfvk.default_address().1, @@ -1069,6 +1102,8 @@ mod tests { }; cb.prev_hash.extend_from_slice(&prev_hash.0); cb.vtx.push(ctx); + cb.sapling_commitment_tree_size = initial_sapling_tree_size + + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(); (cb, note.nf(&dfvk.fvk().vk.nk, 0)) } @@ -1081,6 +1116,7 @@ mod tests { dfvk: &DiversifiableFullViewingKey, to: PaymentAddress, value: Amount, + initial_sapling_tree_size: u32, ) -> CompactBlock { let mut rng = OsRng; let rseed = generate_random_rseed(&network(), height, &mut rng); @@ -1154,6 +1190,8 @@ mod tests { }; cb.prev_hash.extend_from_slice(&prev_hash.0); cb.vtx.push(ctx); + cb.sapling_commitment_tree_size = initial_sapling_tree_size + + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(); cb } @@ -1267,6 +1305,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(5).unwrap(), + 0, ); let (cb2, _) = fake_compact_block( BlockHeight::from_u32(2), @@ -1274,6 +1313,7 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(10).unwrap(), + 1, ); // Write the CompactBlocks to the BlockMeta DB's corresponding disk storage. diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index a05597134..406db08c7 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -67,13 +67,13 @@ use rusqlite::{self, named_params, params, OptionalExtension, ToSql}; use std::collections::HashMap; use std::convert::TryFrom; +use std::io::Cursor; use zcash_primitives::{ block::BlockHash, consensus::{self, BlockHeight, BranchId, NetworkUpgrade, Parameters}, memo::{Memo, MemoBytes}, - merkle_tree::write_commitment_tree, - sapling::CommitmentTree, + merkle_tree::read_commitment_tree, transaction::{components::Amount, Transaction, TxId}, zip32::{ sapling::{DiversifiableFullViewingKey, ExtendedFullViewingKey}, @@ -83,7 +83,7 @@ use zcash_primitives::{ use zcash_client_backend::{ address::{RecipientAddress, UnifiedAddress}, - data_api::{PoolType, Recipient, SentTransactionOutput}, + data_api::{chain::CommitmentTreeMeta, PoolType, Recipient, SentTransactionOutput}, encoding::AddressCodec, keys::UnifiedFullViewingKey, wallet::WalletTx, @@ -536,6 +536,51 @@ pub(crate) fn block_height_extrema( }) } +pub(crate) fn fully_scanned_height( + conn: &rusqlite::Connection, +) -> Result, SqliteClientError> { + let res_opt = conn + .query_row( + "SELECT height, sapling_commitment_tree_size, sapling_tree + FROM blocks + ORDER BY height DESC + LIMIT 1", + [], + |row| { + let max_height: u32 = row.get(0)?; + let sapling_tree_size: Option = row.get(1)?; + let sapling_tree: Vec = row.get(0)?; + Ok(( + BlockHeight::from(max_height), + sapling_tree_size, + sapling_tree, + )) + }, + ) + .optional()?; + + res_opt + .map(|(max_height, sapling_tree_size, sapling_tree)| { + let commitment_tree_meta = + CommitmentTreeMeta::from_parts(if let Some(known_size) = sapling_tree_size { + known_size + } else { + // parse the legacy commitment tree data + read_commitment_tree::< + zcash_primitives::sapling::Node, + _, + { zcash_primitives::sapling::NOTE_COMMITMENT_TREE_DEPTH }, + >(Cursor::new(sapling_tree))? + .size() + .try_into() + .expect("usize values are convertible to u64 on all supported platforms.") + }); + + Ok((max_height, commitment_tree_meta)) + }) + .transpose() +} + /// Returns the block height at which the specified transaction was mined, /// if any. pub(crate) fn get_tx_height( @@ -765,21 +810,24 @@ pub(crate) fn insert_block( block_height: BlockHeight, block_hash: BlockHash, block_time: u32, - commitment_tree: &CommitmentTree, + sapling_commitment_tree_size: Option, ) -> Result<(), SqliteClientError> { - let mut encoded_tree = Vec::new(); - write_commitment_tree(commitment_tree, &mut encoded_tree).unwrap(); - let mut stmt_insert_block = conn.prepare_cached( - "INSERT INTO blocks (height, hash, time, sapling_tree) - VALUES (?, ?, ?, ?)", + "INSERT INTO blocks ( + height, + hash, + time, + sapling_commitment_tree_size, + sapling_tree + ) + VALUES (?, ?, ?, ?, x'00')", )?; stmt_insert_block.execute(params![ u32::from(block_height), &block_hash.0[..], block_time, - encoded_tree + sapling_commitment_tree_size ])?; Ok(()) @@ -951,17 +999,6 @@ pub(crate) fn put_legacy_transparent_utxo( stmt_upsert_legacy_transparent_utxo.query_row(sql_args, |row| row.get::<_, i64>(0).map(UtxoId)) } -/// Removes old incremental witnesses up to the given block height. -pub(crate) fn prune_witnesses( - conn: &rusqlite::Connection, - below_height: BlockHeight, -) -> Result<(), SqliteClientError> { - let mut stmt_prune_witnesses = - conn.prepare_cached("DELETE FROM sapling_witnesses WHERE block < ?")?; - stmt_prune_witnesses.execute([u32::from(below_height)])?; - Ok(()) -} - /// Marks notes that have not been mined in transactions /// as expired, up to the given block height. pub(crate) fn update_expired_notes( diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index 7f5a60ccc..bb8834a6a 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -6,6 +6,7 @@ use rusqlite::{self, types::ToSql}; use schemer::{Migrator, MigratorError}; use schemer_rusqlite::RusqliteAdapter; use secrecy::SecretVec; +use shardtree::ShardTreeError; use uuid::Uuid; use zcash_primitives::{ @@ -34,6 +35,9 @@ pub enum WalletMigrationError { /// Wrapper for amount balance violations BalanceError(BalanceError), + + /// Wrapper for commitment tree invariant violations + CommitmentTree(ShardTreeError), } impl From for WalletMigrationError { @@ -48,6 +52,12 @@ impl From for WalletMigrationError { } } +impl From> for WalletMigrationError { + fn from(e: ShardTreeError) -> Self { + WalletMigrationError::CommitmentTree(e) + } +} + impl fmt::Display for WalletMigrationError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match &self { @@ -62,6 +72,7 @@ impl fmt::Display for WalletMigrationError { } WalletMigrationError::DbError(e) => write!(f, "{}", e), WalletMigrationError::BalanceError(e) => write!(f, "Balance error: {:?}", e), + WalletMigrationError::CommitmentTree(e) => write!(f, "Commitment tree error: {:?}", e), } } } @@ -361,8 +372,9 @@ mod tests { height INTEGER PRIMARY KEY, hash BLOB NOT NULL, time INTEGER NOT NULL, - sapling_tree BLOB NOT NULL - )", + sapling_tree BLOB NOT NULL , + sapling_commitment_tree_size INTEGER, + orchard_commitment_tree_size INTEGER)", "CREATE TABLE sapling_received_notes ( id_note INTEGER PRIMARY KEY, tx INTEGER NOT NULL, @@ -375,6 +387,7 @@ mod tests { is_change INTEGER NOT NULL, memo BLOB, spent INTEGER, + commitment_tree_position INTEGER, FOREIGN KEY (tx) REFERENCES transactions(id_tx), FOREIGN KEY (account) REFERENCES accounts(account), FOREIGN KEY (spent) REFERENCES transactions(id_tx), diff --git a/zcash_client_sqlite/src/wallet/init/migrations.rs b/zcash_client_sqlite/src/wallet/init/migrations.rs index 1cc9bcfc5..e51605ccf 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations.rs @@ -4,6 +4,7 @@ mod addresses_table; mod initial_setup; mod received_notes_nullable_nf; mod sent_notes_to_internal; +mod shardtree_support; mod ufvk_support; mod utxos_table; mod v_transactions_net; @@ -46,5 +47,6 @@ pub(super) fn all_migrations( Box::new(add_transaction_views::Migration), Box::new(v_transactions_net::Migration), Box::new(received_notes_nullable_nf::Migration), + Box::new(shardtree_support::Migration), ] } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs new file mode 100644 index 000000000..b9e4a6bf0 --- /dev/null +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -0,0 +1,56 @@ +//! This migration adds tables to the wallet database that are needed to persist note commitment +//! tree data using the `shardtree` crate, and migrates existing witness data into these data +//! structures. + +use std::collections::HashSet; + +use rusqlite; +use schemer; +use schemer_rusqlite::RusqliteMigration; +use uuid::Uuid; + +use crate::wallet::init::{migrations::received_notes_nullable_nf, WalletMigrationError}; + +pub(super) const MIGRATION_ID: Uuid = Uuid::from_fields( + 0x7da6489d, + 0xe835, + 0x4657, + b"\x8b\xe5\xf5\x12\xbc\xce\x6c\xbf", +); + +pub(super) struct Migration; + +impl schemer::Migration for Migration { + fn id(&self) -> Uuid { + MIGRATION_ID + } + + fn dependencies(&self) -> HashSet { + [received_notes_nullable_nf::MIGRATION_ID] + .into_iter() + .collect() + } + + fn description(&self) -> &'static str { + "Add support for receiving storage of note commitment tree data using the `shardtree` crate." + } +} + +impl RusqliteMigration for Migration { + type Error = WalletMigrationError; + + fn up(&self, transaction: &rusqlite::Transaction) -> Result<(), WalletMigrationError> { + // Add commitment tree sizes to block metadata. + transaction.execute_batch( + "ALTER TABLE blocks ADD COLUMN sapling_commitment_tree_size INTEGER; + ALTER TABLE blocks ADD COLUMN orchard_commitment_tree_size INTEGER; + ALTER TABLE sapling_received_notes ADD COLUMN commitment_tree_position INTEGER;", + )?; + + Ok(()) + } + + fn down(&self, _transaction: &rusqlite::Transaction) -> Result<(), WalletMigrationError> { + Ok(()) + } +} diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 66734cfdc..511333ff7 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -1,12 +1,13 @@ //! Functions for Sapling support in the wallet. + use group::ff::PrimeField; -use rusqlite::{named_params, params, types::Value, Connection, OptionalExtension, Row}; +use incrementalmerkletree::Position; +use rusqlite::{named_params, params, types::Value, Connection, Row}; use std::rc::Rc; use zcash_primitives::{ consensus::BlockHeight, memo::MemoBytes, - merkle_tree::{read_commitment_tree, read_incremental_witness, write_incremental_witness}, sapling::{self, Diversifier, Note, Nullifier, Rseed}, transaction::components::Amount, zip32::AccountId, @@ -21,6 +22,8 @@ use crate::{error::SqliteClientError, NoteId}; use super::memo_repr; +pub(crate) mod commitment_tree; + /// This trait provides a generalization over shielded output representations. pub(crate) trait ReceivedSaplingOutput { fn index(&self) -> usize; @@ -28,10 +31,11 @@ pub(crate) trait ReceivedSaplingOutput { fn note(&self) -> &Note; fn memo(&self) -> Option<&MemoBytes>; fn is_change(&self) -> bool; - fn nullifier(&self) -> Option<&Nullifier>; + fn nullifier(&self) -> Option<&sapling::Nullifier>; + fn note_commitment_tree_position(&self) -> Option; } -impl ReceivedSaplingOutput for WalletSaplingOutput { +impl ReceivedSaplingOutput for WalletSaplingOutput { fn index(&self) -> usize { self.index() } @@ -47,10 +51,12 @@ impl ReceivedSaplingOutput for WalletSaplingOutput { fn is_change(&self) -> bool { WalletSaplingOutput::is_change(self) } - - fn nullifier(&self) -> Option<&Nullifier> { + fn nullifier(&self) -> Option<&sapling::Nullifier> { Some(self.nf()) } + fn note_commitment_tree_position(&self) -> Option { + Some(WalletSaplingOutput::note_commitment_tree_position(self)) + } } impl ReceivedSaplingOutput for DecryptedOutput { @@ -69,7 +75,10 @@ impl ReceivedSaplingOutput for DecryptedOutput { fn is_change(&self) -> bool { self.transfer_type == TransferType::WalletInternal } - fn nullifier(&self) -> Option<&Nullifier> { + fn nullifier(&self) -> Option<&sapling::Nullifier> { + None + } + fn note_commitment_tree_position(&self) -> Option { None } } @@ -105,17 +114,17 @@ fn to_spendable_note(row: &Row) -> Result, SqliteCli Rseed::BeforeZip212(rcm) }; - let witness = { - let d: Vec<_> = row.get(4)?; - read_incremental_witness(&d[..])? - }; + let note_commitment_tree_position = + Position::from(u64::try_from(row.get::<_, i64>(4)?).map_err(|_| { + SqliteClientError::CorruptedData("Note commitment tree position invalid.".to_string()) + })?); Ok(ReceivedSaplingNote { note_id, diversifier, note_value, rseed, - witness, + note_commitment_tree_position, }) } @@ -126,15 +135,13 @@ pub(crate) fn get_spendable_sapling_notes( exclude: &[NoteId], ) -> Result>, SqliteClientError> { let mut stmt_select_notes = conn.prepare_cached( - "SELECT id_note, diversifier, value, rcm, witness - FROM sapling_received_notes - INNER JOIN transactions ON transactions.id_tx = sapling_received_notes.tx - INNER JOIN sapling_witnesses ON sapling_witnesses.note = sapling_received_notes.id_note - WHERE account = :account - AND spent IS NULL - AND transactions.block <= :anchor_height - AND sapling_witnesses.block = :anchor_height - AND id_note NOT IN rarray(:exclude)", + "SELECT id_note, diversifier, value, rcm, commitment_tree_position + FROM sapling_received_notes + INNER JOIN transactions ON transactions.id_tx = sapling_received_notes.tx + WHERE account = :account + AND spent IS NULL + AND transactions.block <= :anchor_height + AND id_note NOT IN rarray(:exclude)", )?; let excluded: Vec = exclude @@ -184,28 +191,22 @@ pub(crate) fn select_spendable_sapling_notes( // // 4) Match the selected notes against the witnesses at the desired height. let mut stmt_select_notes = conn.prepare_cached( - "WITH selected AS ( - WITH eligible AS ( - SELECT id_note, diversifier, value, rcm, - SUM(value) OVER - (PARTITION BY account, spent ORDER BY id_note) AS so_far - FROM sapling_received_notes - INNER JOIN transactions ON transactions.id_tx = sapling_received_notes.tx - WHERE account = :account - AND spent IS NULL - AND transactions.block <= :anchor_height - AND id_note NOT IN rarray(:exclude) - ) - SELECT * FROM eligible WHERE so_far < :target_value - UNION - SELECT * FROM (SELECT * FROM eligible WHERE so_far >= :target_value LIMIT 1) - ), witnesses AS ( - SELECT note, witness FROM sapling_witnesses - WHERE block = :anchor_height - ) - SELECT selected.id_note, selected.diversifier, selected.value, selected.rcm, witnesses.witness - FROM selected - INNER JOIN witnesses ON selected.id_note = witnesses.note", + "WITH eligible AS ( + SELECT id_note, diversifier, value, rcm, commitment_tree_position, + SUM(value) + OVER (PARTITION BY account, spent ORDER BY id_note) AS so_far + FROM sapling_received_notes + INNER JOIN transactions ON transactions.id_tx = sapling_received_notes.tx + WHERE account = :account + AND spent IS NULL + AND transactions.block <= :anchor_height + AND id_note NOT IN rarray(:exclude) + ) + SELECT id_note, diversifier, value, rcm, commitment_tree_position + FROM eligible WHERE so_far < :target_value + UNION + SELECT id_note, diversifier, value, rcm, commitment_tree_position + FROM (SELECT * from eligible WHERE so_far >= :target_value LIMIT 1)", )?; let excluded: Vec = exclude @@ -230,73 +231,6 @@ pub(crate) fn select_spendable_sapling_notes( notes.collect::>() } -/// Returns the commitment tree for the block at the specified height, -/// if any. -pub(crate) fn get_sapling_commitment_tree( - conn: &Connection, - block_height: BlockHeight, -) -> Result, SqliteClientError> { - conn.query_row_and_then( - "SELECT sapling_tree FROM blocks WHERE height = ?", - [u32::from(block_height)], - |row| { - let row_data: Vec = row.get(0)?; - read_commitment_tree(&row_data[..]).map_err(|e| { - rusqlite::Error::FromSqlConversionFailure( - row_data.len(), - rusqlite::types::Type::Blob, - Box::new(e), - ) - }) - }, - ) - .optional() - .map_err(SqliteClientError::from) -} - -/// Returns the incremental witnesses for the block at the specified height, -/// if any. -pub(crate) fn get_sapling_witnesses( - conn: &Connection, - block_height: BlockHeight, -) -> Result, SqliteClientError> { - let mut stmt_fetch_witnesses = - conn.prepare_cached("SELECT note, witness FROM sapling_witnesses WHERE block = ?")?; - - let witnesses = stmt_fetch_witnesses - .query_map([u32::from(block_height)], |row| { - let id_note = NoteId::ReceivedNoteId(row.get(0)?); - let witness_data: Vec = row.get(1)?; - Ok(read_incremental_witness(&witness_data[..]).map(|witness| (id_note, witness))) - }) - .map_err(SqliteClientError::from)?; - - // unwrap database error & IO error from IncrementalWitness::read - let res: Vec<_> = witnesses.collect::, _>>()??; - Ok(res) -} - -/// Records the incremental witness for the specified note, -/// as of the given block height. -pub(crate) fn insert_witness( - conn: &Connection, - note_id: i64, - witness: &sapling::IncrementalWitness, - height: BlockHeight, -) -> Result<(), SqliteClientError> { - let mut stmt_insert_witness = conn.prepare_cached( - "INSERT INTO sapling_witnesses (note, block, witness) - VALUES (?, ?, ?)", - )?; - - let mut encoded = Vec::new(); - write_incremental_witness(witness, &mut encoded).unwrap(); - - stmt_insert_witness.execute(params![note_id, u32::from(height), encoded])?; - - Ok(()) -} - /// Retrieves the set of nullifiers for "potentially spendable" Sapling notes that the /// wallet is tracking. /// @@ -320,7 +254,7 @@ pub(crate) fn get_sapling_nullifiers( let nf_bytes: Vec = row.get(2)?; Ok(( AccountId::from(account), - Nullifier::from_slice(&nf_bytes).unwrap(), + sapling::Nullifier::from_slice(&nf_bytes).unwrap(), )) })?; @@ -343,7 +277,7 @@ pub(crate) fn get_all_sapling_nullifiers( let nf_bytes: Vec = row.get(2)?; Ok(( AccountId::from(account), - Nullifier::from_slice(&nf_bytes).unwrap(), + sapling::Nullifier::from_slice(&nf_bytes).unwrap(), )) })?; @@ -359,7 +293,7 @@ pub(crate) fn get_all_sapling_nullifiers( pub(crate) fn mark_sapling_note_spent( conn: &Connection, tx_ref: i64, - nf: &Nullifier, + nf: &sapling::Nullifier, ) -> Result { let mut stmt_mark_sapling_note_spent = conn.prepare_cached("UPDATE sapling_received_notes SET spent = ? WHERE nf = ?")?; @@ -383,9 +317,19 @@ pub(crate) fn put_received_note( ) -> Result { let mut stmt_upsert_received_note = conn.prepare_cached( "INSERT INTO sapling_received_notes - (tx, output_index, account, diversifier, value, rcm, memo, nf, is_change) - VALUES - (:tx, :output_index, :account, :diversifier, :value, :rcm, :memo, :nf, :is_change) + (tx, output_index, account, diversifier, value, rcm, memo, nf, is_change, commitment_tree_position) + VALUES ( + :tx, + :output_index, + :account, + :diversifier, + :value, + :rcm, + :memo, + :nf, + :is_change, + :commitment_tree_position + ) ON CONFLICT (tx, output_index) DO UPDATE SET account = :account, diversifier = :diversifier, @@ -393,7 +337,8 @@ pub(crate) fn put_received_note( rcm = :rcm, nf = IFNULL(:nf, nf), memo = IFNULL(:memo, memo), - is_change = IFNULL(:is_change, is_change) + is_change = IFNULL(:is_change, is_change), + commitment_tree_position = IFNULL(:commitment_tree_position, commitment_tree_position) RETURNING id_note", )?; @@ -410,7 +355,8 @@ pub(crate) fn put_received_note( ":rcm": &rcm.as_ref(), ":nf": output.nullifier().map(|nf| nf.0.as_ref()), ":memo": memo_repr(output.memo()), - ":is_change": output.is_change() + ":is_change": output.is_change(), + ":commitment_tree_position": output.note_commitment_tree_position().map(u64::from), ]; stmt_upsert_received_note @@ -622,9 +568,10 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 0, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Verified balance matches total balance let (_, anchor_height) = db_data.get_target_and_anchor_heights(10).unwrap().unwrap(); @@ -644,9 +591,10 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 1, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Verified balance does not include the second note let (_, anchor_height2) = db_data.get_target_and_anchor_heights(10).unwrap().unwrap(); @@ -691,10 +639,11 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + i, ); insert_into_cache(&db_cache, &cb); } - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Second spend still fails assert_matches!( @@ -724,9 +673,10 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 11, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Second spend should now succeed assert_matches!( @@ -768,9 +718,10 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 0, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); assert_eq!( get_balance(&db_data.conn, AccountId::from(0)).unwrap(), value @@ -823,10 +774,11 @@ mod tests { &ExtendedSpendingKey::master(&[i as u8]).to_diversifiable_full_viewing_key(), AddressType::DefaultExternal, value, + i, ); insert_into_cache(&db_cache, &cb); } - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Second spend still fails assert_matches!( @@ -855,9 +807,10 @@ mod tests { &ExtendedSpendingKey::master(&[42]).to_diversifiable_full_viewing_key(), AddressType::DefaultExternal, value, + 42, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Second spend should now succeed create_spend_to_address( @@ -898,9 +851,10 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 0, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); assert_eq!( get_balance(&db_data.conn, AccountId::from(0)).unwrap(), value @@ -968,10 +922,11 @@ mod tests { &ExtendedSpendingKey::master(&[i as u8]).to_diversifiable_full_viewing_key(), AddressType::DefaultExternal, value, + i, ); insert_into_cache(&db_cache, &cb); } - scan_cached_blocks(&network, &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&network, &db_cache, &mut db_data, None, None).unwrap(); // Send the funds again, discarding history. // Neither transaction output is decryptable by the sender. @@ -1001,9 +956,10 @@ mod tests { &dfvk, AddressType::DefaultExternal, value, + 0, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Verified balance matches total balance let (_, anchor_height) = db_data.get_target_and_anchor_heights(10).unwrap().unwrap(); @@ -1056,9 +1012,10 @@ mod tests { &dfvk, AddressType::Internal, value, + 0, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Verified balance matches total balance let (_, anchor_height) = db_data.get_target_and_anchor_heights(10).unwrap().unwrap(); @@ -1110,6 +1067,7 @@ mod tests { &dfvk, AddressType::Internal, Amount::from_u64(50000).unwrap(), + 0, ); insert_into_cache(&db_cache, &cb); @@ -1121,11 +1079,12 @@ mod tests { &dfvk, AddressType::DefaultExternal, Amount::from_u64(1000).unwrap(), + i, ); insert_into_cache(&db_cache, &cb); } - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Verified balance matches total balance let total = Amount::from_u64(60000).unwrap(); @@ -1241,9 +1200,10 @@ mod tests { &dfvk, AddressType::Internal, Amount::from_u64(50000).unwrap(), + 0, ); insert_into_cache(&db_cache, &cb); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None).unwrap(); + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); assert_matches!( shield_transparent_funds( diff --git a/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs b/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs new file mode 100644 index 000000000..a02ce4cd8 --- /dev/null +++ b/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs @@ -0,0 +1,123 @@ +use incrementalmerkletree::Address; +use rusqlite; +use shardtree::{Checkpoint, LocatedPrunableTree, PrunableTree, ShardStore}; + +use zcash_primitives::{consensus::BlockHeight, sapling}; + +pub struct WalletDbSaplingShardStore<'conn, 'a> { + pub(crate) conn: &'a rusqlite::Transaction<'conn>, +} + +impl<'conn, 'a> WalletDbSaplingShardStore<'conn, 'a> { + pub(crate) fn from_connection( + conn: &'a rusqlite::Transaction<'conn>, + ) -> Result { + Ok(WalletDbSaplingShardStore { conn }) + } +} + +impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { + type H = sapling::Node; + type CheckpointId = BlockHeight; + type Error = rusqlite::Error; + + fn get_shard( + &self, + _shard_root: Address, + ) -> Result>, Self::Error> { + // SELECT shard_data FROM sapling_tree WHERE shard_index = shard_root.index + todo!() + } + + fn last_shard(&self) -> Result>, Self::Error> { + // SELECT shard_data FROM sapling_tree ORDER BY shard_index DESC LIMIT 1 + todo!() + } + + fn put_shard(&mut self, _subtree: LocatedPrunableTree) -> Result<(), Self::Error> { + todo!() + } + + fn get_shard_roots(&self) -> Result, Self::Error> { + // SELECT + todo!() + } + + fn truncate(&mut self, _from: Address) -> Result<(), Self::Error> { + todo!() + } + + fn get_cap(&self) -> Result, Self::Error> { + todo!() + } + + fn put_cap(&mut self, _cap: PrunableTree) -> Result<(), Self::Error> { + todo!() + } + + fn min_checkpoint_id(&self) -> Result, Self::Error> { + todo!() + } + + fn max_checkpoint_id(&self) -> Result, Self::Error> { + todo!() + } + + fn add_checkpoint( + &mut self, + _checkpoint_id: Self::CheckpointId, + _checkpoint: Checkpoint, + ) -> Result<(), Self::Error> { + todo!() + } + + fn checkpoint_count(&self) -> Result { + todo!() + } + + fn get_checkpoint_at_depth( + &self, + _checkpoint_depth: usize, + ) -> Result, Self::Error> { + todo!() + } + + fn get_checkpoint( + &self, + _checkpoint_id: &Self::CheckpointId, + ) -> Result, Self::Error> { + todo!() + } + + fn with_checkpoints(&mut self, _limit: usize, _callback: F) -> Result<(), Self::Error> + where + F: FnMut(&Self::CheckpointId, &Checkpoint) -> Result<(), Self::Error>, + { + todo!() + } + + fn update_checkpoint_with( + &mut self, + _checkpoint_id: &Self::CheckpointId, + _update: F, + ) -> Result + where + F: Fn(&mut Checkpoint) -> Result<(), Self::Error>, + { + todo!() + } + + fn remove_checkpoint( + &mut self, + _checkpoint_id: &Self::CheckpointId, + ) -> Result<(), Self::Error> { + todo!() + } + + fn truncate_checkpoints( + &mut self, + _checkpoint_id: &Self::CheckpointId, + ) -> Result<(), Self::Error> { + todo!() + } +} diff --git a/zcash_primitives/src/merkle_tree.rs b/zcash_primitives/src/merkle_tree.rs index 176d3b437..6cda449bc 100644 --- a/zcash_primitives/src/merkle_tree.rs +++ b/zcash_primitives/src/merkle_tree.rs @@ -98,7 +98,7 @@ pub fn write_nonempty_frontier_v1( frontier: &NonEmptyFrontier, ) -> io::Result<()> { write_position(&mut writer, frontier.position())?; - if frontier.position().is_odd() { + if frontier.position().is_right_child() { // The v1 serialization wrote the sibling of a right-hand leaf as an optional value, rather // than as part of the ommers vector. frontier From ed2e22b73724b0e95896cd7c13620ef91da555d1 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 24 May 2023 21:57:54 -0600 Subject: [PATCH 03/27] zcash_client_sqlite: Add shard serialization & parsing --- zcash_client_sqlite/Cargo.toml | 5 +- zcash_client_sqlite/src/lib.rs | 1 + zcash_client_sqlite/src/serialization.rs | 114 +++++++++++++++++++++++ 3 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 zcash_client_sqlite/src/serialization.rs diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index 88585e0d2..6e3621b1a 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -17,6 +17,7 @@ rust-version = "1.65" incrementalmerkletree = { version = "0.4", features = ["legacy-api"] } shardtree = { version = "0.0", features = ["legacy-api"] } zcash_client_backend = { version = "0.9", path = "../zcash_client_backend" } +zcash_encoding = { version = "0.2", path = "../components/zcash_encoding" } zcash_primitives = { version = "0.12", path = "../zcash_primitives", default-features = false } # Dependencies exposed in a public API: @@ -28,7 +29,8 @@ hdwallet = { version = "0.4", optional = true } # - Logging and metrics tracing = "0.1" -# - Protobuf interfaces +# - Serialization +byteorder = "1" prost = "0.11" # - Secret management @@ -49,6 +51,7 @@ uuid = "1.1" [dev-dependencies] assert_matches = "1.5" incrementalmerkletree = { version = "0.4", features = ["legacy-api", "test-dependencies"] } +shardtree = { version = "0.0", features = ["legacy-api", "test-dependencies"] } proptest = "1.0.0" rand_core = "0.6" regex = "1.4" diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index e17cbab07..c9062fe94 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -78,6 +78,7 @@ use { pub mod chain; pub mod error; +mod serialization; pub mod wallet; /// The maximum number of blocks the wallet is allowed to rewind. This is diff --git a/zcash_client_sqlite/src/serialization.rs b/zcash_client_sqlite/src/serialization.rs new file mode 100644 index 000000000..99cb90dd8 --- /dev/null +++ b/zcash_client_sqlite/src/serialization.rs @@ -0,0 +1,114 @@ +//! Serialization formats for data stored as SQLite BLOBs + +use byteorder::{ReadBytesExt, WriteBytesExt}; +use core::ops::Deref; +use shardtree::{Node, PrunableTree, RetentionFlags, Tree}; +use std::io::{self, Read, Write}; +use std::rc::Rc; +use zcash_encoding::Optional; +use zcash_primitives::merkle_tree::HashSer; + +const SER_V1: u8 = 1; + +const NIL_TAG: u8 = 0; +const LEAF_TAG: u8 = 1; +const PARENT_TAG: u8 = 2; + +pub fn write_shard_v1( + writer: &mut W, + tree: &PrunableTree, +) -> io::Result<()> { + fn write_inner( + mut writer: &mut W, + tree: &PrunableTree, + ) -> io::Result<()> { + match tree.deref() { + Node::Parent { ann, left, right } => { + writer.write_u8(PARENT_TAG)?; + Optional::write(&mut writer, ann.as_ref(), |w, h| { + ::write(h, w) + })?; + write_inner(writer, left)?; + write_inner(writer, right)?; + Ok(()) + } + Node::Leaf { value } => { + writer.write_u8(LEAF_TAG)?; + value.0.write(&mut writer)?; + writer.write_u8(value.1.bits())?; + Ok(()) + } + Node::Nil => { + writer.write_u8(NIL_TAG)?; + Ok(()) + } + } + } + + writer.write_u8(SER_V1)?; + write_inner(writer, tree) +} + +fn read_shard_v1(mut reader: &mut R) -> io::Result> { + match reader.read_u8()? { + PARENT_TAG => { + let ann = Optional::read(&mut reader, ::read)?.map(Rc::new); + let left = read_shard_v1(reader)?; + let right = read_shard_v1(reader)?; + Ok(Tree::parent(ann, left, right)) + } + LEAF_TAG => { + let value = ::read(&mut reader)?; + let flags = reader.read_u8().and_then(|bits| { + RetentionFlags::from_bits(bits).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + format!( + "Byte value {} does not correspond to a valid set of retention flags", + bits + ), + ) + }) + })?; + Ok(Tree::leaf((value, flags))) + } + NIL_TAG => Ok(Tree::empty()), + other => Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("Node tag not recognized: {}", other), + )), + } +} + +pub fn read_shard(mut reader: R) -> io::Result> { + match reader.read_u8()? { + SER_V1 => read_shard_v1(&mut reader), + other => Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("Shard serialization version not recognized: {}", other), + )), + } +} + +#[cfg(test)] +mod tests { + use incrementalmerkletree::frontier::testing::{arb_test_node, TestNode}; + use proptest::prelude::*; + use shardtree::testing::arb_prunable_tree; + use std::io::Cursor; + + use super::{read_shard, write_shard_v1}; + + proptest! { + #[test] + fn check_shard_roundtrip( + tree in arb_prunable_tree(arb_test_node(), 8, 32) + ) { + let mut tree_data = vec![]; + write_shard_v1(&mut tree_data, &tree).unwrap(); + let cursor = Cursor::new(tree_data); + let tree_result = read_shard::(cursor).unwrap(); + assert_eq!(tree, tree_result); + } + } +} From 9f2bb94a5e30cd8d51c9ef358ce9246e6c6414c9 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 3 Apr 2023 13:53:43 -0600 Subject: [PATCH 04/27] zcash_client_sqlite: Add shard persistence to wallet migration. --- zcash_client_sqlite/Cargo.toml | 6 +- zcash_client_sqlite/src/error.rs | 8 +- zcash_client_sqlite/src/lib.rs | 29 ++-- zcash_client_sqlite/src/wallet/init.rs | 30 +++- .../init/migrations/shardtree_support.rs | 138 +++++++++++++++++- .../src/wallet/sapling/commitment_tree.rs | 33 ++++- 6 files changed, 213 insertions(+), 31 deletions(-) diff --git a/zcash_client_sqlite/Cargo.toml b/zcash_client_sqlite/Cargo.toml index 6e3621b1a..6a951ff10 100644 --- a/zcash_client_sqlite/Cargo.toml +++ b/zcash_client_sqlite/Cargo.toml @@ -32,13 +32,14 @@ tracing = "0.1" # - Serialization byteorder = "1" prost = "0.11" +either = "1.8" +group = "0.13" +jubjub = "0.10" # - Secret management secrecy = "0.8" # - SQLite databases -group = "0.13" -jubjub = "0.10" rusqlite = { version = "0.29.0", features = ["bundled", "time", "array"] } schemer = "0.2" schemer-rusqlite = "0.2.2" @@ -67,6 +68,7 @@ test-dependencies = [ "incrementalmerkletree/test-dependencies", "zcash_primitives/test-dependencies", "zcash_client_backend/test-dependencies", + "incrementalmerkletree/test-dependencies", ] transparent-inputs = ["hdwallet", "zcash_client_backend/transparent-inputs"] unstable = ["zcash_client_backend/unstable"] diff --git a/zcash_client_sqlite/src/error.rs b/zcash_client_sqlite/src/error.rs index 3f1832411..87f88b844 100644 --- a/zcash_client_sqlite/src/error.rs +++ b/zcash_client_sqlite/src/error.rs @@ -1,7 +1,9 @@ //! Error types for problems that may arise when reading or storing wallet data to SQLite. +use either::Either; use std::error; use std::fmt; +use std::io; use shardtree::ShardTreeError; use zcash_client_backend::encoding::{Bech32DecodeError, TransparentCodecError}; @@ -77,7 +79,7 @@ pub enum SqliteClientError { AddressNotRecognized(TransparentAddress), /// An error occurred in inserting data into one of the wallet's note commitment trees. - CommitmentTree(ShardTreeError), + CommitmentTree(ShardTreeError>), } impl error::Error for SqliteClientError { @@ -166,8 +168,8 @@ impl From for SqliteClientError { } } -impl From> for SqliteClientError { - fn from(e: ShardTreeError) -> Self { +impl From>> for SqliteClientError { + fn from(e: ShardTreeError>) -> Self { SqliteClientError::CommitmentTree(e) } } diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index c9062fe94..88b884b12 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -32,9 +32,10 @@ // Catch documentation errors caused by code changes. #![deny(rustdoc::broken_intra_doc_links)] +use either::Either; use rusqlite::{self, Connection}; use secrecy::{ExposeSecret, SecretVec}; -use std::{borrow::Borrow, collections::HashMap, convert::AsRef, fmt, ops::Range, path::Path}; +use std::{borrow::Borrow, collections::HashMap, convert::AsRef, fmt, io, ops::Range, path::Path}; use incrementalmerkletree::Position; use shardtree::{ShardTree, ShardTreeError}; @@ -72,13 +73,13 @@ use crate::{ #[cfg(feature = "unstable")] use { crate::chain::{fsblockdb_with_blocks, BlockMeta}, + std::fs, std::path::PathBuf, - std::{fs, io}, }; pub mod chain; pub mod error; -mod serialization; +pub mod serialization; pub mod wallet; /// The maximum number of blocks the wallet is allowed to rewind. This is @@ -614,7 +615,7 @@ impl WalletWrite for WalletDb } impl WalletCommitmentTrees for WalletDb { - type Error = rusqlite::Error; + type Error = Either; type SaplingShardStore<'a> = WalletDbSaplingShardStore<'a, 'a>; fn with_sapling_tree_mut(&mut self, mut callback: F) -> Result @@ -626,22 +627,26 @@ impl WalletCommitmentTrees for WalletDb, ) -> Result, - E: From>, + E: From>>, { - let tx = self.conn.transaction().map_err(ShardTreeError::Storage)?; - let shard_store = - WalletDbSaplingShardStore::from_connection(&tx).map_err(ShardTreeError::Storage)?; + let tx = self + .conn + .transaction() + .map_err(|e| ShardTreeError::Storage(Either::Right(e)))?; + let shard_store = WalletDbSaplingShardStore::from_connection(&tx) + .map_err(|e| ShardTreeError::Storage(Either::Right(e)))?; let result = { let mut shardtree = ShardTree::new(shard_store, 100); callback(&mut shardtree)? }; - tx.commit().map_err(ShardTreeError::Storage)?; + tx.commit() + .map_err(|e| ShardTreeError::Storage(Either::Right(e)))?; Ok(result) } } impl<'conn, P: consensus::Parameters> WalletCommitmentTrees for WalletDb, P> { - type Error = rusqlite::Error; + type Error = Either; type SaplingShardStore<'a> = WalletDbSaplingShardStore<'a, 'a>; fn with_sapling_tree_mut(&mut self, mut callback: F) -> Result @@ -653,11 +658,11 @@ impl<'conn, P: consensus::Parameters> WalletCommitmentTrees for WalletDb, ) -> Result, - E: From>, + E: From>>, { let mut shardtree = ShardTree::new( WalletDbSaplingShardStore::from_connection(&self.conn.0) - .map_err(ShardTreeError::Storage)?, + .map_err(|e| ShardTreeError::Storage(Either::Right(e)))?, 100, ); let result = callback(&mut shardtree)?; diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index bb8834a6a..e3485c8ab 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -1,6 +1,6 @@ //! Functions for initializing the various databases. -use std::collections::HashMap; -use std::fmt; +use either::Either; +use std::{collections::HashMap, fmt, io}; use rusqlite::{self, types::ToSql}; use schemer::{Migrator, MigratorError}; @@ -37,7 +37,7 @@ pub enum WalletMigrationError { BalanceError(BalanceError), /// Wrapper for commitment tree invariant violations - CommitmentTree(ShardTreeError), + CommitmentTree(ShardTreeError>), } impl From for WalletMigrationError { @@ -52,8 +52,8 @@ impl From for WalletMigrationError { } } -impl From> for WalletMigrationError { - fn from(e: ShardTreeError) -> Self { +impl From>> for WalletMigrationError { + fn from(e: ShardTreeError>) -> Self { WalletMigrationError::CommitmentTree(e) } } @@ -393,6 +393,26 @@ mod tests { FOREIGN KEY (spent) REFERENCES transactions(id_tx), CONSTRAINT tx_output UNIQUE (tx, output_index) )", + "CREATE TABLE sapling_tree_cap ( + cap_data BLOB NOT NULL + )", + "CREATE TABLE sapling_tree_checkpoint_marks_removed ( + checkpoint_id INTEGER NOT NULL, + mark_removed_position INTEGER NOT NULL, + FOREIGN KEY (checkpoint_id) REFERENCES sapling_tree_checkpoints(checkpoint_id) + )", + "CREATE TABLE sapling_tree_checkpoints ( + checkpoint_id INTEGER PRIMARY KEY, + position INTEGER + )", + "CREATE TABLE sapling_tree_shards ( + shard_index INTEGER PRIMARY KEY, + subtree_end_height INTEGER, + root_hash BLOB, + shard_data BLOB, + contains_marked INTEGER, + CONSTRAINT root_unique UNIQUE (root_hash) + )", "CREATE TABLE sapling_witnesses ( id_witness INTEGER PRIMARY KEY, note INTEGER NOT NULL, diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index b9e4a6bf0..f46d63ec3 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -2,14 +2,25 @@ //! tree data using the `shardtree` crate, and migrates existing witness data into these data //! structures. -use std::collections::HashSet; +use std::collections::{BTreeSet, HashSet}; -use rusqlite; +use incrementalmerkletree::Retention; +use rusqlite::{self, named_params, params}; use schemer; use schemer_rusqlite::RusqliteMigration; +use shardtree::ShardTree; use uuid::Uuid; -use crate::wallet::init::{migrations::received_notes_nullable_nf, WalletMigrationError}; +use zcash_primitives::{ + consensus::BlockHeight, + merkle_tree::{read_commitment_tree, read_incremental_witness}, + sapling, +}; + +use crate::wallet::{ + init::{migrations::received_notes_nullable_nf, WalletMigrationError}, + sapling::commitment_tree::WalletDbSaplingShardStore, +}; pub(super) const MIGRATION_ID: Uuid = Uuid::from_fields( 0x7da6489d, @@ -47,10 +58,129 @@ impl RusqliteMigration for Migration { ALTER TABLE sapling_received_notes ADD COLUMN commitment_tree_position INTEGER;", )?; + // Add shard persistence + transaction.execute_batch( + "CREATE TABLE sapling_tree_shards ( + shard_index INTEGER PRIMARY KEY, + subtree_end_height INTEGER, + root_hash BLOB, + shard_data BLOB, + contains_marked INTEGER NOT NULL, + CONSTRAINT root_unique UNIQUE (root_hash) + ); + CREATE TABLE sapling_tree_cap ( + cap_data BLOB NOT NULL + );", + )?; + + // Add checkpoint persistence + transaction.execute_batch( + "CREATE TABLE sapling_tree_checkpoints ( + checkpoint_id INTEGER PRIMARY KEY, + position INTEGER + ); + CREATE TABLE sapling_tree_checkpoint_marks_removed ( + checkpoint_id INTEGER NOT NULL, + mark_removed_position INTEGER NOT NULL, + FOREIGN KEY (checkpoint_id) REFERENCES sapling_tree_checkpoints(checkpoint_id) + );", + )?; + + let shard_store = WalletDbSaplingShardStore::from_connection(transaction)?; + let mut shard_tree: ShardTree< + _, + { sapling::NOTE_COMMITMENT_TREE_DEPTH }, + { sapling::NOTE_COMMITMENT_TREE_DEPTH / 2 }, + > = ShardTree::new(shard_store, 100); + // Insert all the tree information that we can get from block-end commitment trees + { + let mut stmt_blocks = transaction.prepare("SELECT height, sapling_tree FROM blocks")?; + let mut stmt_update_block_sapling_tree_size = transaction + .prepare("UPDATE blocks SET sapling_commitment_tree_size = ? WHERE height = ?")?; + + let mut block_rows = stmt_blocks.query([])?; + while let Some(row) = block_rows.next()? { + let block_height: u32 = row.get(0)?; + let row_data: Vec = row.get(1)?; + let block_end_tree = read_commitment_tree::< + sapling::Node, + _, + { sapling::NOTE_COMMITMENT_TREE_DEPTH }, + >(&row_data[..]) + .map_err(|e| { + rusqlite::Error::FromSqlConversionFailure( + row_data.len(), + rusqlite::types::Type::Blob, + Box::new(e), + ) + })?; + stmt_update_block_sapling_tree_size + .execute(params![block_end_tree.size(), block_height])?; + + if let Some(nonempty_frontier) = block_end_tree.to_frontier().value() { + shard_tree.insert_frontier_nodes( + nonempty_frontier.clone(), + Retention::Checkpoint { + id: BlockHeight::from(block_height), + is_marked: false, + }, + )?; + } + } + } + + // Insert all the tree information that we can get from existing incremental witnesses + { + let mut stmt_blocks = + transaction.prepare("SELECT note, block, witness FROM sapling_witnesses")?; + let mut stmt_set_note_position = transaction.prepare( + "UPDATE sapling_received_notes + SET commitment_tree_position = :position + WHERE id_note = :note_id", + )?; + let mut updated_note_positions = BTreeSet::new(); + let mut rows = stmt_blocks.query([])?; + while let Some(row) = rows.next()? { + let note_id: i64 = row.get(0)?; + let block_height: u32 = row.get(1)?; + let row_data: Vec = row.get(2)?; + let witness = read_incremental_witness::< + sapling::Node, + _, + { sapling::NOTE_COMMITMENT_TREE_DEPTH }, + >(&row_data[..]) + .map_err(|e| { + rusqlite::Error::FromSqlConversionFailure( + row_data.len(), + rusqlite::types::Type::Blob, + Box::new(e), + ) + })?; + + let witnessed_position = witness.witnessed_position(); + if !updated_note_positions.contains(&witnessed_position) { + stmt_set_note_position.execute(named_params![ + ":note_id": note_id, + ":position": u64::from(witnessed_position) + ])?; + updated_note_positions.insert(witnessed_position); + } + + shard_tree.insert_witness_nodes(witness, BlockHeight::from(block_height))?; + } + } + Ok(()) } - fn down(&self, _transaction: &rusqlite::Transaction) -> Result<(), WalletMigrationError> { + fn down(&self, transaction: &rusqlite::Transaction) -> Result<(), WalletMigrationError> { + transaction.execute_batch( + "DROP TABLE sapling_tree_checkpoint_marks_removed; + DROP TABLE sapling_tree_checkpoints; + DROP TABLE sapling_tree_cap; + DROP TABLE sapling_tree_shards;", + )?; + Ok(()) } } diff --git a/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs b/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs index a02ce4cd8..ee7438ad6 100644 --- a/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs @@ -1,9 +1,13 @@ +use either::Either; use incrementalmerkletree::Address; -use rusqlite; +use rusqlite::{self, named_params, OptionalExtension}; use shardtree::{Checkpoint, LocatedPrunableTree, PrunableTree, ShardStore}; +use std::io::{self, Cursor}; use zcash_primitives::{consensus::BlockHeight, sapling}; +use crate::serialization::read_shard; + pub struct WalletDbSaplingShardStore<'conn, 'a> { pub(crate) conn: &'a rusqlite::Transaction<'conn>, } @@ -19,14 +23,13 @@ impl<'conn, 'a> WalletDbSaplingShardStore<'conn, 'a> { impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { type H = sapling::Node; type CheckpointId = BlockHeight; - type Error = rusqlite::Error; + type Error = Either; fn get_shard( &self, - _shard_root: Address, + shard_root: Address, ) -> Result>, Self::Error> { - // SELECT shard_data FROM sapling_tree WHERE shard_index = shard_root.index - todo!() + get_shard(self.conn, shard_root) } fn last_shard(&self) -> Result>, Self::Error> { @@ -121,3 +124,23 @@ impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { todo!() } } + +pub(crate) fn get_shard( + conn: &rusqlite::Connection, + shard_root: Address, +) -> Result>, Either> { + conn.query_row( + "SELECT shard_data + FROM sapling_tree_shards + WHERE shard_index = :shard_index", + named_params![":shard_index": shard_root.index()], + |row| row.get::<_, Vec>(0), + ) + .optional() + .map_err(Either::Right)? + .map(|shard_data| { + let shard_tree = read_shard(&mut Cursor::new(shard_data)).map_err(Either::Left)?; + Ok(LocatedPrunableTree::from_parts(shard_root, shard_tree)) + }) + .transpose() +} From ade882d01cb74062d7cb70671e20c67575d04e02 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 2 Jun 2023 07:53:26 -0600 Subject: [PATCH 05/27] zcash_client_sqlite: Add shard & checkpoint insertion. --- zcash_client_sqlite/src/wallet.rs | 2 +- zcash_client_sqlite/src/wallet/init.rs | 4 +- .../init/migrations/add_transaction_views.rs | 4 +- .../migrations/received_notes_nullable_nf.rs | 2 +- .../init/migrations/shardtree_support.rs | 13 ++- .../init/migrations/v_transactions_net.rs | 8 +- .../src/wallet/sapling/commitment_tree.rs | 88 +++++++++++++++++-- 7 files changed, 98 insertions(+), 23 deletions(-) diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 406db08c7..8f071d55d 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -549,7 +549,7 @@ pub(crate) fn fully_scanned_height( |row| { let max_height: u32 = row.get(0)?; let sapling_tree_size: Option = row.get(1)?; - let sapling_tree: Vec = row.get(0)?; + let sapling_tree: Vec = row.get(2)?; Ok(( BlockHeight::from(max_height), sapling_tree_size, diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index e3485c8ab..6fba9396c 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -875,7 +875,7 @@ mod tests { // add a sapling sent note wdb.conn.execute( - "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, '')", + "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, x'00')", [], )?; @@ -1039,7 +1039,7 @@ mod tests { RecipientAddress::Transparent(*ufvk.default_address().0.transparent().unwrap()) .encode(&tests::network()); wdb.conn.execute( - "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, '')", + "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, x'00')", [], )?; wdb.conn.execute( diff --git a/zcash_client_sqlite/src/wallet/init/migrations/add_transaction_views.rs b/zcash_client_sqlite/src/wallet/init/migrations/add_transaction_views.rs index 70694f842..06efb1327 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/add_transaction_views.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/add_transaction_views.rs @@ -327,7 +327,7 @@ mod tests { .unwrap(); db_data.conn.execute_batch( - "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, ''); + "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, x'00'); INSERT INTO transactions (block, id_tx, txid) VALUES (0, 0, ''); INSERT INTO sent_notes (tx, output_pool, output_index, from_account, address, value) @@ -460,7 +460,7 @@ mod tests { db_data .conn .execute_batch( - "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, '');", + "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, x'00');", ) .unwrap(); db_data.conn.execute( diff --git a/zcash_client_sqlite/src/wallet/init/migrations/received_notes_nullable_nf.rs b/zcash_client_sqlite/src/wallet/init/migrations/received_notes_nullable_nf.rs index 811a1a0e5..5567d60dc 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/received_notes_nullable_nf.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/received_notes_nullable_nf.rs @@ -262,7 +262,7 @@ mod tests { // Tx 0 contains two received notes of 2 and 5 zatoshis that are controlled by account 0. db_data.conn.execute_batch( - "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, ''); + "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, x'00'); INSERT INTO transactions (block, id_tx, txid) VALUES (0, 0, 'tx0'); INSERT INTO received_notes (tx, output_index, account, diversifier, value, rcm, nf, is_change) diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index f46d63ec3..e16c36c8c 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -65,7 +65,7 @@ impl RusqliteMigration for Migration { subtree_end_height INTEGER, root_hash BLOB, shard_data BLOB, - contains_marked INTEGER NOT NULL, + contains_marked INTEGER, CONSTRAINT root_unique UNIQUE (root_hash) ); CREATE TABLE sapling_tree_cap ( @@ -101,19 +101,24 @@ impl RusqliteMigration for Migration { let mut block_rows = stmt_blocks.query([])?; while let Some(row) = block_rows.next()? { let block_height: u32 = row.get(0)?; - let row_data: Vec = row.get(1)?; + let sapling_tree_data: Vec = row.get(1)?; + if sapling_tree_data == vec![0x00] { + continue; + } + let block_end_tree = read_commitment_tree::< sapling::Node, _, { sapling::NOTE_COMMITMENT_TREE_DEPTH }, - >(&row_data[..]) + >(&sapling_tree_data[..]) .map_err(|e| { rusqlite::Error::FromSqlConversionFailure( - row_data.len(), + sapling_tree_data.len(), rusqlite::types::Type::Blob, Box::new(e), ) })?; + stmt_update_block_sapling_tree_size .execute(params![block_end_tree.size(), block_height])?; diff --git a/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_net.rs b/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_net.rs index 10c3a26a9..fc3ab7378 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_net.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/v_transactions_net.rs @@ -253,7 +253,7 @@ mod tests { // - Tx 0 contains two received notes of 2 and 5 zatoshis that are controlled by account 0. db_data.conn.execute_batch( - "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, ''); + "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, x'00'); INSERT INTO transactions (block, id_tx, txid) VALUES (0, 0, 'tx0'); INSERT INTO received_notes (tx, output_index, account, diversifier, value, rcm, nf, is_change) @@ -265,7 +265,7 @@ mod tests { // of 2 zatoshis. This is representative of a historic transaction where no `sent_notes` // entry was created for the change value. db_data.conn.execute_batch( - "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (1, 1, 1, ''); + "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (1, 1, 1, x'00'); INSERT INTO transactions (block, id_tx, txid) VALUES (1, 1, 'tx1'); UPDATE received_notes SET spent = 1 WHERE tx = 0; INSERT INTO sent_notes (tx, output_pool, output_index, from_account, to_account, to_address, value) @@ -279,7 +279,7 @@ mod tests { // other half to the sending account as change. Also there's a random transparent utxo, // received, who knows where it came from but it's for account 0. db_data.conn.execute_batch( - "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (2, 2, 2, ''); + "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (2, 2, 2, x'00'); INSERT INTO transactions (block, id_tx, txid) VALUES (2, 2, 'tx2'); UPDATE received_notes SET spent = 2 WHERE tx = 1; INSERT INTO utxos (received_by_account, address, prevout_txid, prevout_idx, script, value_zat, height) @@ -297,7 +297,7 @@ mod tests { // - Tx 3 just receives transparent funds and does nothing else. For this to work, the // transaction must be retrieved by the wallet. db_data.conn.execute_batch( - "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (3, 3, 3, ''); + "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (3, 3, 3, x'00'); INSERT INTO transactions (block, id_tx, txid) VALUES (3, 3, 'tx3'); INSERT INTO utxos (received_by_account, address, prevout_txid, prevout_idx, script, value_zat, height) diff --git a/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs b/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs index ee7438ad6..80c800fe6 100644 --- a/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs @@ -1,12 +1,14 @@ use either::Either; + use incrementalmerkletree::Address; use rusqlite::{self, named_params, OptionalExtension}; use shardtree::{Checkpoint, LocatedPrunableTree, PrunableTree, ShardStore}; + use std::io::{self, Cursor}; -use zcash_primitives::{consensus::BlockHeight, sapling}; +use zcash_primitives::{consensus::BlockHeight, merkle_tree::HashSer, sapling}; -use crate::serialization::read_shard; +use crate::serialization::{read_shard, write_shard_v1}; pub struct WalletDbSaplingShardStore<'conn, 'a> { pub(crate) conn: &'a rusqlite::Transaction<'conn>, @@ -37,8 +39,8 @@ impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { todo!() } - fn put_shard(&mut self, _subtree: LocatedPrunableTree) -> Result<(), Self::Error> { - todo!() + fn put_shard(&mut self, subtree: LocatedPrunableTree) -> Result<(), Self::Error> { + put_shard(self.conn, subtree) } fn get_shard_roots(&self) -> Result, Self::Error> { @@ -68,14 +70,14 @@ impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { fn add_checkpoint( &mut self, - _checkpoint_id: Self::CheckpointId, - _checkpoint: Checkpoint, + checkpoint_id: Self::CheckpointId, + checkpoint: Checkpoint, ) -> Result<(), Self::Error> { - todo!() + add_checkpoint(self.conn, checkpoint_id, checkpoint) } fn checkpoint_count(&self) -> Result { - todo!() + checkpoint_count(self.conn) } fn get_checkpoint_at_depth( @@ -125,10 +127,12 @@ impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { } } +type Error = Either; + pub(crate) fn get_shard( conn: &rusqlite::Connection, shard_root: Address, -) -> Result>, Either> { +) -> Result>, Error> { conn.query_row( "SELECT shard_data FROM sapling_tree_shards @@ -144,3 +148,69 @@ pub(crate) fn get_shard( }) .transpose() } + +pub(crate) fn put_shard( + conn: &rusqlite::Connection, + subtree: LocatedPrunableTree, +) -> Result<(), Error> { + let subtree_root_hash = subtree + .root() + .annotation() + .and_then(|ann| { + ann.as_ref().map(|rc| { + let mut root_hash = vec![]; + rc.write(&mut root_hash)?; + Ok(root_hash) + }) + }) + .transpose() + .map_err(Either::Left)?; + + let mut subtree_data = vec![]; + write_shard_v1(&mut subtree_data, subtree.root()).map_err(Either::Left)?; + + conn.prepare_cached( + "INSERT INTO sapling_tree_shards (shard_index, root_hash, shard_data) + VALUES (:shard_index, :root_hash, :shard_data) + ON CONFLICT (shard_index) DO UPDATE + SET root_hash = :root_hash, + shard_data = :shard_data", + ) + .and_then(|mut stmt_put_shard| { + stmt_put_shard.execute(named_params![ + ":shard_index": subtree.root_addr().index(), + ":root_hash": subtree_root_hash, + ":shard_data": subtree_data + ]) + }) + .map_err(Either::Right)?; + + Ok(()) +} + +pub(crate) fn add_checkpoint( + conn: &rusqlite::Transaction<'_>, + checkpoint_id: BlockHeight, + checkpoint: Checkpoint, +) -> Result<(), Error> { + conn.prepare_cached( + "INSERT INTO sapling_tree_checkpoints (checkpoint_id, position) + VALUES (:checkpoint_id, :position)", + ) + .and_then(|mut stmt_insert_checkpoint| { + stmt_insert_checkpoint.execute(named_params![ + ":checkpoint_id": u32::from(checkpoint_id), + ":position": checkpoint.position().map(u64::from) + ]) + }) + .map_err(Either::Right)?; + + Ok(()) +} + +pub(crate) fn checkpoint_count(conn: &rusqlite::Connection) -> Result { + conn.query_row("SELECT COUNT(*) FROM sapling_tree_checkpoints", [], |row| { + row.get::<_, usize>(0) + }) + .map_err(Either::Right) +} From d11f3d2acc11d92f3f819e8f765516304501f9ab Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Tue, 13 Jun 2023 11:20:18 -0600 Subject: [PATCH 06/27] zcash_client_sqlite: Add shardtree truncation & checkpoint operations. --- zcash_client_sqlite/src/chain.rs | 4 +- zcash_client_sqlite/src/lib.rs | 53 ++--- zcash_client_sqlite/src/wallet.rs | 17 +- zcash_client_sqlite/src/wallet/init.rs | 5 +- .../init/migrations/shardtree_support.rs | 3 + .../src/wallet/sapling/commitment_tree.rs | 181 +++++++++++++++--- 6 files changed, 209 insertions(+), 54 deletions(-) diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index 11e065f9e..d115482e2 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -539,7 +539,7 @@ mod tests { // "Rewind" to height of last scanned block db_data .transactionally(|wdb| { - truncate_to_height(&wdb.conn.0, &wdb.params, sapling_activation_height() + 1) + truncate_to_height(wdb.conn.0, &wdb.params, sapling_activation_height() + 1) }) .unwrap(); @@ -552,7 +552,7 @@ mod tests { // Rewind so that one block is dropped db_data .transactionally(|wdb| { - truncate_to_height(&wdb.conn.0, &wdb.params, sapling_activation_height()) + truncate_to_height(wdb.conn.0, &wdb.params, sapling_activation_height()) }) .unwrap(); diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 88b884b12..cae3685e7 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -116,11 +116,11 @@ pub struct WalletDb { } /// A wrapper for a SQLite transaction affecting the wallet database. -pub struct SqlTransaction<'conn>(pub(crate) rusqlite::Transaction<'conn>); +pub struct SqlTransaction<'conn>(pub(crate) &'conn rusqlite::Transaction<'conn>); impl Borrow for SqlTransaction<'_> { fn borrow(&self) -> &rusqlite::Connection { - &self.0 + self.0 } } @@ -137,12 +137,13 @@ impl WalletDb { where F: FnOnce(&mut WalletDb, P>) -> Result, { + let tx = self.conn.transaction()?; let mut wdb = WalletDb { - conn: SqlTransaction(self.conn.transaction()?), + conn: SqlTransaction(&tx), params: self.params.clone(), }; let result = f(&mut wdb)?; - wdb.conn.0.commit()?; + tx.commit()?; Ok(result) } } @@ -334,7 +335,7 @@ impl WalletWrite for WalletDb seed: &SecretVec, ) -> Result<(AccountId, UnifiedSpendingKey), Self::Error> { self.transactionally(|wdb| { - let account = wallet::get_max_account_id(&wdb.conn.0)? + let account = wallet::get_max_account_id(wdb.conn.0)? .map(|a| AccountId::from(u32::from(a) + 1)) .unwrap_or_else(|| AccountId::from(0)); @@ -346,7 +347,7 @@ impl WalletWrite for WalletDb .map_err(|_| SqliteClientError::KeyDerivationError(account))?; let ufvk = usk.to_unified_full_viewing_key(); - wallet::add_account(&wdb.conn.0, &wdb.params, account, &ufvk)?; + wallet::add_account(wdb.conn.0, &wdb.params, account, &ufvk)?; Ok((account, usk)) }) @@ -360,7 +361,7 @@ impl WalletWrite for WalletDb |wdb| match wdb.get_unified_full_viewing_keys()?.get(&account) { Some(ufvk) => { let search_from = - match wallet::get_current_address(&wdb.conn.0, &wdb.params, account)? { + match wallet::get_current_address(wdb.conn.0, &wdb.params, account)? { Some((_, mut last_diversifier_index)) => { last_diversifier_index .increment() @@ -375,7 +376,7 @@ impl WalletWrite for WalletDb .ok_or(SqliteClientError::DiversifierIndexOutOfRange)?; wallet::insert_address( - &wdb.conn.0, + wdb.conn.0, &wdb.params, account, diversifier_index, @@ -399,7 +400,7 @@ impl WalletWrite for WalletDb // Insert the block into the database. let block_height = block.block_height; wallet::insert_block( - &wdb.conn.0, + wdb.conn.0, block_height, block.block_hash, block.block_time, @@ -408,16 +409,16 @@ impl WalletWrite for WalletDb let mut wallet_note_ids = vec![]; for tx in &block.transactions { - let tx_row = wallet::put_tx_meta(&wdb.conn.0, tx, block.block_height)?; + let tx_row = wallet::put_tx_meta(wdb.conn.0, tx, block.block_height)?; // Mark notes as spent and remove them from the scanning cache for spend in &tx.sapling_spends { - wallet::sapling::mark_sapling_note_spent(&wdb.conn.0, tx_row, spend.nf())?; + wallet::sapling::mark_sapling_note_spent(wdb.conn.0, tx_row, spend.nf())?; } for output in &tx.sapling_outputs { let received_note_id = - wallet::sapling::put_received_note(&wdb.conn.0, output, tx_row)?; + wallet::sapling::put_received_note(wdb.conn.0, output, tx_row)?; // Save witness for note. wallet_note_ids.push(received_note_id); @@ -435,7 +436,7 @@ impl WalletWrite for WalletDb })?; // Update now-expired transactions that didn't get mined. - wallet::update_expired_notes(&wdb.conn.0, block_height)?; + wallet::update_expired_notes(wdb.conn.0, block_height)?; Ok(wallet_note_ids) }) @@ -446,7 +447,7 @@ impl WalletWrite for WalletDb d_tx: DecryptedTransaction, ) -> Result { self.transactionally(|wdb| { - let tx_ref = wallet::put_tx_data(&wdb.conn.0, d_tx.tx, None, None)?; + let tx_ref = wallet::put_tx_data(wdb.conn.0, d_tx.tx, None, None)?; let mut spending_account_id: Option = None; for output in d_tx.sapling_outputs { @@ -459,7 +460,7 @@ impl WalletWrite for WalletDb }; wallet::put_sent_output( - &wdb.conn.0, + wdb.conn.0, &wdb.params, output.account, tx_ref, @@ -474,7 +475,7 @@ impl WalletWrite for WalletDb )?; if matches!(recipient, Recipient::InternalAccount(_, _)) { - wallet::sapling::put_received_note(&wdb.conn.0, output, tx_ref)?; + wallet::sapling::put_received_note(wdb.conn.0, output, tx_ref)?; } } TransferType::Incoming => { @@ -489,14 +490,14 @@ impl WalletWrite for WalletDb } } - wallet::sapling::put_received_note(&wdb.conn.0, output, tx_ref)?; + wallet::sapling::put_received_note(wdb.conn.0, output, tx_ref)?; } } // If any of the utxos spent in the transaction are ours, mark them as spent. #[cfg(feature = "transparent-inputs")] for txin in d_tx.tx.transparent_bundle().iter().flat_map(|b| b.vin.iter()) { - wallet::mark_transparent_utxo_spent(&wdb.conn.0, tx_ref, &txin.prevout)?; + wallet::mark_transparent_utxo_spent(wdb.conn.0, tx_ref, &txin.prevout)?; } // If we have some transparent outputs: @@ -513,7 +514,7 @@ impl WalletWrite for WalletDb for (output_index, txout) in d_tx.tx.transparent_bundle().iter().flat_map(|b| b.vout.iter()).enumerate() { if let Some(address) = txout.recipient_address() { wallet::put_sent_output( - &wdb.conn.0, + wdb.conn.0, &wdb.params, *account_id, tx_ref, @@ -534,7 +535,7 @@ impl WalletWrite for WalletDb fn store_sent_tx(&mut self, sent_tx: &SentTransaction) -> Result { self.transactionally(|wdb| { let tx_ref = wallet::put_tx_data( - &wdb.conn.0, + wdb.conn.0, sent_tx.tx, Some(sent_tx.fee_amount), Some(sent_tx.created), @@ -551,7 +552,7 @@ impl WalletWrite for WalletDb if let Some(bundle) = sent_tx.tx.sapling_bundle() { for spend in bundle.shielded_spends() { wallet::sapling::mark_sapling_note_spent( - &wdb.conn.0, + wdb.conn.0, tx_ref, spend.nullifier(), )?; @@ -560,12 +561,12 @@ impl WalletWrite for WalletDb #[cfg(feature = "transparent-inputs")] for utxo_outpoint in &sent_tx.utxos_spent { - wallet::mark_transparent_utxo_spent(&wdb.conn.0, tx_ref, utxo_outpoint)?; + wallet::mark_transparent_utxo_spent(wdb.conn.0, tx_ref, utxo_outpoint)?; } for output in &sent_tx.outputs { wallet::insert_sent_output( - &wdb.conn.0, + wdb.conn.0, &wdb.params, tx_ref, sent_tx.account, @@ -574,7 +575,7 @@ impl WalletWrite for WalletDb if let Some((account, note)) = output.sapling_change_to() { wallet::sapling::put_received_note( - &wdb.conn.0, + wdb.conn.0, &DecryptedOutput { index: output.output_index(), note: note.clone(), @@ -596,7 +597,7 @@ impl WalletWrite for WalletDb fn truncate_to_height(&mut self, block_height: BlockHeight) -> Result<(), Self::Error> { self.transactionally(|wdb| { - wallet::truncate_to_height(&wdb.conn.0, &wdb.params, block_height) + wallet::truncate_to_height(wdb.conn.0, &wdb.params, block_height) }) } @@ -661,7 +662,7 @@ impl<'conn, P: consensus::Parameters> WalletCommitmentTrees for WalletDb>>, { let mut shardtree = ShardTree::new( - WalletDbSaplingShardStore::from_connection(&self.conn.0) + WalletDbSaplingShardStore::from_connection(self.conn.0) .map_err(|e| ShardTreeError::Storage(Either::Right(e)))?, 100, ); diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 8f071d55d..6b2d15072 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -89,7 +89,9 @@ use zcash_client_backend::{ wallet::WalletTx, }; -use crate::{error::SqliteClientError, PRUNING_HEIGHT}; +use crate::{ + error::SqliteClientError, SqlTransaction, WalletCommitmentTrees, WalletDb, PRUNING_HEIGHT, +}; #[cfg(feature = "transparent-inputs")] use { @@ -637,7 +639,7 @@ pub(crate) fn get_min_unspent_height( /// block, this function does nothing. /// /// This should only be executed inside a transactional context. -pub(crate) fn truncate_to_height( +pub(crate) fn truncate_to_height( conn: &rusqlite::Transaction, params: &P, block_height: BlockHeight, @@ -662,7 +664,16 @@ pub(crate) fn truncate_to_height( // nothing to do if we're deleting back down to the max height if block_height < last_scanned_height { - // Decrement witnesses. + // Truncate the note commitment trees + let mut wdb = WalletDb { + conn: SqlTransaction(conn), + params: params.clone(), + }; + wdb.with_sapling_tree_mut(|tree| { + tree.truncate_removing_checkpoint(&block_height).map(|_| ()) + })?; + + // Remove any legacy Sapling witnesses conn.execute( "DELETE FROM sapling_witnesses WHERE block > ?", [u32::from(block_height)], diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index 6fba9396c..a98806ee6 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -237,7 +237,7 @@ pub fn init_accounts_table( // Insert accounts atomically for (account, key) in keys.iter() { - wallet::add_account(&wdb.conn.0, &wdb.params, *account, key)?; + wallet::add_account(wdb.conn.0, &wdb.params, *account, key)?; } Ok(()) @@ -394,6 +394,9 @@ mod tests { CONSTRAINT tx_output UNIQUE (tx, output_index) )", "CREATE TABLE sapling_tree_cap ( + -- cap_id exists only to be able to take advantage of `ON CONFLICT` + -- upsert functionality; the table will only ever contain one row + cap_id INTEGER PRIMARY KEY, cap_data BLOB NOT NULL )", "CREATE TABLE sapling_tree_checkpoint_marks_removed ( diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index e16c36c8c..f22b03c20 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -69,6 +69,9 @@ impl RusqliteMigration for Migration { CONSTRAINT root_unique UNIQUE (root_hash) ); CREATE TABLE sapling_tree_cap ( + -- cap_id exists only to be able to take advantage of `ON CONFLICT` + -- upsert functionality; the table will only ever contain one row + cap_id INTEGER PRIMARY KEY, cap_data BLOB NOT NULL );", )?; diff --git a/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs b/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs index 80c800fe6..912831fcf 100644 --- a/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs @@ -1,10 +1,14 @@ use either::Either; -use incrementalmerkletree::Address; -use rusqlite::{self, named_params, OptionalExtension}; -use shardtree::{Checkpoint, LocatedPrunableTree, PrunableTree, ShardStore}; +use incrementalmerkletree::{Address, Position}; +use rusqlite::{self, named_params, Connection, OptionalExtension}; +use shardtree::{Checkpoint, LocatedPrunableTree, PrunableTree, ShardStore, TreeState}; -use std::io::{self, Cursor}; +use std::{ + collections::BTreeSet, + io::{self, Cursor}, + ops::Deref, +}; use zcash_primitives::{consensus::BlockHeight, merkle_tree::HashSer, sapling}; @@ -48,16 +52,16 @@ impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { todo!() } - fn truncate(&mut self, _from: Address) -> Result<(), Self::Error> { - todo!() + fn truncate(&mut self, from: Address) -> Result<(), Self::Error> { + truncate(self.conn, from) } fn get_cap(&self) -> Result, Self::Error> { - todo!() + get_cap(self.conn) } - fn put_cap(&mut self, _cap: PrunableTree) -> Result<(), Self::Error> { - todo!() + fn put_cap(&mut self, cap: PrunableTree) -> Result<(), Self::Error> { + put_cap(self.conn, cap) } fn min_checkpoint_id(&self) -> Result, Self::Error> { @@ -89,9 +93,9 @@ impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { fn get_checkpoint( &self, - _checkpoint_id: &Self::CheckpointId, + checkpoint_id: &Self::CheckpointId, ) -> Result, Self::Error> { - todo!() + get_checkpoint(self.conn, *checkpoint_id) } fn with_checkpoints(&mut self, _limit: usize, _callback: F) -> Result<(), Self::Error> @@ -103,27 +107,24 @@ impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { fn update_checkpoint_with( &mut self, - _checkpoint_id: &Self::CheckpointId, - _update: F, + checkpoint_id: &Self::CheckpointId, + update: F, ) -> Result where F: Fn(&mut Checkpoint) -> Result<(), Self::Error>, { - todo!() + update_checkpoint_with(self.conn, *checkpoint_id, update) } - fn remove_checkpoint( - &mut self, - _checkpoint_id: &Self::CheckpointId, - ) -> Result<(), Self::Error> { - todo!() + fn remove_checkpoint(&mut self, checkpoint_id: &Self::CheckpointId) -> Result<(), Self::Error> { + remove_checkpoint(self.conn, *checkpoint_id) } fn truncate_checkpoints( &mut self, - _checkpoint_id: &Self::CheckpointId, + checkpoint_id: &Self::CheckpointId, ) -> Result<(), Self::Error> { - todo!() + truncate_checkpoints(self.conn, *checkpoint_id) } } @@ -134,7 +135,7 @@ pub(crate) fn get_shard( shard_root: Address, ) -> Result>, Error> { conn.query_row( - "SELECT shard_data + "SELECT shard_data FROM sapling_tree_shards WHERE shard_index = :shard_index", named_params![":shard_index": shard_root.index()], @@ -188,6 +189,47 @@ pub(crate) fn put_shard( Ok(()) } +pub(crate) fn truncate(conn: &rusqlite::Transaction<'_>, from: Address) -> Result<(), Error> { + conn.execute( + "DELETE FROM sapling_tree_shards WHERE shard_index >= ?", + [from.index()], + ) + .map_err(Either::Right) + .map(|_| ()) +} + +pub(crate) fn get_cap(conn: &rusqlite::Connection) -> Result, Error> { + conn.query_row("SELECT cap_data FROM sapling_tree_cap", [], |row| { + row.get::<_, Vec>(0) + }) + .optional() + .map_err(Either::Right)? + .map_or_else( + || Ok(PrunableTree::empty()), + |cap_data| read_shard(&mut Cursor::new(cap_data)).map_err(Either::Left), + ) +} + +pub(crate) fn put_cap( + conn: &rusqlite::Transaction<'_>, + cap: PrunableTree, +) -> Result<(), Error> { + let mut stmt = conn + .prepare_cached( + "INSERT INTO sapling_tree_cap (cap_id, cap_data) + VALUES (0, :cap_data) + ON CONFLICT (cap_id) DO UPDATE + SET cap_data = :cap_data", + ) + .map_err(Either::Right)?; + + let mut cap_data = vec![]; + write_shard_v1(&mut cap_data, &cap).map_err(Either::Left)?; + stmt.execute([cap_data]).map_err(Either::Right)?; + + Ok(()) +} + pub(crate) fn add_checkpoint( conn: &rusqlite::Transaction<'_>, checkpoint_id: BlockHeight, @@ -214,3 +256,98 @@ pub(crate) fn checkpoint_count(conn: &rusqlite::Connection) -> Result>( + conn: &C, + checkpoint_id: BlockHeight, +) -> Result, Either> { + let checkpoint_position = conn + .query_row( + "SELECT position + FROM sapling_tree_checkpoints + WHERE checkpoint_id = ?", + [u32::from(checkpoint_id)], + |row| { + row.get::<_, Option>(0) + .map(|opt| opt.map(Position::from)) + }, + ) + .optional() + .map_err(Either::Right)?; + + let mut marks_removed = BTreeSet::new(); + let mut stmt = conn + .prepare_cached( + "SELECT mark_removed_position + FROM sapling_tree_checkpoint_marks_removed + WHERE checkpoint_id = ?", + ) + .map_err(Either::Right)?; + let mut mark_removed_rows = stmt + .query([u32::from(checkpoint_id)]) + .map_err(Either::Right)?; + + while let Some(row) = mark_removed_rows.next().map_err(Either::Right)? { + marks_removed.insert( + row.get::<_, u64>(0) + .map(Position::from) + .map_err(Either::Right)?, + ); + } + + Ok(checkpoint_position.map(|pos_opt| { + Checkpoint::from_parts( + pos_opt.map_or(TreeState::Empty, TreeState::AtPosition), + marks_removed, + ) + })) +} + +pub(crate) fn update_checkpoint_with( + conn: &rusqlite::Transaction<'_>, + checkpoint_id: BlockHeight, + update: F, +) -> Result +where + F: Fn(&mut Checkpoint) -> Result<(), Error>, +{ + if let Some(mut c) = get_checkpoint(conn, checkpoint_id)? { + update(&mut c)?; + remove_checkpoint(conn, checkpoint_id)?; + add_checkpoint(conn, checkpoint_id, c)?; + Ok(true) + } else { + Ok(false) + } +} + +pub(crate) fn remove_checkpoint( + conn: &rusqlite::Transaction<'_>, + checkpoint_id: BlockHeight, +) -> Result<(), Error> { + conn.execute( + "DELETE FROM sapling_tree_checkpoints WHERE checkpoint_id = ?", + [u32::from(checkpoint_id)], + ) + .map_err(Either::Right)?; + + Ok(()) +} + +pub(crate) fn truncate_checkpoints( + conn: &rusqlite::Transaction<'_>, + checkpoint_id: BlockHeight, +) -> Result<(), Error> { + conn.execute( + "DELETE FROM sapling_tree_checkpoints WHERE checkpoint_id >= ?", + [u32::from(checkpoint_id)], + ) + .map_err(Either::Right)?; + + conn.execute( + "DELETE FROM sapling_tree_checkpoint_marks_removed WHERE checkpoint_id >= ?", + [u32::from(checkpoint_id)], + ) + .map_err(Either::Right)?; + Ok(()) +} From c42cffeb1d915cfd11f49bf12c3e37cf91680315 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 14 Jun 2023 15:34:28 -0600 Subject: [PATCH 07/27] zcash_client_backend: Replace `WalletWrite::advance_by_block` with `WalletWrite::put_block` Also, add assertions to prevent attempting the creation of zero-conf shielded spends. --- zcash_client_backend/CHANGELOG.md | 11 ++++++++-- zcash_client_backend/src/data_api.rs | 4 ++-- zcash_client_backend/src/data_api/chain.rs | 16 +++++++------- zcash_client_backend/src/data_api/wallet.rs | 22 ++++++++++++++++--- .../src/data_api/wallet/input_selection.rs | 17 +++++++++++++- zcash_client_backend/src/wallet.rs | 1 + zcash_client_sqlite/src/lib.rs | 2 +- 7 files changed, 56 insertions(+), 17 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index c896b6d70..28ab39864 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -7,9 +7,11 @@ and this library adheres to Rust's notion of ## [Unreleased] ### Added -- `impl Eq for zcash_client_backend::address::RecipientAddress` -- `impl Eq for zcash_client_backend::zip321::{Payment, TransactionRequest}` +- `impl Eq for address::RecipientAddress` +- `impl Eq for zip321::{Payment, TransactionRequest}` - `data_api::NullifierQuery` for use with `WalletRead::get_sapling_nullifiers` +- `WalletWrite::put_block` +- `impl Debug` for `{data_api::wallet::input_selection::Proposal, wallet::ReceivedSaplingNote} ### Changed - MSRV is now 1.65.0. @@ -21,9 +23,14 @@ and this library adheres to Rust's notion of - `WalletRead::get_nullifiers` has been renamed to `WalletRead::get_sapling_nullifiers` and its signature has changed; it now subsumes the removed `WalletRead::get_all_nullifiers`. - `wallet::SpendableNote` has been renamed to `wallet::ReceivedSaplingNote`. +- `data_api::chain::scan_cached_blocks` now takes a `from_height` argument that + permits the caller to control the starting position of the scan range. +- `WalletWrite::advance_by_block` has been replaced by `WalletWrite::put_block` + to reflect the semantic change that scanning is no longer a linear operation. ### Removed - `WalletRead::get_all_nullifiers` +- `WalletWrite::advance_by_block` ## [0.9.0] - 2023-04-28 ### Added diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 80ad24f55..eafa0e89c 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -401,7 +401,7 @@ pub trait WalletWrite: WalletRead { /// along with the note commitments that were detected when scanning the block for transactions /// pertaining to this wallet. #[allow(clippy::type_complexity)] - fn advance_by_block( + fn put_block( &mut self, block: PrunedBlock, ) -> Result, Self::Error>; @@ -660,7 +660,7 @@ pub mod testing { } #[allow(clippy::type_complexity)] - fn advance_by_block( + fn put_block( &mut self, _block: PrunedBlock, ) -> Result, Self::Error> { diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index ce0eb2a81..dbec6768d 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -175,7 +175,7 @@ where // comparing against the `validate_from` hash. block_source.with_blocks::<_, Infallible, Infallible>( - validate_from.map(|(h, _)| h), + validate_from.map(|(h, _)| h + 1), limit, move |block| { if let Some((valid_height, valid_hash)) = validate_from { @@ -260,18 +260,20 @@ where ); // Start at either the provided height, or where we synced up to previously. - let (last_scanned_height, commitment_tree_meta) = from_height.map_or_else( + let (from_height, commitment_tree_meta) = from_height.map_or_else( || { data_db.fully_scanned_height().map_or_else( |e| Err(Error::Wallet(e)), - |next| Ok(next.map_or_else(|| (None, None), |(h, m)| (Some(h), Some(m)))), + |last_scanned| { + Ok(last_scanned.map_or_else(|| (None, None), |(h, m)| (Some(h + 1), Some(m)))) + }, ) }, |h| Ok((Some(h), None)), )?; block_source.with_blocks::<_, DbT::Error, DbT::NoteRef>( - last_scanned_height, + from_height, limit, |block: CompactBlock| { add_block_to_runner(params, block, &mut batch_runner); @@ -282,7 +284,7 @@ where batch_runner.flush(); block_source.with_blocks::<_, DbT::Error, DbT::NoteRef>( - last_scanned_height, + from_height, limit, |block: CompactBlock| { let pruned_block = scan_block_with_runner( @@ -308,9 +310,7 @@ where .map(|out| (out.account(), *out.nf())) })); - data_db - .advance_by_block(pruned_block) - .map_err(Error::Wallet)?; + data_db.put_block(pruned_block).map_err(Error::Wallet)?; Ok(()) }, diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index b0930d966..d37a03329 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -119,7 +119,8 @@ where /// can allow the sender to view the resulting notes on the blockchain. /// * `min_confirmations`: The minimum number of confirmations that a previously /// received note must have in the blockchain in order to be considered for being -/// spent. A value of 10 confirmations is recommended. +/// spent. A value of 10 confirmations is recommended and 0-conf transactions are +/// not supported. /// /// # Examples /// @@ -318,6 +319,10 @@ where ParamsT: consensus::Parameters + Clone, InputsT: InputSelector, { + assert!( + min_confirmations > 0, + "zero-conf transactions are not supported" + ); let account = wallet_db .get_account_for_ufvk(&usk.to_unified_full_viewing_key()) .map_err(Error::DataSource)? @@ -372,6 +377,10 @@ where ParamsT: consensus::Parameters + Clone, InputsT: InputSelector, { + assert!( + min_confirmations > 0, + "zero-conf transactions are not supported" + ); input_selector .propose_transaction( params, @@ -409,6 +418,10 @@ where DbT::NoteRef: Copy + Eq + Ord, InputsT: InputSelector, { + assert!( + min_confirmations > 0, + "zero-conf transactions are not supported" + ); input_selector .propose_shielding( params, @@ -453,6 +466,10 @@ where ParamsT: consensus::Parameters + Clone, FeeRuleT: FeeRule, { + assert!( + min_confirmations > 0, + "zero-conf transactions are not supported" + ); let account = wallet_db .get_account_for_ufvk(&usk.to_unified_full_viewing_key()) .map_err(Error::DataSource)? @@ -495,8 +512,7 @@ where selected, usk.sapling(), &dfvk, - min_confirmations - .try_into() + usize::try_from(min_confirmations - 1) .expect("min_confirmations should never be anywhere close to usize::MAX"), )? .ok_or(Error::NoteMismatch(selected.note_id))?; diff --git a/zcash_client_backend/src/data_api/wallet/input_selection.rs b/zcash_client_backend/src/data_api/wallet/input_selection.rs index 403497c0d..5017ae335 100644 --- a/zcash_client_backend/src/data_api/wallet/input_selection.rs +++ b/zcash_client_backend/src/data_api/wallet/input_selection.rs @@ -1,8 +1,8 @@ //! Types related to the process of selecting inputs to be spent given a transaction request. use core::marker::PhantomData; -use std::collections::BTreeSet; use std::fmt; +use std::{collections::BTreeSet, fmt::Debug}; use zcash_primitives::{ consensus::{self, BlockHeight}, @@ -124,6 +124,21 @@ impl Proposal { } } +impl Debug for Proposal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Proposal") + .field("transaction_request", &self.transaction_request) + .field("transparent_inputs", &self.transparent_inputs) + .field("sapling_inputs", &self.sapling_inputs.len()) + .field("balance", &self.balance) + //.field("fee_rule", &self.fee_rule) + .field("min_target_height", &self.min_target_height) + .field("min_anchor_height", &self.min_anchor_height) + .field("is_shielding", &self.is_shielding) + .finish() + } +} + /// A strategy for selecting transaction inputs and proposing transaction outputs. /// /// Proposals should include only economically useful inputs, as determined by `Self::FeeRule`; diff --git a/zcash_client_backend/src/wallet.rs b/zcash_client_backend/src/wallet.rs index c702cfa73..ccfe5a75b 100644 --- a/zcash_client_backend/src/wallet.rs +++ b/zcash_client_backend/src/wallet.rs @@ -175,6 +175,7 @@ impl WalletSaplingOutput { /// Information about a note that is tracked by the wallet that is available for spending, /// with sufficient information for use in note selection. +#[derive(Debug)] pub struct ReceivedSaplingNote { pub note_id: NoteRef, pub diversifier: sapling::Diversifier, diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index cae3685e7..810a17bf4 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -392,7 +392,7 @@ impl WalletWrite for WalletDb #[tracing::instrument(skip_all, fields(height = u32::from(block.block_height)))] #[allow(clippy::type_complexity)] - fn advance_by_block( + fn put_block( &mut self, block: PrunedBlock, ) -> Result, Self::Error> { From 425b5e01d7da676eb3a92f04882cdee68131968b Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Wed, 14 Jun 2023 16:49:16 -0600 Subject: [PATCH 08/27] zcash_client_sqlite: Support shardtree checkpoint functionality --- zcash_client_sqlite/src/chain.rs | 113 +++++++---- zcash_client_sqlite/src/lib.rs | 2 +- zcash_client_sqlite/src/wallet.rs | 48 +++-- .../init/migrations/shardtree_support.rs | 3 +- zcash_client_sqlite/src/wallet/sapling.rs | 30 +-- .../src/wallet/sapling/commitment_tree.rs | 179 ++++++++++++++---- 6 files changed, 270 insertions(+), 105 deletions(-) diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index d115482e2..f5300de90 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -23,12 +23,12 @@ pub mod migrations; /// Implements a traversal of `limit` blocks of the block cache database. /// -/// Starting at the next block above `last_scanned_height`, the `with_row` callback is invoked with -/// each block retrieved from the backing store. If the `limit` value provided is `None`, all -/// blocks are traversed up to the maximum height. +/// Starting at `from_height`, the `with_row` callback is invoked with each block retrieved from +/// the backing store. If the `limit` value provided is `None`, all blocks are traversed up to the +/// maximum height. pub(crate) fn blockdb_with_blocks( block_source: &BlockDb, - last_scanned_height: Option, + from_height: Option, limit: Option, mut with_row: F, ) -> Result<(), Error> @@ -43,15 +43,15 @@ where let mut stmt_blocks = block_source .0 .prepare( - "SELECT height, data FROM compactblocks - WHERE height > ? + "SELECT height, data FROM compactblocks + WHERE height >= ? ORDER BY height ASC LIMIT ?", ) .map_err(to_chain_error)?; let mut rows = stmt_blocks .query(params![ - last_scanned_height.map_or(0u32, u32::from), + from_height.map_or(0u32, u32::from), limit.unwrap_or(u32::max_value()), ]) .map_err(to_chain_error)?; @@ -191,13 +191,13 @@ pub(crate) fn blockmetadb_find_block( /// Implements a traversal of `limit` blocks of the filesystem-backed /// block cache. /// -/// Starting at the next block height above `last_scanned_height`, the `with_row` callback is -/// invoked with each block retrieved from the backing store. If the `limit` value provided is -/// `None`, all blocks are traversed up to the maximum height for which metadata is available. +/// Starting at `from_height`, the `with_row` callback is invoked with each block retrieved from +/// the backing store. If the `limit` value provided is `None`, all blocks are traversed up to the +/// maximum height for which metadata is available. #[cfg(feature = "unstable")] pub(crate) fn fsblockdb_with_blocks( cache: &FsBlockDb, - last_scanned_height: Option, + from_height: Option, limit: Option, mut with_block: F, ) -> Result<(), Error> @@ -214,7 +214,7 @@ where .prepare( "SELECT height, blockhash, time, sapling_outputs_count, orchard_actions_count FROM compactblocks_meta - WHERE height > ? + WHERE height >= ? ORDER BY height ASC LIMIT ?", ) .map_err(to_chain_error)?; @@ -222,7 +222,7 @@ where let rows = stmt_blocks .query_map( params![ - last_scanned_height.map_or(0u32, u32::from), + from_height.map_or(0u32, u32::from), limit.unwrap_or(u32::max_value()), ], |row| { @@ -269,14 +269,22 @@ mod tests { use tempfile::NamedTempFile; use zcash_primitives::{ - block::BlockHash, transaction::components::Amount, zip32::ExtendedSpendingKey, + block::BlockHash, + transaction::{components::Amount, fees::zip317::FeeRule}, + zip32::ExtendedSpendingKey, }; - use zcash_client_backend::data_api::chain::{ - error::{Cause, Error}, - scan_cached_blocks, validate_chain, + use zcash_client_backend::{ + address::RecipientAddress, + data_api::{ + chain::{error::Error, scan_cached_blocks, validate_chain}, + wallet::{input_selection::GreedyInputSelector, spend}, + WalletRead, WalletWrite, + }, + fees::{zip317::SingleOutputChangeStrategy, DustOutputPolicy}, + wallet::OvkPolicy, + zip321::{Payment, TransactionRequest}, }; - use zcash_client_backend::data_api::WalletRead; use crate::{ chain::init::init_cache_database, @@ -573,7 +581,7 @@ mod tests { } #[test] - fn scan_cached_blocks_requires_sequential_blocks() { + fn scan_cached_blocks_allows_blocks_out_of_order() { let cache_file = NamedTempFile::new().unwrap(); let db_cache = BlockDb::for_path(cache_file.path()).unwrap(); init_cache_database(&db_cache).unwrap(); @@ -583,7 +591,9 @@ mod tests { init_wallet_db(&mut db_data, Some(Secret::new(vec![]))).unwrap(); // Add an account to the wallet - let (dfvk, _taddr) = init_test_accounts_table(&mut db_data); + let seed = Secret::new([0u8; 32].to_vec()); + let (_, usk) = db_data.create_account(&seed).unwrap(); + let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); // Create a block with height SAPLING_ACTIVATION_HEIGHT let value = Amount::from_u64(50000).unwrap(); @@ -602,7 +612,7 @@ mod tests { value ); - // We cannot scan a block of height SAPLING_ACTIVATION_HEIGHT + 2 next + // Create blocks to reach SAPLING_ACTIVATION_HEIGHT + 2 let (cb2, _) = fake_compact_block( sapling_activation_height() + 1, cb1.hash(), @@ -619,25 +629,62 @@ mod tests { value, 2, ); + + // Scan the later block first insert_into_cache(&db_cache, &cb3); - match scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None) { - Err(Error::Chain(e)) => { - assert_matches!( - e.cause(), - Cause::BlockHeightDiscontinuity(h) if *h - == sapling_activation_height() + 2 - ); - } - Ok(_) | Err(_) => panic!("Should have failed"), - } + assert_matches!( + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + Some(sapling_activation_height() + 2), + None + ), + Ok(_) + ); - // If we add a block of height SAPLING_ACTIVATION_HEIGHT + 1, we can now scan both + // If we add a block of height SAPLING_ACTIVATION_HEIGHT + 1, we can now scan that insert_into_cache(&db_cache, &cb2); - scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); + scan_cached_blocks( + &tests::network(), + &db_cache, + &mut db_data, + Some(sapling_activation_height() + 1), + Some(1), + ) + .unwrap(); assert_eq!( get_balance(&db_data.conn, AccountId::from(0)).unwrap(), Amount::from_u64(150_000).unwrap() ); + + // We can spend the received notes + let req = TransactionRequest::new(vec![Payment { + recipient_address: RecipientAddress::Shielded(dfvk.default_address().1), + amount: Amount::from_u64(110_000).unwrap(), + memo: None, + label: None, + message: None, + other_params: vec![], + }]) + .unwrap(); + let input_selector = GreedyInputSelector::new( + SingleOutputChangeStrategy::new(FeeRule::standard()), + DustOutputPolicy::default(), + ); + assert_matches!( + spend( + &mut db_data, + &tests::network(), + crate::wallet::sapling::tests::test_prover(), + &input_selector, + &usk, + req, + OvkPolicy::Sender, + 1, + ), + Ok(_) + ); } #[test] diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 810a17bf4..0d46a1ce0 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -399,7 +399,7 @@ impl WalletWrite for WalletDb self.transactionally(|wdb| { // Insert the block into the database. let block_height = block.block_height; - wallet::insert_block( + wallet::put_block( wdb.conn.0, block_height, block.block_hash, diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 6b2d15072..7ecbc080c 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -64,7 +64,7 @@ //! wallet. //! - `memo` the shielded memo associated with the output, if any. -use rusqlite::{self, named_params, params, OptionalExtension, ToSql}; +use rusqlite::{self, named_params, OptionalExtension, ToSql}; use std::collections::HashMap; use std::convert::TryFrom; use std::io::Cursor; @@ -735,15 +735,18 @@ pub(crate) fn get_unspent_transparent_outputs( FROM utxos u LEFT OUTER JOIN transactions tx ON tx.id_tx = u.spent_in_tx - WHERE u.address = ? - AND u.height <= ? + WHERE u.address = :address + AND u.height <= :max_height AND tx.block IS NULL", )?; let addr_str = address.encode(params); let mut utxos = Vec::::new(); - let mut rows = stmt_blocks.query(params![addr_str, u32::from(max_height)])?; + let mut rows = stmt_blocks.query(named_params![ + ":address": addr_str, + ":max_height": u32::from(max_height) + ])?; let excluded: BTreeSet = exclude.iter().cloned().collect(); while let Some(row) = rows.next()? { let txid: Vec = row.get(0)?; @@ -796,14 +799,17 @@ pub(crate) fn get_transparent_balances( FROM utxos u LEFT OUTER JOIN transactions tx ON tx.id_tx = u.spent_in_tx - WHERE u.received_by_account = ? - AND u.height <= ? + WHERE u.received_by_account = :account_id + AND u.height <= :max_height AND tx.block IS NULL GROUP BY u.address", )?; let mut res = HashMap::new(); - let mut rows = stmt_blocks.query(params![u32::from(account), u32::from(max_height)])?; + let mut rows = stmt_blocks.query(named_params![ + ":account_id": u32::from(account), + ":max_height": u32::from(max_height) + ])?; while let Some(row) = rows.next()? { let taddr_str: String = row.get(0)?; let taddr = TransparentAddress::decode(params, &taddr_str)?; @@ -816,14 +822,14 @@ pub(crate) fn get_transparent_balances( } /// Inserts information about a scanned block into the database. -pub(crate) fn insert_block( +pub(crate) fn put_block( conn: &rusqlite::Connection, block_height: BlockHeight, block_hash: BlockHash, block_time: u32, sapling_commitment_tree_size: Option, ) -> Result<(), SqliteClientError> { - let mut stmt_insert_block = conn.prepare_cached( + let mut stmt_upsert_block = conn.prepare_cached( "INSERT INTO blocks ( height, hash, @@ -831,14 +837,24 @@ pub(crate) fn insert_block( sapling_commitment_tree_size, sapling_tree ) - VALUES (?, ?, ?, ?, x'00')", + VALUES ( + :height, + :hash, + :block_time, + :sapling_commitment_tree_size, + x'00' + ) + ON CONFLICT (height) DO UPDATE + SET hash = :hash, + time = :block_time, + sapling_commitment_tree_size = :sapling_commitment_tree_size", )?; - stmt_insert_block.execute(params![ - u32::from(block_height), - &block_hash.0[..], - block_time, - sapling_commitment_tree_size + stmt_upsert_block.execute(named_params![ + ":height": u32::from(block_height), + ":hash": &block_hash.0[..], + ":block_time": block_time, + ":sapling_commitment_tree_size": sapling_commitment_tree_size ])?; Ok(()) @@ -981,7 +997,7 @@ pub(crate) fn put_legacy_transparent_utxo( #[cfg(feature = "transparent-inputs")] let mut stmt_upsert_legacy_transparent_utxo = conn.prepare_cached( "INSERT INTO utxos ( - prevout_txid, prevout_idx, + prevout_txid, prevout_idx, received_by_account, address, script, value_zat, height) VALUES diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index f22b03c20..ded5b2d1f 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -11,6 +11,7 @@ use schemer_rusqlite::RusqliteMigration; use shardtree::ShardTree; use uuid::Uuid; +use zcash_client_backend::data_api::SAPLING_SHARD_HEIGHT; use zcash_primitives::{ consensus::BlockHeight, merkle_tree::{read_commitment_tree, read_incremental_witness}, @@ -93,7 +94,7 @@ impl RusqliteMigration for Migration { let mut shard_tree: ShardTree< _, { sapling::NOTE_COMMITMENT_TREE_DEPTH }, - { sapling::NOTE_COMMITMENT_TREE_DEPTH / 2 }, + SAPLING_SHARD_HEIGHT, > = ShardTree::new(shard_store, 100); // Insert all the tree information that we can get from block-end commitment trees { diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 511333ff7..c763534d0 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -368,7 +368,7 @@ pub(crate) fn put_received_note( #[cfg(test)] #[allow(deprecated)] -mod tests { +pub(crate) mod tests { use rusqlite::Connection; use secrecy::Secret; use tempfile::NamedTempFile; @@ -427,7 +427,7 @@ mod tests { }, }; - fn test_prover() -> impl TxProver { + pub fn test_prover() -> impl TxProver { match LocalTxProver::with_default_location() { Some(tx_prover) => tx_prover, None => { @@ -463,7 +463,7 @@ mod tests { Amount::from_u64(1).unwrap(), None, OvkPolicy::Sender, - 10, + 1, ), Err(data_api::error::Error::KeyNotRecognized) ); @@ -492,7 +492,7 @@ mod tests { Amount::from_u64(1).unwrap(), None, OvkPolicy::Sender, - 10, + 1, ), Err(data_api::error::Error::ScanRequired) ); @@ -535,7 +535,7 @@ mod tests { Amount::from_u64(1).unwrap(), None, OvkPolicy::Sender, - 10, + 1, ), Err(data_api::error::Error::InsufficientFunds { available, @@ -740,7 +740,7 @@ mod tests { Amount::from_u64(15000).unwrap(), None, OvkPolicy::Sender, - 10, + 1, ), Ok(_) ); @@ -756,7 +756,7 @@ mod tests { Amount::from_u64(2000).unwrap(), None, OvkPolicy::Sender, - 10, + 1, ), Err(data_api::error::Error::InsufficientFunds { available, @@ -791,7 +791,7 @@ mod tests { Amount::from_u64(2000).unwrap(), None, OvkPolicy::Sender, - 10, + 1, ), Err(data_api::error::Error::InsufficientFunds { available, @@ -822,7 +822,7 @@ mod tests { Amount::from_u64(2000).unwrap(), None, OvkPolicy::Sender, - 10, + 1, ) .unwrap(); } @@ -874,7 +874,7 @@ mod tests { Amount::from_u64(15000).unwrap(), None, ovk_policy, - 10, + 1, ) .unwrap(); @@ -962,7 +962,7 @@ mod tests { scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Verified balance matches total balance - let (_, anchor_height) = db_data.get_target_and_anchor_heights(10).unwrap().unwrap(); + let (_, anchor_height) = db_data.get_target_and_anchor_heights(1).unwrap().unwrap(); assert_eq!( get_balance(&db_data.conn, AccountId::from(0)).unwrap(), value @@ -983,7 +983,7 @@ mod tests { Amount::from_u64(50000).unwrap(), None, OvkPolicy::Sender, - 10, + 1, ), Ok(_) ); @@ -1039,7 +1039,7 @@ mod tests { Amount::from_u64(50000).unwrap(), None, OvkPolicy::Sender, - 10, + 1, ), Ok(_) ); @@ -1193,7 +1193,7 @@ mod tests { DustOutputPolicy::default(), ); - // Add funds to the wallet + // Ensure that the wallet has at least one block let (cb, _) = fake_compact_block( sapling_activation_height(), BlockHash([0; 32]), @@ -1215,7 +1215,7 @@ mod tests { &usk, &[*taddr], &MemoBytes::empty(), - 0 + 1 ), Ok(_) ); diff --git a/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs b/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs index 912831fcf..5ea9e29fe 100644 --- a/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs @@ -1,19 +1,22 @@ use either::Either; - -use incrementalmerkletree::{Address, Position}; use rusqlite::{self, named_params, Connection, OptionalExtension}; -use shardtree::{Checkpoint, LocatedPrunableTree, PrunableTree, ShardStore, TreeState}; - use std::{ collections::BTreeSet, io::{self, Cursor}, ops::Deref, }; +use incrementalmerkletree::{Address, Level, Position}; +use shardtree::{Checkpoint, LocatedPrunableTree, PrunableTree, ShardStore, TreeState}; + use zcash_primitives::{consensus::BlockHeight, merkle_tree::HashSer, sapling}; +use zcash_client_backend::data_api::SAPLING_SHARD_HEIGHT; + use crate::serialization::{read_shard, write_shard_v1}; +const SHARD_ROOT_LEVEL: Level = Level::new(SAPLING_SHARD_HEIGHT); + pub struct WalletDbSaplingShardStore<'conn, 'a> { pub(crate) conn: &'a rusqlite::Transaction<'conn>, } @@ -39,8 +42,7 @@ impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { } fn last_shard(&self) -> Result>, Self::Error> { - // SELECT shard_data FROM sapling_tree ORDER BY shard_index DESC LIMIT 1 - todo!() + last_shard(self.conn) } fn put_shard(&mut self, subtree: LocatedPrunableTree) -> Result<(), Self::Error> { @@ -48,8 +50,7 @@ impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { } fn get_shard_roots(&self) -> Result, Self::Error> { - // SELECT - todo!() + get_shard_roots(self.conn) } fn truncate(&mut self, from: Address) -> Result<(), Self::Error> { @@ -86,9 +87,9 @@ impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { fn get_checkpoint_at_depth( &self, - _checkpoint_depth: usize, + checkpoint_depth: usize, ) -> Result, Self::Error> { - todo!() + get_checkpoint_at_depth(self.conn, checkpoint_depth) } fn get_checkpoint( @@ -150,6 +151,31 @@ pub(crate) fn get_shard( .transpose() } +pub(crate) fn last_shard( + conn: &rusqlite::Connection, +) -> Result>, Error> { + conn.query_row( + "SELECT shard_index, shard_data + FROM sapling_tree_shards + ORDER BY shard_index DESC + LIMIT 1", + [], + |row| { + let shard_index: u64 = row.get(0)?; + let shard_data: Vec = row.get(1)?; + Ok((shard_index, shard_data)) + }, + ) + .optional() + .map_err(Either::Right)? + .map(|(shard_index, shard_data)| { + let shard_root = Address::from_parts(SHARD_ROOT_LEVEL, shard_index); + let shard_tree = read_shard(&mut Cursor::new(shard_data)).map_err(Either::Left)?; + Ok(LocatedPrunableTree::from_parts(shard_root, shard_tree)) + }) + .transpose() +} + pub(crate) fn put_shard( conn: &rusqlite::Connection, subtree: LocatedPrunableTree, @@ -172,10 +198,10 @@ pub(crate) fn put_shard( conn.prepare_cached( "INSERT INTO sapling_tree_shards (shard_index, root_hash, shard_data) - VALUES (:shard_index, :root_hash, :shard_data) - ON CONFLICT (shard_index) DO UPDATE - SET root_hash = :root_hash, - shard_data = :shard_data", + VALUES (:shard_index, :root_hash, :shard_data) + ON CONFLICT (shard_index) DO UPDATE + SET root_hash = :root_hash, + shard_data = :shard_data", ) .and_then(|mut stmt_put_shard| { stmt_put_shard.execute(named_params![ @@ -189,6 +215,22 @@ pub(crate) fn put_shard( Ok(()) } +pub(crate) fn get_shard_roots(conn: &rusqlite::Connection) -> Result, Error> { + let mut stmt = conn + .prepare("SELECT shard_index FROM sapling_tree_shards ORDER BY shard_index") + .map_err(Either::Right)?; + let mut rows = stmt.query([]).map_err(Either::Right)?; + + let mut res = vec![]; + while let Some(row) = rows.next().map_err(Either::Right)? { + res.push(Address::from_parts( + SHARD_ROOT_LEVEL, + row.get(0).map_err(Either::Right)?, + )); + } + Ok(res) +} + pub(crate) fn truncate(conn: &rusqlite::Transaction<'_>, from: Address) -> Result<(), Error> { conn.execute( "DELETE FROM sapling_tree_shards WHERE shard_index >= ?", @@ -264,8 +306,8 @@ pub(crate) fn get_checkpoint>( let checkpoint_position = conn .query_row( "SELECT position - FROM sapling_tree_checkpoints - WHERE checkpoint_id = ?", + FROM sapling_tree_checkpoints + WHERE checkpoint_id = ?", [u32::from(checkpoint_id)], |row| { row.get::<_, Option>(0) @@ -275,32 +317,91 @@ pub(crate) fn get_checkpoint>( .optional() .map_err(Either::Right)?; - let mut marks_removed = BTreeSet::new(); - let mut stmt = conn - .prepare_cached( - "SELECT mark_removed_position - FROM sapling_tree_checkpoint_marks_removed - WHERE checkpoint_id = ?", + checkpoint_position + .map(|pos_opt| { + let mut marks_removed = BTreeSet::new(); + let mut stmt = conn + .prepare_cached( + "SELECT mark_removed_position + FROM sapling_tree_checkpoint_marks_removed + WHERE checkpoint_id = ?", + ) + .map_err(Either::Right)?; + let mut mark_removed_rows = stmt + .query([u32::from(checkpoint_id)]) + .map_err(Either::Right)?; + + while let Some(row) = mark_removed_rows.next().map_err(Either::Right)? { + marks_removed.insert( + row.get::<_, u64>(0) + .map(Position::from) + .map_err(Either::Right)?, + ); + } + + Ok(Checkpoint::from_parts( + pos_opt.map_or(TreeState::Empty, TreeState::AtPosition), + marks_removed, + )) + }) + .transpose() +} + +pub(crate) fn get_checkpoint_at_depth>( + conn: &C, + checkpoint_depth: usize, +) -> Result, Either> { + let checkpoint_parts = conn + .query_row( + "SELECT checkpoint_id, position + FROM sapling_tree_checkpoints + ORDER BY checkpoint_id DESC + LIMIT 1 + OFFSET :offset", + named_params![":offset": checkpoint_depth], + |row| { + let checkpoint_id: u32 = row.get(0)?; + let position: Option = row.get(1)?; + Ok(( + BlockHeight::from(checkpoint_id), + position.map(Position::from), + )) + }, ) - .map_err(Either::Right)?; - let mut mark_removed_rows = stmt - .query([u32::from(checkpoint_id)]) + .optional() .map_err(Either::Right)?; - while let Some(row) = mark_removed_rows.next().map_err(Either::Right)? { - marks_removed.insert( - row.get::<_, u64>(0) - .map(Position::from) - .map_err(Either::Right)?, - ); - } - - Ok(checkpoint_position.map(|pos_opt| { - Checkpoint::from_parts( - pos_opt.map_or(TreeState::Empty, TreeState::AtPosition), - marks_removed, - ) - })) + checkpoint_parts + .map(|(checkpoint_id, pos_opt)| { + let mut marks_removed = BTreeSet::new(); + let mut stmt = conn + .prepare_cached( + "SELECT mark_removed_position + FROM sapling_tree_checkpoint_marks_removed + WHERE checkpoint_id = ?", + ) + .map_err(Either::Right)?; + let mut mark_removed_rows = stmt + .query([u32::from(checkpoint_id)]) + .map_err(Either::Right)?; + + while let Some(row) = mark_removed_rows.next().map_err(Either::Right)? { + marks_removed.insert( + row.get::<_, u64>(0) + .map(Position::from) + .map_err(Either::Right)?, + ); + } + + Ok(( + checkpoint_id, + Checkpoint::from_parts( + pos_opt.map_or(TreeState::Empty, TreeState::AtPosition), + marks_removed, + ), + )) + }) + .transpose() } pub(crate) fn update_checkpoint_with( From 0a4236f725a3f46fce7247fa0860ba44871eb8d8 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Thu, 15 Jun 2023 13:50:07 -0600 Subject: [PATCH 09/27] zcash_client_sqlite: Add tests for sqlite-backed ShardTree & fix revealed issues. --- zcash_client_sqlite/src/lib.rs | 14 +- zcash_client_sqlite/src/wallet/init.rs | 1 + .../init/migrations/shardtree_support.rs | 8 +- .../src/wallet/sapling/commitment_tree.rs | 425 +++++++++++++++--- zcash_primitives/src/consensus.rs | 6 + zcash_primitives/src/merkle_tree.rs | 18 + 6 files changed, 395 insertions(+), 77 deletions(-) diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 0d46a1ce0..9df48d5ca 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -66,9 +66,7 @@ use zcash_client_backend::{ DecryptedOutput, TransferType, }; -use crate::{ - error::SqliteClientError, wallet::sapling::commitment_tree::WalletDbSaplingShardStore, -}; +use crate::{error::SqliteClientError, wallet::sapling::commitment_tree::SqliteShardStore}; #[cfg(feature = "unstable")] use { @@ -617,7 +615,8 @@ impl WalletWrite for WalletDb impl WalletCommitmentTrees for WalletDb { type Error = Either; - type SaplingShardStore<'a> = WalletDbSaplingShardStore<'a, 'a>; + type SaplingShardStore<'a> = + SqliteShardStore<&'a rusqlite::Transaction<'a>, sapling::Node, SAPLING_SHARD_HEIGHT>; fn with_sapling_tree_mut(&mut self, mut callback: F) -> Result where @@ -634,7 +633,7 @@ impl WalletCommitmentTrees for WalletDb WalletCommitmentTrees for WalletDb WalletCommitmentTrees for WalletDb, P> { type Error = Either; - type SaplingShardStore<'a> = WalletDbSaplingShardStore<'a, 'a>; + type SaplingShardStore<'a> = + SqliteShardStore<&'a rusqlite::Transaction<'a>, sapling::Node, SAPLING_SHARD_HEIGHT>; fn with_sapling_tree_mut(&mut self, mut callback: F) -> Result where @@ -662,7 +662,7 @@ impl<'conn, P: consensus::Parameters> WalletCommitmentTrees for WalletDb>>, { let mut shardtree = ShardTree::new( - WalletDbSaplingShardStore::from_connection(self.conn.0) + SqliteShardStore::from_connection(self.conn.0) .map_err(|e| ShardTreeError::Storage(Either::Right(e)))?, 100, ); diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index a98806ee6..c730aea73 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -403,6 +403,7 @@ mod tests { checkpoint_id INTEGER NOT NULL, mark_removed_position INTEGER NOT NULL, FOREIGN KEY (checkpoint_id) REFERENCES sapling_tree_checkpoints(checkpoint_id) + ON DELETE CASCADE )", "CREATE TABLE sapling_tree_checkpoints ( checkpoint_id INTEGER PRIMARY KEY, diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index ded5b2d1f..6a597d56c 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -20,7 +20,7 @@ use zcash_primitives::{ use crate::wallet::{ init::{migrations::received_notes_nullable_nf, WalletMigrationError}, - sapling::commitment_tree::WalletDbSaplingShardStore, + sapling::commitment_tree::SqliteShardStore, }; pub(super) const MIGRATION_ID: Uuid = Uuid::from_fields( @@ -87,10 +87,14 @@ impl RusqliteMigration for Migration { checkpoint_id INTEGER NOT NULL, mark_removed_position INTEGER NOT NULL, FOREIGN KEY (checkpoint_id) REFERENCES sapling_tree_checkpoints(checkpoint_id) + ON DELETE CASCADE );", )?; - let shard_store = WalletDbSaplingShardStore::from_connection(transaction)?; + let shard_store = + SqliteShardStore::<_, sapling::Node, SAPLING_SHARD_HEIGHT>::from_connection( + transaction, + )?; let mut shard_tree: ShardTree< _, { sapling::NOTE_COMMITMENT_TREE_DEPTH }, diff --git a/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs b/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs index 5ea9e29fe..f685350c1 100644 --- a/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs @@ -1,36 +1,38 @@ use either::Either; -use rusqlite::{self, named_params, Connection, OptionalExtension}; +use rusqlite::{self, named_params, OptionalExtension}; use std::{ collections::BTreeSet, io::{self, Cursor}, - ops::Deref, + marker::PhantomData, }; use incrementalmerkletree::{Address, Level, Position}; use shardtree::{Checkpoint, LocatedPrunableTree, PrunableTree, ShardStore, TreeState}; -use zcash_primitives::{consensus::BlockHeight, merkle_tree::HashSer, sapling}; - -use zcash_client_backend::data_api::SAPLING_SHARD_HEIGHT; +use zcash_primitives::{consensus::BlockHeight, merkle_tree::HashSer}; use crate::serialization::{read_shard, write_shard_v1}; -const SHARD_ROOT_LEVEL: Level = Level::new(SAPLING_SHARD_HEIGHT); - -pub struct WalletDbSaplingShardStore<'conn, 'a> { - pub(crate) conn: &'a rusqlite::Transaction<'conn>, +pub struct SqliteShardStore { + pub(crate) conn: C, + _hash_type: PhantomData, } -impl<'conn, 'a> WalletDbSaplingShardStore<'conn, 'a> { - pub(crate) fn from_connection( - conn: &'a rusqlite::Transaction<'conn>, - ) -> Result { - Ok(WalletDbSaplingShardStore { conn }) +impl SqliteShardStore { + const SHARD_ROOT_LEVEL: Level = Level::new(SHARD_HEIGHT); + + pub(crate) fn from_connection(conn: C) -> Result { + Ok(SqliteShardStore { + conn, + _hash_type: PhantomData, + }) } } -impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { - type H = sapling::Node; +impl<'conn, 'a: 'conn, H: HashSer, const SHARD_HEIGHT: u8> ShardStore + for SqliteShardStore<&'a rusqlite::Transaction<'conn>, H, SHARD_HEIGHT> +{ + type H = H; type CheckpointId = BlockHeight; type Error = Either; @@ -42,7 +44,7 @@ impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { } fn last_shard(&self) -> Result>, Self::Error> { - last_shard(self.conn) + last_shard(self.conn, Self::SHARD_ROOT_LEVEL) } fn put_shard(&mut self, subtree: LocatedPrunableTree) -> Result<(), Self::Error> { @@ -50,7 +52,7 @@ impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { } fn get_shard_roots(&self) -> Result, Self::Error> { - get_shard_roots(self.conn) + get_shard_roots(self.conn, Self::SHARD_ROOT_LEVEL) } fn truncate(&mut self, from: Address) -> Result<(), Self::Error> { @@ -66,11 +68,11 @@ impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { } fn min_checkpoint_id(&self) -> Result, Self::Error> { - todo!() + min_checkpoint_id(self.conn) } fn max_checkpoint_id(&self) -> Result, Self::Error> { - todo!() + max_checkpoint_id(self.conn) } fn add_checkpoint( @@ -99,11 +101,11 @@ impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { get_checkpoint(self.conn, *checkpoint_id) } - fn with_checkpoints(&mut self, _limit: usize, _callback: F) -> Result<(), Self::Error> + fn with_checkpoints(&mut self, limit: usize, callback: F) -> Result<(), Self::Error> where F: FnMut(&Self::CheckpointId, &Checkpoint) -> Result<(), Self::Error>, { - todo!() + with_checkpoints(self.conn, limit, callback) } fn update_checkpoint_with( @@ -129,12 +131,128 @@ impl<'conn, 'a: 'conn> ShardStore for WalletDbSaplingShardStore<'conn, 'a> { } } +impl ShardStore + for SqliteShardStore +{ + type H = H; + type CheckpointId = BlockHeight; + type Error = Either; + + fn get_shard( + &self, + shard_root: Address, + ) -> Result>, Self::Error> { + get_shard(&self.conn, shard_root) + } + + fn last_shard(&self) -> Result>, Self::Error> { + last_shard(&self.conn, Self::SHARD_ROOT_LEVEL) + } + + fn put_shard(&mut self, subtree: LocatedPrunableTree) -> Result<(), Self::Error> { + let tx = self.conn.transaction().map_err(Either::Right)?; + put_shard(&tx, subtree)?; + tx.commit().map_err(Either::Right)?; + Ok(()) + } + + fn get_shard_roots(&self) -> Result, Self::Error> { + get_shard_roots(&self.conn, Self::SHARD_ROOT_LEVEL) + } + + fn truncate(&mut self, from: Address) -> Result<(), Self::Error> { + truncate(&self.conn, from) + } + + fn get_cap(&self) -> Result, Self::Error> { + get_cap(&self.conn) + } + + fn put_cap(&mut self, cap: PrunableTree) -> Result<(), Self::Error> { + put_cap(&self.conn, cap) + } + + fn min_checkpoint_id(&self) -> Result, Self::Error> { + min_checkpoint_id(&self.conn) + } + + fn max_checkpoint_id(&self) -> Result, Self::Error> { + max_checkpoint_id(&self.conn) + } + + fn add_checkpoint( + &mut self, + checkpoint_id: Self::CheckpointId, + checkpoint: Checkpoint, + ) -> Result<(), Self::Error> { + let tx = self.conn.transaction().map_err(Either::Right)?; + add_checkpoint(&tx, checkpoint_id, checkpoint)?; + tx.commit().map_err(Either::Right) + } + + fn checkpoint_count(&self) -> Result { + checkpoint_count(&self.conn) + } + + fn get_checkpoint_at_depth( + &self, + checkpoint_depth: usize, + ) -> Result, Self::Error> { + get_checkpoint_at_depth(&self.conn, checkpoint_depth) + } + + fn get_checkpoint( + &self, + checkpoint_id: &Self::CheckpointId, + ) -> Result, Self::Error> { + get_checkpoint(&self.conn, *checkpoint_id) + } + + fn with_checkpoints(&mut self, limit: usize, callback: F) -> Result<(), Self::Error> + where + F: FnMut(&Self::CheckpointId, &Checkpoint) -> Result<(), Self::Error>, + { + let tx = self.conn.transaction().map_err(Either::Right)?; + with_checkpoints(&tx, limit, callback)?; + tx.commit().map_err(Either::Right) + } + + fn update_checkpoint_with( + &mut self, + checkpoint_id: &Self::CheckpointId, + update: F, + ) -> Result + where + F: Fn(&mut Checkpoint) -> Result<(), Self::Error>, + { + let tx = self.conn.transaction().map_err(Either::Right)?; + let result = update_checkpoint_with(&tx, *checkpoint_id, update)?; + tx.commit().map_err(Either::Right)?; + Ok(result) + } + + fn remove_checkpoint(&mut self, checkpoint_id: &Self::CheckpointId) -> Result<(), Self::Error> { + let tx = self.conn.transaction().map_err(Either::Right)?; + remove_checkpoint(&tx, *checkpoint_id)?; + tx.commit().map_err(Either::Right) + } + + fn truncate_checkpoints( + &mut self, + checkpoint_id: &Self::CheckpointId, + ) -> Result<(), Self::Error> { + let tx = self.conn.transaction().map_err(Either::Right)?; + truncate_checkpoints(&tx, *checkpoint_id)?; + tx.commit().map_err(Either::Right) + } +} + type Error = Either; -pub(crate) fn get_shard( +pub(crate) fn get_shard( conn: &rusqlite::Connection, shard_root: Address, -) -> Result>, Error> { +) -> Result>, Error> { conn.query_row( "SELECT shard_data FROM sapling_tree_shards @@ -151,9 +269,10 @@ pub(crate) fn get_shard( .transpose() } -pub(crate) fn last_shard( +pub(crate) fn last_shard( conn: &rusqlite::Connection, -) -> Result>, Error> { + shard_root_level: Level, +) -> Result>, Error> { conn.query_row( "SELECT shard_index, shard_data FROM sapling_tree_shards @@ -169,16 +288,16 @@ pub(crate) fn last_shard( .optional() .map_err(Either::Right)? .map(|(shard_index, shard_data)| { - let shard_root = Address::from_parts(SHARD_ROOT_LEVEL, shard_index); + let shard_root = Address::from_parts(shard_root_level, shard_index); let shard_tree = read_shard(&mut Cursor::new(shard_data)).map_err(Either::Left)?; Ok(LocatedPrunableTree::from_parts(shard_root, shard_tree)) }) .transpose() } -pub(crate) fn put_shard( - conn: &rusqlite::Connection, - subtree: LocatedPrunableTree, +pub(crate) fn put_shard( + conn: &rusqlite::Transaction<'_>, + subtree: LocatedPrunableTree, ) -> Result<(), Error> { let subtree_root_hash = subtree .root() @@ -196,26 +315,31 @@ pub(crate) fn put_shard( let mut subtree_data = vec![]; write_shard_v1(&mut subtree_data, subtree.root()).map_err(Either::Left)?; - conn.prepare_cached( - "INSERT INTO sapling_tree_shards (shard_index, root_hash, shard_data) - VALUES (:shard_index, :root_hash, :shard_data) - ON CONFLICT (shard_index) DO UPDATE - SET root_hash = :root_hash, - shard_data = :shard_data", - ) - .and_then(|mut stmt_put_shard| { - stmt_put_shard.execute(named_params![ + let mut stmt_put_shard = conn + .prepare_cached( + "INSERT INTO sapling_tree_shards (shard_index, root_hash, shard_data) + VALUES (:shard_index, :root_hash, :shard_data) + ON CONFLICT (shard_index) DO UPDATE + SET root_hash = :root_hash, + shard_data = :shard_data", + ) + .map_err(Either::Right)?; + + stmt_put_shard + .execute(named_params![ ":shard_index": subtree.root_addr().index(), ":root_hash": subtree_root_hash, ":shard_data": subtree_data ]) - }) - .map_err(Either::Right)?; + .map_err(Either::Right)?; Ok(()) } -pub(crate) fn get_shard_roots(conn: &rusqlite::Connection) -> Result, Error> { +pub(crate) fn get_shard_roots( + conn: &rusqlite::Connection, + shard_root_level: Level, +) -> Result, Error> { let mut stmt = conn .prepare("SELECT shard_index FROM sapling_tree_shards ORDER BY shard_index") .map_err(Either::Right)?; @@ -224,14 +348,14 @@ pub(crate) fn get_shard_roots(conn: &rusqlite::Connection) -> Result, from: Address) -> Result<(), Error> { +pub(crate) fn truncate(conn: &rusqlite::Connection, from: Address) -> Result<(), Error> { conn.execute( "DELETE FROM sapling_tree_shards WHERE shard_index >= ?", [from.index()], @@ -240,7 +364,7 @@ pub(crate) fn truncate(conn: &rusqlite::Transaction<'_>, from: Address) -> Resul .map(|_| ()) } -pub(crate) fn get_cap(conn: &rusqlite::Connection) -> Result, Error> { +pub(crate) fn get_cap(conn: &rusqlite::Connection) -> Result, Error> { conn.query_row("SELECT cap_data FROM sapling_tree_cap", [], |row| { row.get::<_, Vec>(0) }) @@ -252,9 +376,9 @@ pub(crate) fn get_cap(conn: &rusqlite::Connection) -> Result, - cap: PrunableTree, +pub(crate) fn put_cap( + conn: &rusqlite::Connection, + cap: PrunableTree, ) -> Result<(), Error> { let mut stmt = conn .prepare_cached( @@ -272,22 +396,62 @@ pub(crate) fn put_cap( Ok(()) } +pub(crate) fn min_checkpoint_id(conn: &rusqlite::Connection) -> Result, Error> { + conn.query_row( + "SELECT MIN(checkpoint_id) FROM sapling_tree_checkpoints", + [], + |row| { + row.get::<_, Option>(0) + .map(|opt| opt.map(BlockHeight::from)) + }, + ) + .map_err(Either::Right) +} + +pub(crate) fn max_checkpoint_id(conn: &rusqlite::Connection) -> Result, Error> { + conn.query_row( + "SELECT MAX(checkpoint_id) FROM sapling_tree_checkpoints", + [], + |row| { + row.get::<_, Option>(0) + .map(|opt| opt.map(BlockHeight::from)) + }, + ) + .map_err(Either::Right) +} + pub(crate) fn add_checkpoint( conn: &rusqlite::Transaction<'_>, checkpoint_id: BlockHeight, checkpoint: Checkpoint, ) -> Result<(), Error> { - conn.prepare_cached( - "INSERT INTO sapling_tree_checkpoints (checkpoint_id, position) - VALUES (:checkpoint_id, :position)", - ) - .and_then(|mut stmt_insert_checkpoint| { - stmt_insert_checkpoint.execute(named_params![ + let mut stmt_insert_checkpoint = conn + .prepare_cached( + "INSERT INTO sapling_tree_checkpoints (checkpoint_id, position) + VALUES (:checkpoint_id, :position)", + ) + .map_err(Either::Right)?; + + stmt_insert_checkpoint + .execute(named_params![ ":checkpoint_id": u32::from(checkpoint_id), ":position": checkpoint.position().map(u64::from) ]) - }) - .map_err(Either::Right)?; + .map_err(Either::Right)?; + + let mut stmt_insert_mark_removed = conn.prepare_cached( + "INSERT INTO sapling_tree_checkpoint_marks_removed (checkpoint_id, mark_removed_position) + VALUES (:checkpoint_id, :position)", + ).map_err(Either::Right)?; + + for pos in checkpoint.marks_removed() { + stmt_insert_mark_removed + .execute(named_params![ + ":checkpoint_id": u32::from(checkpoint_id), + ":position": u64::from(*pos) + ]) + .map_err(Either::Right)?; + } Ok(()) } @@ -299,10 +463,10 @@ pub(crate) fn checkpoint_count(conn: &rusqlite::Connection) -> Result>( - conn: &C, +pub(crate) fn get_checkpoint( + conn: &rusqlite::Connection, checkpoint_id: BlockHeight, -) -> Result, Either> { +) -> Result, Error> { let checkpoint_position = conn .query_row( "SELECT position @@ -347,10 +511,14 @@ pub(crate) fn get_checkpoint>( .transpose() } -pub(crate) fn get_checkpoint_at_depth>( - conn: &C, +pub(crate) fn get_checkpoint_at_depth( + conn: &rusqlite::Connection, checkpoint_depth: usize, -) -> Result, Either> { +) -> Result, Error> { + if checkpoint_depth == 0 { + return Ok(None); + } + let checkpoint_parts = conn .query_row( "SELECT checkpoint_id, position @@ -358,7 +526,7 @@ pub(crate) fn get_checkpoint_at_depth>( ORDER BY checkpoint_id DESC LIMIT 1 OFFSET :offset", - named_params![":offset": checkpoint_depth], + named_params![":offset": checkpoint_depth - 1], |row| { let checkpoint_id: u32 = row.get(0)?; let position: Option = row.get(1)?; @@ -404,6 +572,62 @@ pub(crate) fn get_checkpoint_at_depth>( .transpose() } +pub(crate) fn with_checkpoints( + conn: &rusqlite::Transaction<'_>, + limit: usize, + mut callback: F, +) -> Result<(), Error> +where + F: FnMut(&BlockHeight, &Checkpoint) -> Result<(), Error>, +{ + let mut stmt_get_checkpoints = conn + .prepare_cached( + "SELECT checkpoint_id, position + FROM sapling_tree_checkpoints + LIMIT :limit", + ) + .map_err(Either::Right)?; + + let mut stmt_get_checkpoint_marks_removed = conn + .prepare_cached( + "SELECT mark_removed_position + FROM sapling_tree_checkpoint_marks_removed + WHERE checkpoint_id = :checkpoint_id", + ) + .map_err(Either::Right)?; + + let mut rows = stmt_get_checkpoints + .query(named_params![":limit": limit]) + .map_err(Either::Right)?; + + while let Some(row) = rows.next().map_err(Either::Right)? { + let checkpoint_id = row.get::<_, u32>(0).map_err(Either::Right)?; + let tree_state = row + .get::<_, Option>(1) + .map(|opt| opt.map_or_else(|| TreeState::Empty, |p| TreeState::AtPosition(p.into()))) + .map_err(Either::Right)?; + + let mut mark_removed_rows = stmt_get_checkpoint_marks_removed + .query(named_params![":checkpoint_id": checkpoint_id]) + .map_err(Either::Right)?; + let mut marks_removed = BTreeSet::new(); + while let Some(mr_row) = mark_removed_rows.next().map_err(Either::Right)? { + let mark_removed_position = mr_row + .get::<_, u64>(0) + .map(Position::from) + .map_err(Either::Right)?; + marks_removed.insert(mark_removed_position); + } + + callback( + &BlockHeight::from(checkpoint_id), + &Checkpoint::from_parts(tree_state, marks_removed), + )? + } + + Ok(()) +} + pub(crate) fn update_checkpoint_with( conn: &rusqlite::Transaction<'_>, checkpoint_id: BlockHeight, @@ -426,11 +650,17 @@ pub(crate) fn remove_checkpoint( conn: &rusqlite::Transaction<'_>, checkpoint_id: BlockHeight, ) -> Result<(), Error> { - conn.execute( - "DELETE FROM sapling_tree_checkpoints WHERE checkpoint_id = ?", - [u32::from(checkpoint_id)], - ) - .map_err(Either::Right)?; + // sapling_tree_checkpoints is constructed with `ON DELETE CASCADE` + let mut stmt_delete_checkpoint = conn + .prepare_cached( + "DELETE FROM sapling_tree_checkpoints + WHERE checkpoint_id = :checkpoint_id", + ) + .map_err(Either::Right)?; + + stmt_delete_checkpoint + .execute(named_params![":checkpoint_id": u32::from(checkpoint_id),]) + .map_err(Either::Right)?; Ok(()) } @@ -452,3 +682,62 @@ pub(crate) fn truncate_checkpoints( .map_err(Either::Right)?; Ok(()) } + +#[cfg(test)] +mod tests { + use tempfile::NamedTempFile; + + use incrementalmerkletree::testing::{ + check_append, check_checkpoint_rewind, check_remove_mark, check_rewind_remove_mark, + check_root_hashes, check_witness_consistency, check_witnesses, + }; + use shardtree::ShardTree; + + use super::SqliteShardStore; + use crate::{tests, wallet::init::init_wallet_db, WalletDb}; + + fn new_tree(m: usize) -> ShardTree, 4, 3> { + let data_file = NamedTempFile::new().unwrap(); + let mut db_data = WalletDb::for_path(data_file.path(), tests::network()).unwrap(); + data_file.keep().unwrap(); + + init_wallet_db(&mut db_data, None).unwrap(); + let store = SqliteShardStore::<_, String, 3>::from_connection(db_data.conn).unwrap(); + ShardTree::new(store, m) + } + + #[test] + fn append() { + check_append(new_tree); + } + + #[test] + fn root_hashes() { + check_root_hashes(new_tree); + } + + #[test] + fn witnesses() { + check_witnesses(new_tree); + } + + #[test] + fn witness_consistency() { + check_witness_consistency(new_tree); + } + + #[test] + fn checkpoint_rewind() { + check_checkpoint_rewind(new_tree); + } + + #[test] + fn remove_mark() { + check_remove_mark(new_tree); + } + + #[test] + fn rewind_remove_mark() { + check_rewind_remove_mark(new_tree); + } +} diff --git a/zcash_primitives/src/consensus.rs b/zcash_primitives/src/consensus.rs index dc972f700..563c69806 100644 --- a/zcash_primitives/src/consensus.rs +++ b/zcash_primitives/src/consensus.rs @@ -627,6 +627,12 @@ pub mod testing { ) }) } + + impl incrementalmerkletree::testing::TestCheckpoint for BlockHeight { + fn from_u64(value: u64) -> Self { + BlockHeight(u32::try_from(value).expect("Test checkpoint ids do not exceed 32 bits")) + } + } } #[cfg(test)] diff --git a/zcash_primitives/src/merkle_tree.rs b/zcash_primitives/src/merkle_tree.rs index 6cda449bc..0a24b8a1f 100644 --- a/zcash_primitives/src/merkle_tree.rs +++ b/zcash_primitives/src/merkle_tree.rs @@ -292,6 +292,7 @@ pub mod testing { use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use incrementalmerkletree::frontier::testing::TestNode; use std::io::{self, Read, Write}; + use zcash_encoding::Vector; use super::HashSer; @@ -304,6 +305,23 @@ pub mod testing { writer.write_u64::(self.0) } } + + impl HashSer for String { + fn read(reader: R) -> io::Result { + Vector::read(reader, |r| r.read_u8()).and_then(|xs| { + String::from_utf8(xs).map_err(|e| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("Not a valid utf8 string: {:?}", e), + ) + }) + }) + } + + fn write(&self, writer: W) -> io::Result<()> { + Vector::write(writer, self.as_bytes(), |w, b| w.write_u8(*b)) + } + } } #[cfg(test)] From 106669d9773b452ec410b76a7ae05e297209315c Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 16 Jun 2023 14:30:53 -0600 Subject: [PATCH 10/27] zcash_client_sqlite: Generalize SQLite-backed ShardStore impl to make it reusable for Orchard. --- zcash_client_sqlite/src/lib.rs | 4 +- .../init/migrations/shardtree_support.rs | 1 + .../src/wallet/sapling/commitment_tree.rs | 286 +++++++++++------- 3 files changed, 183 insertions(+), 108 deletions(-) diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 9df48d5ca..edf8d78a9 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -633,7 +633,7 @@ impl WalletCommitmentTrees for WalletDb WalletCommitmentTrees for WalletDb>>, { let mut shardtree = ShardTree::new( - SqliteShardStore::from_connection(self.conn.0) + SqliteShardStore::from_connection(self.conn.0, "sapling") .map_err(|e| ShardTreeError::Storage(Either::Right(e)))?, 100, ); diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index 6a597d56c..e5b60ad11 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -94,6 +94,7 @@ impl RusqliteMigration for Migration { let shard_store = SqliteShardStore::<_, sapling::Node, SAPLING_SHARD_HEIGHT>::from_connection( transaction, + "sapling", )?; let mut shard_tree: ShardTree< _, diff --git a/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs b/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs index f685350c1..8ce583f9d 100644 --- a/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs @@ -15,15 +15,20 @@ use crate::serialization::{read_shard, write_shard_v1}; pub struct SqliteShardStore { pub(crate) conn: C, + table_prefix: &'static str, _hash_type: PhantomData, } impl SqliteShardStore { const SHARD_ROOT_LEVEL: Level = Level::new(SHARD_HEIGHT); - pub(crate) fn from_connection(conn: C) -> Result { + pub(crate) fn from_connection( + conn: C, + table_prefix: &'static str, + ) -> Result { Ok(SqliteShardStore { conn, + table_prefix, _hash_type: PhantomData, }) } @@ -40,39 +45,39 @@ impl<'conn, 'a: 'conn, H: HashSer, const SHARD_HEIGHT: u8> ShardStore &self, shard_root: Address, ) -> Result>, Self::Error> { - get_shard(self.conn, shard_root) + get_shard(self.conn, self.table_prefix, shard_root) } fn last_shard(&self) -> Result>, Self::Error> { - last_shard(self.conn, Self::SHARD_ROOT_LEVEL) + last_shard(self.conn, self.table_prefix, Self::SHARD_ROOT_LEVEL) } fn put_shard(&mut self, subtree: LocatedPrunableTree) -> Result<(), Self::Error> { - put_shard(self.conn, subtree) + put_shard(self.conn, self.table_prefix, subtree) } fn get_shard_roots(&self) -> Result, Self::Error> { - get_shard_roots(self.conn, Self::SHARD_ROOT_LEVEL) + get_shard_roots(self.conn, self.table_prefix, Self::SHARD_ROOT_LEVEL) } fn truncate(&mut self, from: Address) -> Result<(), Self::Error> { - truncate(self.conn, from) + truncate(self.conn, self.table_prefix, from) } fn get_cap(&self) -> Result, Self::Error> { - get_cap(self.conn) + get_cap(self.conn, self.table_prefix) } fn put_cap(&mut self, cap: PrunableTree) -> Result<(), Self::Error> { - put_cap(self.conn, cap) + put_cap(self.conn, self.table_prefix, cap) } fn min_checkpoint_id(&self) -> Result, Self::Error> { - min_checkpoint_id(self.conn) + min_checkpoint_id(self.conn, self.table_prefix) } fn max_checkpoint_id(&self) -> Result, Self::Error> { - max_checkpoint_id(self.conn) + max_checkpoint_id(self.conn, self.table_prefix) } fn add_checkpoint( @@ -80,32 +85,32 @@ impl<'conn, 'a: 'conn, H: HashSer, const SHARD_HEIGHT: u8> ShardStore checkpoint_id: Self::CheckpointId, checkpoint: Checkpoint, ) -> Result<(), Self::Error> { - add_checkpoint(self.conn, checkpoint_id, checkpoint) + add_checkpoint(self.conn, self.table_prefix, checkpoint_id, checkpoint) } fn checkpoint_count(&self) -> Result { - checkpoint_count(self.conn) + checkpoint_count(self.conn, self.table_prefix) } fn get_checkpoint_at_depth( &self, checkpoint_depth: usize, ) -> Result, Self::Error> { - get_checkpoint_at_depth(self.conn, checkpoint_depth) + get_checkpoint_at_depth(self.conn, self.table_prefix, checkpoint_depth) } fn get_checkpoint( &self, checkpoint_id: &Self::CheckpointId, ) -> Result, Self::Error> { - get_checkpoint(self.conn, *checkpoint_id) + get_checkpoint(self.conn, self.table_prefix, *checkpoint_id) } fn with_checkpoints(&mut self, limit: usize, callback: F) -> Result<(), Self::Error> where F: FnMut(&Self::CheckpointId, &Checkpoint) -> Result<(), Self::Error>, { - with_checkpoints(self.conn, limit, callback) + with_checkpoints(self.conn, self.table_prefix, limit, callback) } fn update_checkpoint_with( @@ -116,18 +121,18 @@ impl<'conn, 'a: 'conn, H: HashSer, const SHARD_HEIGHT: u8> ShardStore where F: Fn(&mut Checkpoint) -> Result<(), Self::Error>, { - update_checkpoint_with(self.conn, *checkpoint_id, update) + update_checkpoint_with(self.conn, self.table_prefix, *checkpoint_id, update) } fn remove_checkpoint(&mut self, checkpoint_id: &Self::CheckpointId) -> Result<(), Self::Error> { - remove_checkpoint(self.conn, *checkpoint_id) + remove_checkpoint(self.conn, self.table_prefix, *checkpoint_id) } fn truncate_checkpoints( &mut self, checkpoint_id: &Self::CheckpointId, ) -> Result<(), Self::Error> { - truncate_checkpoints(self.conn, *checkpoint_id) + truncate_checkpoints(self.conn, self.table_prefix, *checkpoint_id) } } @@ -142,42 +147,42 @@ impl ShardStore &self, shard_root: Address, ) -> Result>, Self::Error> { - get_shard(&self.conn, shard_root) + get_shard(&self.conn, self.table_prefix, shard_root) } fn last_shard(&self) -> Result>, Self::Error> { - last_shard(&self.conn, Self::SHARD_ROOT_LEVEL) + last_shard(&self.conn, self.table_prefix, Self::SHARD_ROOT_LEVEL) } fn put_shard(&mut self, subtree: LocatedPrunableTree) -> Result<(), Self::Error> { let tx = self.conn.transaction().map_err(Either::Right)?; - put_shard(&tx, subtree)?; + put_shard(&tx, self.table_prefix, subtree)?; tx.commit().map_err(Either::Right)?; Ok(()) } fn get_shard_roots(&self) -> Result, Self::Error> { - get_shard_roots(&self.conn, Self::SHARD_ROOT_LEVEL) + get_shard_roots(&self.conn, self.table_prefix, Self::SHARD_ROOT_LEVEL) } fn truncate(&mut self, from: Address) -> Result<(), Self::Error> { - truncate(&self.conn, from) + truncate(&self.conn, self.table_prefix, from) } fn get_cap(&self) -> Result, Self::Error> { - get_cap(&self.conn) + get_cap(&self.conn, self.table_prefix) } fn put_cap(&mut self, cap: PrunableTree) -> Result<(), Self::Error> { - put_cap(&self.conn, cap) + put_cap(&self.conn, self.table_prefix, cap) } fn min_checkpoint_id(&self) -> Result, Self::Error> { - min_checkpoint_id(&self.conn) + min_checkpoint_id(&self.conn, self.table_prefix) } fn max_checkpoint_id(&self) -> Result, Self::Error> { - max_checkpoint_id(&self.conn) + max_checkpoint_id(&self.conn, self.table_prefix) } fn add_checkpoint( @@ -186,26 +191,26 @@ impl ShardStore checkpoint: Checkpoint, ) -> Result<(), Self::Error> { let tx = self.conn.transaction().map_err(Either::Right)?; - add_checkpoint(&tx, checkpoint_id, checkpoint)?; + add_checkpoint(&tx, self.table_prefix, checkpoint_id, checkpoint)?; tx.commit().map_err(Either::Right) } fn checkpoint_count(&self) -> Result { - checkpoint_count(&self.conn) + checkpoint_count(&self.conn, self.table_prefix) } fn get_checkpoint_at_depth( &self, checkpoint_depth: usize, ) -> Result, Self::Error> { - get_checkpoint_at_depth(&self.conn, checkpoint_depth) + get_checkpoint_at_depth(&self.conn, self.table_prefix, checkpoint_depth) } fn get_checkpoint( &self, checkpoint_id: &Self::CheckpointId, ) -> Result, Self::Error> { - get_checkpoint(&self.conn, *checkpoint_id) + get_checkpoint(&self.conn, self.table_prefix, *checkpoint_id) } fn with_checkpoints(&mut self, limit: usize, callback: F) -> Result<(), Self::Error> @@ -213,7 +218,7 @@ impl ShardStore F: FnMut(&Self::CheckpointId, &Checkpoint) -> Result<(), Self::Error>, { let tx = self.conn.transaction().map_err(Either::Right)?; - with_checkpoints(&tx, limit, callback)?; + with_checkpoints(&tx, self.table_prefix, limit, callback)?; tx.commit().map_err(Either::Right) } @@ -226,14 +231,14 @@ impl ShardStore F: Fn(&mut Checkpoint) -> Result<(), Self::Error>, { let tx = self.conn.transaction().map_err(Either::Right)?; - let result = update_checkpoint_with(&tx, *checkpoint_id, update)?; + let result = update_checkpoint_with(&tx, self.table_prefix, *checkpoint_id, update)?; tx.commit().map_err(Either::Right)?; Ok(result) } fn remove_checkpoint(&mut self, checkpoint_id: &Self::CheckpointId) -> Result<(), Self::Error> { let tx = self.conn.transaction().map_err(Either::Right)?; - remove_checkpoint(&tx, *checkpoint_id)?; + remove_checkpoint(&tx, self.table_prefix, *checkpoint_id)?; tx.commit().map_err(Either::Right) } @@ -242,7 +247,7 @@ impl ShardStore checkpoint_id: &Self::CheckpointId, ) -> Result<(), Self::Error> { let tx = self.conn.transaction().map_err(Either::Right)?; - truncate_checkpoints(&tx, *checkpoint_id)?; + truncate_checkpoints(&tx, self.table_prefix, *checkpoint_id)?; tx.commit().map_err(Either::Right) } } @@ -251,12 +256,16 @@ type Error = Either; pub(crate) fn get_shard( conn: &rusqlite::Connection, + table_prefix: &'static str, shard_root: Address, ) -> Result>, Error> { conn.query_row( - "SELECT shard_data - FROM sapling_tree_shards - WHERE shard_index = :shard_index", + &format!( + "SELECT shard_data + FROM {}_tree_shards + WHERE shard_index = :shard_index", + table_prefix + ), named_params![":shard_index": shard_root.index()], |row| row.get::<_, Vec>(0), ) @@ -271,13 +280,17 @@ pub(crate) fn get_shard( pub(crate) fn last_shard( conn: &rusqlite::Connection, + table_prefix: &'static str, shard_root_level: Level, ) -> Result>, Error> { conn.query_row( - "SELECT shard_index, shard_data - FROM sapling_tree_shards - ORDER BY shard_index DESC - LIMIT 1", + &format!( + "SELECT shard_index, shard_data + FROM {}_tree_shards + ORDER BY shard_index DESC + LIMIT 1", + table_prefix + ), [], |row| { let shard_index: u64 = row.get(0)?; @@ -297,6 +310,7 @@ pub(crate) fn last_shard( pub(crate) fn put_shard( conn: &rusqlite::Transaction<'_>, + table_prefix: &'static str, subtree: LocatedPrunableTree, ) -> Result<(), Error> { let subtree_root_hash = subtree @@ -316,13 +330,14 @@ pub(crate) fn put_shard( write_shard_v1(&mut subtree_data, subtree.root()).map_err(Either::Left)?; let mut stmt_put_shard = conn - .prepare_cached( - "INSERT INTO sapling_tree_shards (shard_index, root_hash, shard_data) + .prepare_cached(&format!( + "INSERT INTO {}_tree_shards (shard_index, root_hash, shard_data) VALUES (:shard_index, :root_hash, :shard_data) ON CONFLICT (shard_index) DO UPDATE SET root_hash = :root_hash, shard_data = :shard_data", - ) + table_prefix + )) .map_err(Either::Right)?; stmt_put_shard @@ -338,10 +353,14 @@ pub(crate) fn put_shard( pub(crate) fn get_shard_roots( conn: &rusqlite::Connection, + table_prefix: &'static str, shard_root_level: Level, ) -> Result, Error> { let mut stmt = conn - .prepare("SELECT shard_index FROM sapling_tree_shards ORDER BY shard_index") + .prepare(&format!( + "SELECT shard_index FROM {}_tree_shards ORDER BY shard_index", + table_prefix + )) .map_err(Either::Right)?; let mut rows = stmt.query([]).map_err(Either::Right)?; @@ -355,19 +374,31 @@ pub(crate) fn get_shard_roots( Ok(res) } -pub(crate) fn truncate(conn: &rusqlite::Connection, from: Address) -> Result<(), Error> { +pub(crate) fn truncate( + conn: &rusqlite::Connection, + table_prefix: &'static str, + from: Address, +) -> Result<(), Error> { conn.execute( - "DELETE FROM sapling_tree_shards WHERE shard_index >= ?", + &format!( + "DELETE FROM {}_tree_shards WHERE shard_index >= ?", + table_prefix + ), [from.index()], ) .map_err(Either::Right) .map(|_| ()) } -pub(crate) fn get_cap(conn: &rusqlite::Connection) -> Result, Error> { - conn.query_row("SELECT cap_data FROM sapling_tree_cap", [], |row| { - row.get::<_, Vec>(0) - }) +pub(crate) fn get_cap( + conn: &rusqlite::Connection, + table_prefix: &'static str, +) -> Result, Error> { + conn.query_row( + &format!("SELECT cap_data FROM {}_tree_cap", table_prefix), + [], + |row| row.get::<_, Vec>(0), + ) .optional() .map_err(Either::Right)? .map_or_else( @@ -378,15 +409,17 @@ pub(crate) fn get_cap(conn: &rusqlite::Connection) -> Result( conn: &rusqlite::Connection, + table_prefix: &'static str, cap: PrunableTree, ) -> Result<(), Error> { let mut stmt = conn - .prepare_cached( - "INSERT INTO sapling_tree_cap (cap_id, cap_data) - VALUES (0, :cap_data) - ON CONFLICT (cap_id) DO UPDATE - SET cap_data = :cap_data", - ) + .prepare_cached(&format!( + "INSERT INTO {}_tree_cap (cap_id, cap_data) + VALUES (0, :cap_data) + ON CONFLICT (cap_id) DO UPDATE + SET cap_data = :cap_data", + table_prefix + )) .map_err(Either::Right)?; let mut cap_data = vec![]; @@ -396,9 +429,15 @@ pub(crate) fn put_cap( Ok(()) } -pub(crate) fn min_checkpoint_id(conn: &rusqlite::Connection) -> Result, Error> { +pub(crate) fn min_checkpoint_id( + conn: &rusqlite::Connection, + table_prefix: &'static str, +) -> Result, Error> { conn.query_row( - "SELECT MIN(checkpoint_id) FROM sapling_tree_checkpoints", + &format!( + "SELECT MIN(checkpoint_id) FROM {}_tree_checkpoints", + table_prefix + ), [], |row| { row.get::<_, Option>(0) @@ -408,9 +447,15 @@ pub(crate) fn min_checkpoint_id(conn: &rusqlite::Connection) -> Result Result, Error> { +pub(crate) fn max_checkpoint_id( + conn: &rusqlite::Connection, + table_prefix: &'static str, +) -> Result, Error> { conn.query_row( - "SELECT MAX(checkpoint_id) FROM sapling_tree_checkpoints", + &format!( + "SELECT MAX(checkpoint_id) FROM {}_tree_checkpoints", + table_prefix + ), [], |row| { row.get::<_, Option>(0) @@ -422,14 +467,16 @@ pub(crate) fn max_checkpoint_id(conn: &rusqlite::Connection) -> Result, + table_prefix: &'static str, checkpoint_id: BlockHeight, checkpoint: Checkpoint, ) -> Result<(), Error> { let mut stmt_insert_checkpoint = conn - .prepare_cached( - "INSERT INTO sapling_tree_checkpoints (checkpoint_id, position) + .prepare_cached(&format!( + "INSERT INTO {}_tree_checkpoints (checkpoint_id, position) VALUES (:checkpoint_id, :position)", - ) + table_prefix + )) .map_err(Either::Right)?; stmt_insert_checkpoint @@ -439,10 +486,13 @@ pub(crate) fn add_checkpoint( ]) .map_err(Either::Right)?; - let mut stmt_insert_mark_removed = conn.prepare_cached( - "INSERT INTO sapling_tree_checkpoint_marks_removed (checkpoint_id, mark_removed_position) - VALUES (:checkpoint_id, :position)", - ).map_err(Either::Right)?; + let mut stmt_insert_mark_removed = conn + .prepare_cached(&format!( + "INSERT INTO {}_tree_checkpoint_marks_removed (checkpoint_id, mark_removed_position) + VALUES (:checkpoint_id, :position)", + table_prefix + )) + .map_err(Either::Right)?; for pos in checkpoint.marks_removed() { stmt_insert_mark_removed @@ -456,22 +506,31 @@ pub(crate) fn add_checkpoint( Ok(()) } -pub(crate) fn checkpoint_count(conn: &rusqlite::Connection) -> Result { - conn.query_row("SELECT COUNT(*) FROM sapling_tree_checkpoints", [], |row| { - row.get::<_, usize>(0) - }) +pub(crate) fn checkpoint_count( + conn: &rusqlite::Connection, + table_prefix: &'static str, +) -> Result { + conn.query_row( + &format!("SELECT COUNT(*) FROM {}_tree_checkpoints", table_prefix), + [], + |row| row.get::<_, usize>(0), + ) .map_err(Either::Right) } pub(crate) fn get_checkpoint( conn: &rusqlite::Connection, + table_prefix: &'static str, checkpoint_id: BlockHeight, ) -> Result, Error> { let checkpoint_position = conn .query_row( - "SELECT position - FROM sapling_tree_checkpoints + &format!( + "SELECT position + FROM {}_tree_checkpoints WHERE checkpoint_id = ?", + table_prefix + ), [u32::from(checkpoint_id)], |row| { row.get::<_, Option>(0) @@ -485,11 +544,12 @@ pub(crate) fn get_checkpoint( .map(|pos_opt| { let mut marks_removed = BTreeSet::new(); let mut stmt = conn - .prepare_cached( + .prepare_cached(&format!( "SELECT mark_removed_position - FROM sapling_tree_checkpoint_marks_removed + FROM {}_tree_checkpoint_marks_removed WHERE checkpoint_id = ?", - ) + table_prefix + )) .map_err(Either::Right)?; let mut mark_removed_rows = stmt .query([u32::from(checkpoint_id)]) @@ -513,6 +573,7 @@ pub(crate) fn get_checkpoint( pub(crate) fn get_checkpoint_at_depth( conn: &rusqlite::Connection, + table_prefix: &'static str, checkpoint_depth: usize, ) -> Result, Error> { if checkpoint_depth == 0 { @@ -521,11 +582,14 @@ pub(crate) fn get_checkpoint_at_depth( let checkpoint_parts = conn .query_row( - "SELECT checkpoint_id, position - FROM sapling_tree_checkpoints - ORDER BY checkpoint_id DESC - LIMIT 1 - OFFSET :offset", + &format!( + "SELECT checkpoint_id, position + FROM {}_tree_checkpoints + ORDER BY checkpoint_id DESC + LIMIT 1 + OFFSET :offset", + table_prefix + ), named_params![":offset": checkpoint_depth - 1], |row| { let checkpoint_id: u32 = row.get(0)?; @@ -543,11 +607,12 @@ pub(crate) fn get_checkpoint_at_depth( .map(|(checkpoint_id, pos_opt)| { let mut marks_removed = BTreeSet::new(); let mut stmt = conn - .prepare_cached( + .prepare_cached(&format!( "SELECT mark_removed_position - FROM sapling_tree_checkpoint_marks_removed + FROM {}_tree_checkpoint_marks_removed WHERE checkpoint_id = ?", - ) + table_prefix + )) .map_err(Either::Right)?; let mut mark_removed_rows = stmt .query([u32::from(checkpoint_id)]) @@ -574,6 +639,7 @@ pub(crate) fn get_checkpoint_at_depth( pub(crate) fn with_checkpoints( conn: &rusqlite::Transaction<'_>, + table_prefix: &'static str, limit: usize, mut callback: F, ) -> Result<(), Error> @@ -581,19 +647,21 @@ where F: FnMut(&BlockHeight, &Checkpoint) -> Result<(), Error>, { let mut stmt_get_checkpoints = conn - .prepare_cached( + .prepare_cached(&format!( "SELECT checkpoint_id, position - FROM sapling_tree_checkpoints + FROM {}_tree_checkpoints LIMIT :limit", - ) + table_prefix + )) .map_err(Either::Right)?; let mut stmt_get_checkpoint_marks_removed = conn - .prepare_cached( + .prepare_cached(&format!( "SELECT mark_removed_position - FROM sapling_tree_checkpoint_marks_removed + FROM {}_tree_checkpoint_marks_removed WHERE checkpoint_id = :checkpoint_id", - ) + table_prefix + )) .map_err(Either::Right)?; let mut rows = stmt_get_checkpoints @@ -630,16 +698,17 @@ where pub(crate) fn update_checkpoint_with( conn: &rusqlite::Transaction<'_>, + table_prefix: &'static str, checkpoint_id: BlockHeight, update: F, ) -> Result where F: Fn(&mut Checkpoint) -> Result<(), Error>, { - if let Some(mut c) = get_checkpoint(conn, checkpoint_id)? { + if let Some(mut c) = get_checkpoint(conn, table_prefix, checkpoint_id)? { update(&mut c)?; - remove_checkpoint(conn, checkpoint_id)?; - add_checkpoint(conn, checkpoint_id, c)?; + remove_checkpoint(conn, table_prefix, checkpoint_id)?; + add_checkpoint(conn, table_prefix, checkpoint_id, c)?; Ok(true) } else { Ok(false) @@ -648,14 +717,17 @@ where pub(crate) fn remove_checkpoint( conn: &rusqlite::Transaction<'_>, + table_prefix: &'static str, checkpoint_id: BlockHeight, ) -> Result<(), Error> { - // sapling_tree_checkpoints is constructed with `ON DELETE CASCADE` + // cascading delete here obviates the need to manually delete from + // `tree_checkpoint_marks_removed` let mut stmt_delete_checkpoint = conn - .prepare_cached( - "DELETE FROM sapling_tree_checkpoints + .prepare_cached(&format!( + "DELETE FROM {}_tree_checkpoints WHERE checkpoint_id = :checkpoint_id", - ) + table_prefix + )) .map_err(Either::Right)?; stmt_delete_checkpoint @@ -667,19 +739,20 @@ pub(crate) fn remove_checkpoint( pub(crate) fn truncate_checkpoints( conn: &rusqlite::Transaction<'_>, + table_prefix: &'static str, checkpoint_id: BlockHeight, ) -> Result<(), Error> { + // cascading delete here obviates the need to manually delete from + // `tree_checkpoint_marks_removed` conn.execute( - "DELETE FROM sapling_tree_checkpoints WHERE checkpoint_id >= ?", + &format!( + "DELETE FROM {}_tree_checkpoints WHERE checkpoint_id >= ?", + table_prefix + ), [u32::from(checkpoint_id)], ) .map_err(Either::Right)?; - conn.execute( - "DELETE FROM sapling_tree_checkpoint_marks_removed WHERE checkpoint_id >= ?", - [u32::from(checkpoint_id)], - ) - .map_err(Either::Right)?; Ok(()) } @@ -702,7 +775,8 @@ mod tests { data_file.keep().unwrap(); init_wallet_db(&mut db_data, None).unwrap(); - let store = SqliteShardStore::<_, String, 3>::from_connection(db_data.conn).unwrap(); + let store = + SqliteShardStore::<_, String, 3>::from_connection(db_data.conn, "sapling").unwrap(); ShardTree::new(store, m) } From 547634e210b7817764e1f107206234d79d9ef0e1 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 16 Jun 2023 14:42:36 -0600 Subject: [PATCH 11/27] zcash_client_sqlite: Move the SqliteShardStore implementation out of the `wallet::sapling` module. --- zcash_client_sqlite/src/lib.rs | 2 +- zcash_client_sqlite/src/wallet.rs | 1 + zcash_client_sqlite/src/wallet/{sapling => }/commitment_tree.rs | 0 .../src/wallet/init/migrations/shardtree_support.rs | 2 +- zcash_client_sqlite/src/wallet/sapling.rs | 2 -- 5 files changed, 3 insertions(+), 4 deletions(-) rename zcash_client_sqlite/src/wallet/{sapling => }/commitment_tree.rs (100%) diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index edf8d78a9..1c4e887d2 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -66,7 +66,7 @@ use zcash_client_backend::{ DecryptedOutput, TransferType, }; -use crate::{error::SqliteClientError, wallet::sapling::commitment_tree::SqliteShardStore}; +use crate::{error::SqliteClientError, wallet::commitment_tree::SqliteShardStore}; #[cfg(feature = "unstable")] use { diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 7ecbc080c..757fde8bf 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -104,6 +104,7 @@ use { }, }; +pub(crate) mod commitment_tree; pub mod init; pub(crate) mod sapling; diff --git a/zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs b/zcash_client_sqlite/src/wallet/commitment_tree.rs similarity index 100% rename from zcash_client_sqlite/src/wallet/sapling/commitment_tree.rs rename to zcash_client_sqlite/src/wallet/commitment_tree.rs diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index e5b60ad11..0d292adb1 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -19,8 +19,8 @@ use zcash_primitives::{ }; use crate::wallet::{ + commitment_tree::SqliteShardStore, init::{migrations::received_notes_nullable_nf, WalletMigrationError}, - sapling::commitment_tree::SqliteShardStore, }; pub(super) const MIGRATION_ID: Uuid = Uuid::from_fields( diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index c763534d0..7b3269140 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -22,8 +22,6 @@ use crate::{error::SqliteClientError, NoteId}; use super::memo_repr; -pub(crate) mod commitment_tree; - /// This trait provides a generalization over shielded output representations. pub(crate) trait ReceivedSaplingOutput { fn index(&self) -> usize; From ba709177d3725ec81cc5ce41f349ade9028fd60a Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Thu, 29 Jun 2023 14:28:12 -0600 Subject: [PATCH 12/27] Reorganize Sapling and Orchard note commitment tree sizes in CompactBlock. We move thes fields out into a separate BlockMetadata struct to ensure that future additions to block metadata are structurally separated from future additions to block data. --- .../proto/compact_formats.proto | 25 +++++++++------ .../src/proto/compact_formats.rs | 22 +++++++++---- zcash_client_backend/src/proto/service.rs | 13 ++++++++ zcash_client_backend/src/welding_rig.rs | 31 ++++++++++++------- zcash_client_sqlite/src/lib.rs | 16 +++++++--- 5 files changed, 75 insertions(+), 32 deletions(-) diff --git a/zcash_client_backend/proto/compact_formats.proto b/zcash_client_backend/proto/compact_formats.proto index eac2b2f2f..740d7f7f3 100644 --- a/zcash_client_backend/proto/compact_formats.proto +++ b/zcash_client_backend/proto/compact_formats.proto @@ -10,20 +10,27 @@ option swift_prefix = ""; // Remember that proto3 fields are all optional. A field that is not present will be set to its zero value. // bytes fields of hashes are in canonical little-endian format. +// BlockMetadata represents information about a block that may not be +// represented directly in the block data, but is instead derived from chain +// data or other external sources. +message BlockMetadata { + uint32 saplingCommitmentTreeSize = 1; // the size of the Sapling note commitment tree as of the end of this block + uint32 orchardCommitmentTreeSize = 2; // the size of the Orchard note commitment tree as of the end of this block +} + // CompactBlock is a packaging of ONLY the data from a block that's needed to: // 1. Detect a payment to your shielded Sapling address // 2. Detect a spend of your shielded Sapling notes // 3. Update your witnesses to generate new Sapling spend proofs. message CompactBlock { - uint32 protoVersion = 1; // the version of this wire format, for storage - uint64 height = 2; // the height of this block - bytes hash = 3; // the ID (hash) of this block, same as in block explorers - bytes prevHash = 4; // the ID (hash) of this block's predecessor - uint32 time = 5; // Unix epoch time when the block was mined - bytes header = 6; // (hash, prevHash, and time) OR (full header) - repeated CompactTx vtx = 7; // zero or more compact transactions from this block - uint32 saplingCommitmentTreeSize = 8; // the size of the Sapling note commitment tree as of the end of this block - uint32 orchardCommitmentTreeSize = 9; // the size of the Orchard note commitment tree as of the end of this block + uint32 protoVersion = 1; // the version of this wire format, for storage + uint64 height = 2; // the height of this block + bytes hash = 3; // the ID (hash) of this block, same as in block explorers + bytes prevHash = 4; // the ID (hash) of this block's predecessor + uint32 time = 5; // Unix epoch time when the block was mined + bytes header = 6; // (hash, prevHash, and time) OR (full header) + repeated CompactTx vtx = 7; // zero or more compact transactions from this block + BlockMetadata blockMetadata = 8; // information about this block derived from the chain or other sources } // CompactTx contains the minimum information for a wallet to know if this transaction diff --git a/zcash_client_backend/src/proto/compact_formats.rs b/zcash_client_backend/src/proto/compact_formats.rs index c8d45173c..bf023eacf 100644 --- a/zcash_client_backend/src/proto/compact_formats.rs +++ b/zcash_client_backend/src/proto/compact_formats.rs @@ -1,3 +1,16 @@ +/// BlockMetadata represents information about a block that may not be +/// represented directly in the block data, but is instead derived from chain +/// data or other external sources. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BlockMetadata { + /// the size of the Sapling note commitment tree as of the end of this block + #[prost(uint32, tag = "1")] + pub sapling_commitment_tree_size: u32, + /// the size of the Orchard note commitment tree as of the end of this block + #[prost(uint32, tag = "2")] + pub orchard_commitment_tree_size: u32, +} /// CompactBlock is a packaging of ONLY the data from a block that's needed to: /// 1. Detect a payment to your shielded Sapling address /// 2. Detect a spend of your shielded Sapling notes @@ -26,12 +39,9 @@ pub struct CompactBlock { /// zero or more compact transactions from this block #[prost(message, repeated, tag = "7")] pub vtx: ::prost::alloc::vec::Vec, - /// the size of the Sapling note commitment tree as of the end of this block - #[prost(uint32, tag = "8")] - pub sapling_commitment_tree_size: u32, - /// the size of the Orchard note commitment tree as of the end of this block - #[prost(uint32, tag = "9")] - pub orchard_commitment_tree_size: u32, + /// information about this block derived from the chain or other sources + #[prost(message, optional, tag = "8")] + pub block_metadata: ::core::option::Option, } /// CompactTx contains the minimum information for a wallet to know if this transaction /// is relevant to it (either pays to it or spends from it) via shielded elements diff --git a/zcash_client_backend/src/proto/service.rs b/zcash_client_backend/src/proto/service.rs index 38b15abdb..581762bb3 100644 --- a/zcash_client_backend/src/proto/service.rs +++ b/zcash_client_backend/src/proto/service.rs @@ -1,3 +1,16 @@ +/// BlockMetadata represents information about a block that may not be +/// represented directly in the block data, but is instead derived from chain +/// data or other external sources. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BlockMetadata { + /// the size of the Sapling note commitment tree as of the end of this block + #[prost(uint32, tag = "1")] + pub sapling_commitment_tree_size: u32, + /// the size of the Orchard note commitment tree as of the end of this block + #[prost(uint32, tag = "2")] + pub orchard_commitment_tree_size: u32, +} /// A BlockID message contains identifiers to select a block: a height or a /// hash. Specification by hash is not implemented, but may be in the future. #[allow(clippy::derive_partial_eq_without_eq)] diff --git a/zcash_client_backend/src/welding_rig.rs b/zcash_client_backend/src/welding_rig.rs index 3266d4059..a9e7d3f56 100644 --- a/zcash_client_backend/src/welding_rig.rs +++ b/zcash_client_backend/src/welding_rig.rs @@ -218,16 +218,21 @@ pub(crate) fn scan_block_with_runner< // to use it. `block.sapling_commitment_tree_size` is expected to be correct as of the end of // the block, and we can't have a note of ours in a block with no outputs so treating the zero // default value from the protobuf as `None` is always correct. - let mut sapling_tree_position = if block.sapling_commitment_tree_size == 0 { - initial_commitment_tree_meta.map(|m| (m.sapling_tree_size() + 1).into()) - } else { - let end_position_exclusive = Position::from(u64::from(block.sapling_commitment_tree_size)); + let mut sapling_tree_position = if let Some(sapling_tree_size) = block + .block_metadata + .as_ref() + .map(|m| m.sapling_commitment_tree_size) + .filter(|s| *s != 0) + { + let end_position_exclusive = Position::from(u64::from(sapling_tree_size)); let output_count = block .vtx .iter() .map(|tx| u64::try_from(tx.outputs.len()).unwrap()) .sum(); Some(end_position_exclusive - output_count) + } else { + initial_commitment_tree_meta.map(|m| (m.sapling_tree_size() + 1).into()) }; for tx in block.vtx.into_iter() { @@ -404,11 +409,10 @@ pub(crate) fn scan_block_with_runner< block_hash, block_time: block.time, transactions: wtxs, - sapling_commitment_tree_size: if block.sapling_commitment_tree_size == 0 { - None - } else { - Some(block.sapling_commitment_tree_size) - }, + sapling_commitment_tree_size: block + .block_metadata + .map(|m| m.sapling_commitment_tree_size) + .filter(|s| *s != 0), sapling_commitments: sapling_note_commitments, }) } @@ -439,7 +443,7 @@ mod tests { use crate::{ data_api::chain::CommitmentTreeMeta, proto::compact_formats::{ - CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, + BlockMetadata, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, }, scan::BatchRunner, }; @@ -547,8 +551,11 @@ mod tests { cb.vtx.push(tx); } - cb.sapling_commitment_tree_size = initial_sapling_tree_size - + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(); + cb.block_metadata = Some(BlockMetadata { + sapling_commitment_tree_size: initial_sapling_tree_size + + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(), + ..Default::default() + }); cb } diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 1c4e887d2..60447a55e 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -964,7 +964,7 @@ mod tests { data_api::{WalletRead, WalletWrite}, keys::{sapling, UnifiedFullViewingKey}, proto::compact_formats::{ - CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, + BlockMetadata, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, }, }; @@ -1109,8 +1109,11 @@ mod tests { }; cb.prev_hash.extend_from_slice(&prev_hash.0); cb.vtx.push(ctx); - cb.sapling_commitment_tree_size = initial_sapling_tree_size - + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(); + cb.block_metadata = Some(BlockMetadata { + sapling_commitment_tree_size: initial_sapling_tree_size + + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(), + ..Default::default() + }); (cb, note.nf(&dfvk.fvk().vk.nk, 0)) } @@ -1197,8 +1200,11 @@ mod tests { }; cb.prev_hash.extend_from_slice(&prev_hash.0); cb.vtx.push(ctx); - cb.sapling_commitment_tree_size = initial_sapling_tree_size - + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(); + cb.block_metadata = Some(BlockMetadata { + sapling_commitment_tree_size: initial_sapling_tree_size + + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(), + ..Default::default() + }); cb } From d65b129b43cc29b3d5b51de61f860fab23c60544 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Thu, 29 Jun 2023 16:26:22 -0600 Subject: [PATCH 13/27] Apply changelog, documentation & style suggestions from code review Co-authored-by: str4d --- zcash_client_backend/CHANGELOG.md | 77 +++++++++++++++---- zcash_client_backend/src/data_api.rs | 4 + zcash_client_backend/src/data_api/chain.rs | 49 +++++++----- zcash_client_backend/src/data_api/wallet.rs | 4 + .../src/data_api/wallet/input_selection.rs | 6 +- zcash_client_backend/src/welding_rig.rs | 31 ++++++++ zcash_client_sqlite/CHANGELOG.md | 9 +++ zcash_client_sqlite/src/error.rs | 3 +- zcash_client_sqlite/src/lib.rs | 18 +++-- zcash_client_sqlite/src/serialization.rs | 18 +++-- zcash_client_sqlite/src/wallet.rs | 2 +- .../src/wallet/commitment_tree.rs | 6 +- .../init/migrations/shardtree_support.rs | 18 ++--- zcash_client_sqlite/src/wallet/sapling.rs | 2 +- zcash_primitives/CHANGELOG.md | 3 + 15 files changed, 181 insertions(+), 69 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 28ab39864..0110fa88b 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -7,30 +7,73 @@ and this library adheres to Rust's notion of ## [Unreleased] ### Added -- `impl Eq for address::RecipientAddress` -- `impl Eq for zip321::{Payment, TransactionRequest}` -- `data_api::NullifierQuery` for use with `WalletRead::get_sapling_nullifiers` -- `WalletWrite::put_block` -- `impl Debug` for `{data_api::wallet::input_selection::Proposal, wallet::ReceivedSaplingNote} +- `impl Eq for zcash_client_backend::address::RecipientAddress` +- `impl Eq for zcash_client_backend::zip321::{Payment, TransactionRequest}` +- `impl Debug` for `zcash_client_backend::{data_api::wallet::input_selection::Proposal, wallet::ReceivedSaplingNote}` +- `zcash_client_backend::data_api`: + - `WalletRead::{fully_scanned_height, suggest_scan_ranges}` + - `WalletWrite::put_block` + - `WalletCommitmentTrees` + - `testing::MockWalletDb::new` + - `NullifierQuery` for use with `WalletRead::get_sapling_nullifiers` + - `wallet::input_sellection::Proposal::{min_target_height, min_anchor_height}`: +- `zcash_client_backend::wallet::WalletSaplingOutput::note_commitment_tree_position` +- `zcash_client_backend::welding_rig::SyncError` ### Changed - MSRV is now 1.65.0. - Bumped dependencies to `hdwallet 0.4`, `zcash_primitives 0.12`, `zcash_note_encryption 0.4`, `incrementalmerkletree 0.4`, `orchard 0.5`, `bs58 0.5` -- `WalletRead::get_memo` now returns `Result, Self::Error>` - instead of `Result` in order to make representable - wallet states where the full note plaintext is not available. -- `WalletRead::get_nullifiers` has been renamed to `WalletRead::get_sapling_nullifiers` - and its signature has changed; it now subsumes the removed `WalletRead::get_all_nullifiers`. -- `wallet::SpendableNote` has been renamed to `wallet::ReceivedSaplingNote`. -- `data_api::chain::scan_cached_blocks` now takes a `from_height` argument that - permits the caller to control the starting position of the scan range. -- `WalletWrite::advance_by_block` has been replaced by `WalletWrite::put_block` - to reflect the semantic change that scanning is no longer a linear operation. +- `zcash_client_backend::data_api`: + - `WalletRead::get_memo` now returns `Result, Self::Error>` + instead of `Result` in order to make representable + wallet states where the full note plaintext is not available. + - `WalletRead::get_nullifiers` has been renamed to `WalletRead::get_sapling_nullifiers` + and its signature has changed; it now subsumes the removed `WalletRead::get_all_nullifiers`. + - `chain::scan_cached_blocks` now takes a `from_height` argument that + permits the caller to control the starting position of the scan range. + - A new `CommitmentTree` variant has been added to `data_api::error::Error` + - `data_api::wallet::{create_spend_to_address, create_proposed_transaction, + shield_transparent_funds}` all now require that `WalletCommitmentTrees` be + implemented for the type passed to them for the `wallet_db` parameter. + - `data_api::wallet::create_proposed_transaction` now takes an additional + `min_confirmations` argument. + - A new `Sync` variant has been added to `data_api::chain::error::Error`. + - A new `SyncRequired` variant has been added to `data_api::wallet::input_selection::InputSelectorError`. +- `zcash_client_backend::wallet`: + - `SpendableNote` has been renamed to `ReceivedSaplingNote`. + - Arguments to `WalletSaplingOutput::from_parts` have changed. +- `zcash_client_backend::data_api::wallet::input_selection::InputSelector`: + - Arguments to `{propose_transaction, propose_shielding}` have changed. +- `zcash_client_backend::wallet::ReceivedSaplingNote::note_commitment_tree_position` + has replaced the `witness` field in the same struct. +- `zcash_client_backend::welding_rig::ScanningKey::sapling_nf` has been changed to + take a note position instead of an incremental witness for the note. +- Arguments to `zcash_client_backend::welding_rig::scan_block` have changed. This + method now takes an optional `CommitmentTreeMeta` argument instead of a base commitment + tree and incremental witnesses for each previously-known note. + ### Removed -- `WalletRead::get_all_nullifiers` -- `WalletWrite::advance_by_block` +- `zcash_client_backend::data_api`: + - `WalletRead::get_all_nullifiers` + - `WalletRead::{get_commitment_tree, get_witnesses}` (use + `WalletRead::fully_scanned_height` instead). + - `WalletWrite::advance_by_block` (use `WalletWrite::put_block` instead). + - The `commitment_tree` and `transactions` properties of the `PrunedBlock` + struct are now owned values instead of references, making this a fully owned type. + It is now parameterized by the nullifier type instead of by a lifetime for the + fields that were previously references. In addition, two new properties, + `sapling_commitment_tree_size` and `sapling_commitments` have been added. + - `testing::MockWalletDb`, which is available under the `test-dependencies` + feature flag, has been modified by the addition of a `sapling_tree` property. + - `wallet::input_selection`: + - `Proposal::target_height` (use `Proposal::min_target_height` instead). +- `zcash_client_backend::wallet::WalletSaplingOutput::{witness, witness_mut}` + have been removed as individual incremental witnesses are no longer tracked on a + per-note basis. The global note commitment tree for the wallet should be used + to obtain witnesses for spend operations instead. + ## [0.9.0] - 2023-04-28 ### Added diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index eafa0e89c..d1031e570 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -438,6 +438,10 @@ pub trait WalletWrite: WalletRead { ) -> Result; } +/// This trait describes a capability for manipulating wallet note commitment trees. +/// +/// At present, this only serves the Sapling protocol, but it will be modified to +/// also provide operations related to Orchard note commitment trees in the future. pub trait WalletCommitmentTrees { type Error; type SaplingShardStore<'a>: ShardStore< diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index dbec6768d..66355b06a 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -101,16 +101,20 @@ use crate::{ pub mod error; use error::{ChainError, Error}; +/// Metadata describing the sizes of the zcash note commitment trees as of a particular block. pub struct CommitmentTreeMeta { sapling_tree_size: u64, //TODO: orchard_tree_size: u64 } impl CommitmentTreeMeta { + /// Constructs a new [`CommitmentTreeMeta`] value from its constituent parts. pub fn from_parts(sapling_tree_size: u64) -> Self { Self { sapling_tree_size } } + /// Returns the size of the Sapling note commitment tree as of the block that this + /// [`CommitmentTreeMeta`] describes. pub fn sapling_tree_size(&self) -> u64 { self.sapling_tree_size } @@ -199,23 +203,19 @@ where /// Scans at most `limit` new blocks added to the block source for any transactions received by the /// tracked accounts. /// -/// This function will return without error after scanning at most `limit` new blocks, to enable -/// the caller to update their UI with scanning progress. Repeatedly calling this function will -/// process sequential ranges of blocks, and is equivalent to calling `scan_cached_blocks` and -/// passing `None` for the optional `limit` value. -/// -/// This function pays attention only to cached blocks with heights greater than the highest -/// scanned block in `data`. Cached blocks with lower heights are not verified against -/// previously-scanned blocks. In particular, this function **assumes** that the caller is handling -/// rollbacks. +/// If the `from_height` argument is not `None`, then the block source will begin requesting blocks +/// from the provided block source at the specified height; if `from_height` is `None then this +/// will begin scanning at first block after the position to which the wallet has previously +/// fully scanned the chain, thereby beginning or continuing a linear scan over all blocks. /// -/// For brand-new light client databases, this function starts scanning from the Sapling activation -/// height. This height can be fast-forwarded to a more recent block by initializing the client -/// database with a starting block (for example, calling `init_blocks_table` before this function -/// if using `zcash_client_sqlite`). +/// This function will return without error after scanning at most `limit` new blocks, to enable +/// the caller to update their UI with scanning progress. Repeatedly calling this function with +/// `from_height == None` will process sequential ranges of blocks. /// -/// Scanned blocks are required to be height-sequential. If a block is missing from the block -/// source, an error will be returned with cause [`error::Cause::BlockHeightDiscontinuity`]. +/// For brand-new light client databases, if `from_height == None` this function starts scanning +/// from the Sapling activation height. This height can be fast-forwarded to a more recent block by +/// initializing the client database with a starting block (for example, calling +/// `init_blocks_table` before this function if using `zcash_client_sqlite`). #[tracing::instrument(skip(params, block_source, data_db))] #[allow(clippy::type_complexity)] pub fn scan_cached_blocks( @@ -260,7 +260,7 @@ where ); // Start at either the provided height, or where we synced up to previously. - let (from_height, commitment_tree_meta) = from_height.map_or_else( + let (scan_from, commitment_tree_meta) = from_height.map_or_else( || { data_db.fully_scanned_height().map_or_else( |e| Err(Error::Wallet(e)), @@ -273,7 +273,7 @@ where )?; block_source.with_blocks::<_, DbT::Error, DbT::NoteRef>( - from_height, + scan_from, limit, |block: CompactBlock| { add_block_to_runner(params, block, &mut batch_runner); @@ -283,10 +283,22 @@ where batch_runner.flush(); + let mut last_scanned = None; block_source.with_blocks::<_, DbT::Error, DbT::NoteRef>( - from_height, + scan_from, limit, |block: CompactBlock| { + // block heights should be sequential within a single scan range + let block_height = block.height(); + if let Some(h) = last_scanned { + if block_height != h + 1 { + return Err(Error::Chain(ChainError::block_height_discontinuity( + block.height(), + h, + ))); + } + } + let pruned_block = scan_block_with_runner( params, block, @@ -312,6 +324,7 @@ where data_db.put_block(pruned_block).map_err(Error::Wallet)?; + last_scanned = Some(block_height); Ok(()) }, )?; diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index d37a03329..a47528920 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -122,6 +122,10 @@ where /// spent. A value of 10 confirmations is recommended and 0-conf transactions are /// not supported. /// +/// # Panics +/// +/// Panics if `min_confirmations == 0`; 0-conf transactions are not supported. +/// /// # Examples /// /// ``` diff --git a/zcash_client_backend/src/data_api/wallet/input_selection.rs b/zcash_client_backend/src/data_api/wallet/input_selection.rs index 5017ae335..90c701336 100644 --- a/zcash_client_backend/src/data_api/wallet/input_selection.rs +++ b/zcash_client_backend/src/data_api/wallet/input_selection.rs @@ -62,7 +62,9 @@ impl fmt::Display for InputSelectorError write!(f, "No chain data is available."), + InputSelectorError::SyncRequired => { + write!(f, "Insufficient chain data is available, sync required.") + } } } } @@ -135,7 +137,7 @@ impl Debug for Proposal { .field("min_target_height", &self.min_target_height) .field("min_anchor_height", &self.min_anchor_height) .field("is_shielding", &self.is_shielding) - .finish() + .finish_non_exhaustive() } } diff --git a/zcash_client_backend/src/welding_rig.rs b/zcash_client_backend/src/welding_rig.rs index a9e7d3f56..a8a98069e 100644 --- a/zcash_client_backend/src/welding_rig.rs +++ b/zcash_client_backend/src/welding_rig.rs @@ -672,6 +672,22 @@ mod tests { assert_eq!(tx.sapling_outputs[0].index(), 0); assert_eq!(tx.sapling_outputs[0].account(), AccountId::from(0)); assert_eq!(tx.sapling_outputs[0].note().value().inner(), 5); + + assert_eq!( + pruned_block + .sapling_commitments + .iter() + .map(|(_, retention)| *retention) + .collect::>(), + vec![ + Retention::Ephemeral, + Retention::Marked, + Retention::Checkpoint { + id: pruned_block.block_height, + is_marked: false + } + ] + ); } go(false); @@ -714,5 +730,20 @@ mod tests { assert_eq!(tx.sapling_spends[0].index(), 0); assert_eq!(tx.sapling_spends[0].nf(), &nf); assert_eq!(tx.sapling_spends[0].account(), account); + + assert_eq!( + pruned_block + .sapling_commitments + .iter() + .map(|(_, retention)| *retention) + .collect::>(), + vec![ + Retention::Ephemeral, + Retention::Checkpoint { + id: pruned_block.block_height, + is_marked: false + } + ] + ); } } diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index d72fe90ac..b1e332913 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -6,10 +6,19 @@ and this library adheres to Rust's notion of [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- `zcash_client_sqlite::serialization` Serialization formats for data stored + as SQLite BLOBs in the wallet database. + ### Changed - MSRV is now 1.65.0. - Bumped dependencies to `hdwallet 0.4`, `incrementalmerkletree 0.4`, `bs58 0.5`, `zcash_primitives 0.12` +- A `CommitmentTree` variant has been added to `zcash_client_sqlite::wallet::init::WalletMigrationError` +- `min_confirmations` parameter values are now more strongly enforced. Previously, + a note could be spent with fewer than `min_confirmations` confirmations if the + wallet did not contain enough observed blocks to satisfy the `min_confirmations` + value specified; this situation is now treated as an error. ### Removed - The empty `wallet::transact` module has been removed. diff --git a/zcash_client_sqlite/src/error.rs b/zcash_client_sqlite/src/error.rs index 87f88b844..6eb0939f2 100644 --- a/zcash_client_sqlite/src/error.rs +++ b/zcash_client_sqlite/src/error.rs @@ -78,7 +78,8 @@ pub enum SqliteClientError { #[cfg(feature = "transparent-inputs")] AddressNotRecognized(TransparentAddress), - /// An error occurred in inserting data into one of the wallet's note commitment trees. + /// An error occurred in inserting data into or accessing data from one of the wallet's note + /// commitment trees. CommitmentTree(ShardTreeError>), } diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 60447a55e..9c1ac261a 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -85,6 +85,8 @@ pub mod wallet; /// this delta from the chain tip to be pruned. pub(crate) const PRUNING_HEIGHT: u32 = 100; +pub(crate) const SAPLING_TABLES_PREFIX: &'static str = "sapling"; + /// A newtype wrapper for sqlite primary key values for the notes /// table. #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] @@ -396,10 +398,9 @@ impl WalletWrite for WalletDb ) -> Result, Self::Error> { self.transactionally(|wdb| { // Insert the block into the database. - let block_height = block.block_height; wallet::put_block( wdb.conn.0, - block_height, + block.block_height, block.block_hash, block.block_time, block.sapling_commitment_tree_size.map(|s| s.into()), @@ -423,18 +424,19 @@ impl WalletWrite for WalletDb } } + let sapling_commitments_len = block.sapling_commitments.len(); let mut sapling_commitments = block.sapling_commitments.into_iter(); wdb.with_sapling_tree_mut::<_, _, SqliteClientError>(move |sapling_tree| { if let Some(sapling_tree_size) = block.sapling_commitment_tree_size { let start_position = Position::from(u64::from(sapling_tree_size)) - - u64::try_from(sapling_commitments.len()).unwrap(); + - u64::try_from(sapling_commitments_len).unwrap(); sapling_tree.batch_insert(start_position, &mut sapling_commitments)?; } Ok(()) })?; // Update now-expired transactions that didn't get mined. - wallet::update_expired_notes(wdb.conn.0, block_height)?; + wallet::update_expired_notes(wdb.conn.0, block.block_height)?; Ok(wallet_note_ids) }) @@ -633,10 +635,10 @@ impl WalletCommitmentTrees for WalletDb WalletCommitmentTrees for WalletDb>>, { let mut shardtree = ShardTree::new( - SqliteShardStore::from_connection(self.conn.0, "sapling") + SqliteShardStore::from_connection(self.conn.0, SAPLING_TABLES_PREFIX) .map_err(|e| ShardTreeError::Storage(Either::Right(e)))?, - 100, + PRUNING_HEIGHT.try_into().unwrap(), ); let result = callback(&mut shardtree)?; diff --git a/zcash_client_sqlite/src/serialization.rs b/zcash_client_sqlite/src/serialization.rs index 99cb90dd8..eb1176465 100644 --- a/zcash_client_sqlite/src/serialization.rs +++ b/zcash_client_sqlite/src/serialization.rs @@ -14,10 +14,11 @@ const NIL_TAG: u8 = 0; const LEAF_TAG: u8 = 1; const PARENT_TAG: u8 = 2; -pub fn write_shard_v1( - writer: &mut W, - tree: &PrunableTree, -) -> io::Result<()> { +/// Writes a [`PrunableTree`] to the provided [`Write`] instance. +/// +/// This is the primary method used for ShardTree shard persistence. It writes a version identifier +/// for the most-current serialized form, followed by the tree data. +pub fn write_shard(writer: &mut W, tree: &PrunableTree) -> io::Result<()> { fn write_inner( mut writer: &mut W, tree: &PrunableTree, @@ -80,6 +81,11 @@ fn read_shard_v1(mut reader: &mut R) -> io::Result(mut reader: R) -> io::Result> { match reader.read_u8()? { SER_V1 => read_shard_v1(&mut reader), @@ -97,7 +103,7 @@ mod tests { use shardtree::testing::arb_prunable_tree; use std::io::Cursor; - use super::{read_shard, write_shard_v1}; + use super::{read_shard, write_shard}; proptest! { #[test] @@ -105,7 +111,7 @@ mod tests { tree in arb_prunable_tree(arb_test_node(), 8, 32) ) { let mut tree_data = vec![]; - write_shard_v1(&mut tree_data, &tree).unwrap(); + write_shard(&mut tree_data, &tree).unwrap(); let cursor = Cursor::new(tree_data); let tree_result = read_shard::(cursor).unwrap(); assert_eq!(tree, tree_result); diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 757fde8bf..108b5f5d3 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -640,7 +640,7 @@ pub(crate) fn get_min_unspent_height( /// block, this function does nothing. /// /// This should only be executed inside a transactional context. -pub(crate) fn truncate_to_height( +pub(crate) fn truncate_to_height( conn: &rusqlite::Transaction, params: &P, block_height: BlockHeight, diff --git a/zcash_client_sqlite/src/wallet/commitment_tree.rs b/zcash_client_sqlite/src/wallet/commitment_tree.rs index 8ce583f9d..d28b5c701 100644 --- a/zcash_client_sqlite/src/wallet/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/commitment_tree.rs @@ -11,7 +11,7 @@ use shardtree::{Checkpoint, LocatedPrunableTree, PrunableTree, ShardStore, TreeS use zcash_primitives::{consensus::BlockHeight, merkle_tree::HashSer}; -use crate::serialization::{read_shard, write_shard_v1}; +use crate::serialization::{read_shard, write_shard}; pub struct SqliteShardStore { pub(crate) conn: C, @@ -327,7 +327,7 @@ pub(crate) fn put_shard( .map_err(Either::Left)?; let mut subtree_data = vec![]; - write_shard_v1(&mut subtree_data, subtree.root()).map_err(Either::Left)?; + write_shard(&mut subtree_data, subtree.root()).map_err(Either::Left)?; let mut stmt_put_shard = conn .prepare_cached(&format!( @@ -423,7 +423,7 @@ pub(crate) fn put_cap( .map_err(Either::Right)?; let mut cap_data = vec![]; - write_shard_v1(&mut cap_data, &cap).map_err(Either::Left)?; + write_shard(&mut cap_data, &cap).map_err(Either::Left)?; stmt.execute([cap_data]).map_err(Either::Right)?; Ok(()) diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index 0d292adb1..f4bce16b8 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -18,10 +18,10 @@ use zcash_primitives::{ sapling, }; -use crate::wallet::{ +use crate::{wallet::{ commitment_tree::SqliteShardStore, init::{migrations::received_notes_nullable_nf, WalletMigrationError}, -}; +}, SAPLING_TABLES_PREFIX}; pub(super) const MIGRATION_ID: Uuid = Uuid::from_fields( 0x7da6489d, @@ -94,7 +94,7 @@ impl RusqliteMigration for Migration { let shard_store = SqliteShardStore::<_, sapling::Node, SAPLING_SHARD_HEIGHT>::from_connection( transaction, - "sapling", + SAPLING_TABLES_PREFIX, )?; let mut shard_tree: ShardTree< _, @@ -187,14 +187,8 @@ impl RusqliteMigration for Migration { Ok(()) } - fn down(&self, transaction: &rusqlite::Transaction) -> Result<(), WalletMigrationError> { - transaction.execute_batch( - "DROP TABLE sapling_tree_checkpoint_marks_removed; - DROP TABLE sapling_tree_checkpoints; - DROP TABLE sapling_tree_cap; - DROP TABLE sapling_tree_shards;", - )?; - - Ok(()) + fn down(&self, _transaction: &rusqlite::Transaction) -> Result<(), WalletMigrationError> { + // TODO: something better than just panic? + panic!("Cannot revert this migration."); } } diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 7b3269140..6544b166e 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -425,7 +425,7 @@ pub(crate) mod tests { }, }; - pub fn test_prover() -> impl TxProver { + pub(crate) fn test_prover() -> impl TxProver { match LocalTxProver::with_default_location() { Some(tx_prover) => tx_prover, None => { diff --git a/zcash_primitives/CHANGELOG.md b/zcash_primitives/CHANGELOG.md index 47c2f3a4b..f2404ea94 100644 --- a/zcash_primitives/CHANGELOG.md +++ b/zcash_primitives/CHANGELOG.md @@ -12,6 +12,9 @@ and this library adheres to Rust's notion of - `Builder::add_orchard_spend` - `Builder::add_orchard_output` - `zcash_primitives::transaction::components::orchard::builder` module +- `impl HashSer for String` is provided under the `test-dependencies` feature + flag. This is a test-only impl; the identity leaf value is `_` and the combining + operation is concatenation. ### Changed - `zcash_primitives::transaction`: From 8fa3a08c0b33c79501bf124714fdbabe542a7b28 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Thu, 29 Jun 2023 20:24:33 -0600 Subject: [PATCH 14/27] Fix indexing error in checkpoint determination. --- zcash_client_backend/src/welding_rig.rs | 65 +++++++++++++------------ 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/zcash_client_backend/src/welding_rig.rs b/zcash_client_backend/src/welding_rig.rs index a8a98069e..22aabb0bb 100644 --- a/zcash_client_backend/src/welding_rig.rs +++ b/zcash_client_backend/src/welding_rig.rs @@ -235,9 +235,9 @@ pub(crate) fn scan_block_with_runner< initial_commitment_tree_meta.map(|m| (m.sapling_tree_size() + 1).into()) }; - for tx in block.vtx.into_iter() { + let block_tx_count = block.vtx.len(); + for (tx_idx, tx) in block.vtx.into_iter().enumerate() { let txid = tx.txid(); - let index = tx.index as usize; // Check for spent notes. The only step that is not constant-time is // the filter() at the end. @@ -338,9 +338,20 @@ pub(crate) fn scan_block_with_runner< .collect() }; - for (index, ((_, output), dec_output)) in decoded.iter().zip(decrypted).enumerate() { + for (output_idx, ((_, output), dec_output)) in decoded.iter().zip(decrypted).enumerate() + { // Collect block note commitments let node = sapling::Node::from_cmu(&output.cmu); + let is_checkpoint = output_idx + 1 == decoded.len() && tx_idx + 1 == block_tx_count; + let retention = match (dec_output.is_some(), is_checkpoint) { + (is_marked, true) => Retention::Checkpoint { + id: block_height, + is_marked, + }, + (true, false) => Retention::Marked, + (false, false) => Retention::Ephemeral, + }; + if let Some((note, account, nk)) = dec_output { // A note is marked as "change" if the account that received it // also spent notes in the same transaction. This will catch, @@ -351,11 +362,11 @@ pub(crate) fn scan_block_with_runner< let is_change = spent_from_accounts.contains(&account); let note_commitment_tree_position = sapling_tree_position .ok_or(SyncError::SaplingTreeSizeUnknown(block_height))? - + index.try_into().unwrap(); + + output_idx.try_into().unwrap(); let nf = K::sapling_nf(&nk, ¬e, note_commitment_tree_position); shielded_outputs.push(WalletSaplingOutput::from_parts( - index, + output_idx, output.cmu, output.ephemeral_key.clone(), account, @@ -364,38 +375,16 @@ pub(crate) fn scan_block_with_runner< note_commitment_tree_position, nf, )); - - sapling_note_commitments.push(( - node, - if index == decoded.len() - 1 { - Retention::Checkpoint { - id: block_height, - is_marked: true, - } - } else { - Retention::Marked - }, - )); - } else { - sapling_note_commitments.push(( - node, - if index == decoded.len() - 1 { - Retention::Checkpoint { - id: block_height, - is_marked: false, - } - } else { - Retention::Ephemeral - }, - )); } + + sapling_note_commitments.push((node, retention)); } } if !(shielded_spends.is_empty() && shielded_outputs.is_empty()) { wtxs.push(WalletTx { txid, - index, + index: tx.index as usize, sapling_spends: shielded_spends, sapling_outputs: shielded_outputs, }); @@ -423,6 +412,7 @@ mod tests { ff::{Field, PrimeField}, GroupEncoding, }; + use incrementalmerkletree::Retention; use rand_core::{OsRng, RngCore}; use zcash_note_encryption::Domain; use zcash_primitives::{ @@ -613,6 +603,21 @@ mod tests { assert_eq!(tx.sapling_outputs[0].index(), 0); assert_eq!(tx.sapling_outputs[0].account(), account); assert_eq!(tx.sapling_outputs[0].note().value().inner(), 5); + + assert_eq!( + pruned_block + .sapling_commitments + .iter() + .map(|(_, retention)| *retention) + .collect::>(), + vec![ + Retention::Ephemeral, + Retention::Checkpoint { + id: pruned_block.block_height, + is_marked: true + } + ] + ); } go(false); From c05b3d0c8cc543b3a6780d4df425a9c0e004d8a2 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 30 Jun 2023 08:24:22 -0600 Subject: [PATCH 15/27] Add a test demonstrating off-by-one error in `scan_block_with_runner` --- zcash_client_backend/src/welding_rig.rs | 29 +++++++++++++++++-------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/zcash_client_backend/src/welding_rig.rs b/zcash_client_backend/src/welding_rig.rs index 22aabb0bb..9a9022651 100644 --- a/zcash_client_backend/src/welding_rig.rs +++ b/zcash_client_backend/src/welding_rig.rs @@ -412,7 +412,7 @@ mod tests { ff::{Field, PrimeField}, GroupEncoding, }; - use incrementalmerkletree::Retention; + use incrementalmerkletree::{Retention, Position}; use rand_core::{OsRng, RngCore}; use zcash_note_encryption::Domain; use zcash_primitives::{ @@ -475,13 +475,16 @@ mod tests { /// Create a fake CompactBlock at the given height, with a transaction containing a /// single spend of the given nullifier and a single output paying the given address. /// Returns the CompactBlock. + /// + /// Set `initial_sapling_tree_size` to `None` to simulate a `CompactBlock` retrieved + /// from a `lightwalletd` that is not currently tracking note commitment tree sizes. fn fake_compact_block( height: BlockHeight, nf: Nullifier, dfvk: &DiversifiableFullViewingKey, value: Amount, tx_after: bool, - initial_sapling_tree_size: u32, + initial_sapling_tree_size: Option, ) -> CompactBlock { let to = dfvk.default_address().1; @@ -541,9 +544,12 @@ mod tests { cb.vtx.push(tx); } - cb.block_metadata = Some(BlockMetadata { - sapling_commitment_tree_size: initial_sapling_tree_size - + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(), + cb.block_metadata = initial_sapling_tree_size.map(|s| BlockMetadata { + sapling_commitment_tree_size: s + cb + .vtx + .iter() + .map(|tx| tx.outputs.len() as u32) + .sum::(), ..Default::default() }); @@ -563,7 +569,7 @@ mod tests { &dfvk, Amount::from_u64(5).unwrap(), false, - 0, + None, ); assert_eq!(cb.vtx.len(), 2); @@ -603,7 +609,12 @@ mod tests { assert_eq!(tx.sapling_outputs[0].index(), 0); assert_eq!(tx.sapling_outputs[0].account(), account); assert_eq!(tx.sapling_outputs[0].note().value().inner(), 5); + assert_eq!( + tx.sapling_outputs[0].note_commitment_tree_position(), + Position::from(1) + ); + assert_eq!(pruned_block.sapling_commitment_tree_size, Some(2)); assert_eq!( pruned_block .sapling_commitments @@ -637,7 +648,7 @@ mod tests { &dfvk, Amount::from_u64(5).unwrap(), true, - 0, + Some(0), ); assert_eq!(cb.vtx.len(), 3); @@ -663,7 +674,7 @@ mod tests { cb, &[(&AccountId::from(0), &dfvk)], &[], - Some(&CommitmentTreeMeta::from_parts(0)), + None, batch_runner.as_mut(), ) .unwrap(); @@ -712,7 +723,7 @@ mod tests { &dfvk, Amount::from_u64(5).unwrap(), false, - 0, + Some(0), ); assert_eq!(cb.vtx.len(), 2); let vks: Vec<(&AccountId, &SaplingIvk)> = vec![]; From 45177a51e1836bf228ffdff5ec01cbf02b9707ca Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 30 Jun 2023 09:26:00 -0600 Subject: [PATCH 16/27] Fix off-by-one error in scan_block_with_runner. --- zcash_client_backend/src/data_api/chain.rs | 8 ++-- zcash_client_backend/src/welding_rig.rs | 47 +++++++++++----------- zcash_client_sqlite/src/wallet.rs | 2 +- 3 files changed, 28 insertions(+), 29 deletions(-) diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index 66355b06a..b09432198 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -103,19 +103,19 @@ use error::{ChainError, Error}; /// Metadata describing the sizes of the zcash note commitment trees as of a particular block. pub struct CommitmentTreeMeta { - sapling_tree_size: u64, - //TODO: orchard_tree_size: u64 + sapling_tree_size: u32, + //TODO: orchard_tree_size: u32 } impl CommitmentTreeMeta { /// Constructs a new [`CommitmentTreeMeta`] value from its constituent parts. - pub fn from_parts(sapling_tree_size: u64) -> Self { + pub fn from_parts(sapling_tree_size: u32) -> Self { Self { sapling_tree_size } } /// Returns the size of the Sapling note commitment tree as of the block that this /// [`CommitmentTreeMeta`] describes. - pub fn sapling_tree_size(&self) -> u64 { + pub fn sapling_tree_size(&self) -> u32 { self.sapling_tree_size } } diff --git a/zcash_client_backend/src/welding_rig.rs b/zcash_client_backend/src/welding_rig.rs index 9a9022651..79e3a4c39 100644 --- a/zcash_client_backend/src/welding_rig.rs +++ b/zcash_client_backend/src/welding_rig.rs @@ -218,22 +218,24 @@ pub(crate) fn scan_block_with_runner< // to use it. `block.sapling_commitment_tree_size` is expected to be correct as of the end of // the block, and we can't have a note of ours in a block with no outputs so treating the zero // default value from the protobuf as `None` is always correct. - let mut sapling_tree_position = if let Some(sapling_tree_size) = block + let mut sapling_commitment_tree_size = block .block_metadata .as_ref() - .map(|m| m.sapling_commitment_tree_size) - .filter(|s| *s != 0) - { - let end_position_exclusive = Position::from(u64::from(sapling_tree_size)); - let output_count = block - .vtx - .iter() - .map(|tx| u64::try_from(tx.outputs.len()).unwrap()) - .sum(); - Some(end_position_exclusive - output_count) - } else { - initial_commitment_tree_meta.map(|m| (m.sapling_tree_size() + 1).into()) - }; + .and_then(|m| { + if m.sapling_commitment_tree_size == 0 { + None + } else { + let block_note_count: u32 = block + .vtx + .iter() + .map(|tx| { + u32::try_from(tx.outputs.len()).expect("output count cannot exceed a u32") + }) + .sum(); + Some(m.sapling_commitment_tree_size - block_note_count) + } + }) + .or_else(|| initial_commitment_tree_meta.map(|m| m.sapling_tree_size())); let block_tx_count = block.vtx.len(); for (tx_idx, tx) in block.vtx.into_iter().enumerate() { @@ -274,7 +276,7 @@ pub(crate) fn scan_block_with_runner< // Check for incoming notes while incrementing tree and witnesses let mut shielded_outputs: Vec> = vec![]; - let tx_outputs_len = u64::try_from(tx.outputs.len()).unwrap(); + let tx_outputs_len = u32::try_from(tx.outputs.len()).unwrap(); { let decoded = &tx .outputs @@ -360,9 +362,9 @@ pub(crate) fn scan_block_with_runner< // - Notes created by consolidation transactions. // - Notes sent from one account to itself. let is_change = spent_from_accounts.contains(&account); - let note_commitment_tree_position = sapling_tree_position - .ok_or(SyncError::SaplingTreeSizeUnknown(block_height))? - + output_idx.try_into().unwrap(); + let note_commitment_tree_position = sapling_commitment_tree_size + .map(|s| Position::from(u64::from(s + u32::try_from(output_idx).unwrap()))) + .ok_or(SyncError::SaplingTreeSizeUnknown(block_height))?; let nf = K::sapling_nf(&nk, ¬e, note_commitment_tree_position); shielded_outputs.push(WalletSaplingOutput::from_parts( @@ -390,7 +392,7 @@ pub(crate) fn scan_block_with_runner< }); } - sapling_tree_position = sapling_tree_position.map(|pos| pos + tx_outputs_len); + sapling_commitment_tree_size = sapling_commitment_tree_size.map(|s| s + tx_outputs_len); } Ok(PrunedBlock { @@ -398,10 +400,7 @@ pub(crate) fn scan_block_with_runner< block_hash, block_time: block.time, transactions: wtxs, - sapling_commitment_tree_size: block - .block_metadata - .map(|m| m.sapling_commitment_tree_size) - .filter(|s| *s != 0), + sapling_commitment_tree_size, sapling_commitments: sapling_note_commitments, }) } @@ -412,7 +411,7 @@ mod tests { ff::{Field, PrimeField}, GroupEncoding, }; - use incrementalmerkletree::{Retention, Position}; + use incrementalmerkletree::{Position, Retention}; use rand_core::{OsRng, RngCore}; use zcash_note_encryption::Domain; use zcash_primitives::{ diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 108b5f5d3..cbce42617 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -551,7 +551,7 @@ pub(crate) fn fully_scanned_height( [], |row| { let max_height: u32 = row.get(0)?; - let sapling_tree_size: Option = row.get(1)?; + let sapling_tree_size: Option = row.get(1)?; let sapling_tree: Vec = row.get(2)?; Ok(( BlockHeight::from(max_height), From 95745dd6207d93059b343088d486319e6eb9f628 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 30 Jun 2023 09:45:06 -0600 Subject: [PATCH 17/27] Use ruqlite::Rows::mapped to allow `collect` --- .../src/wallet/commitment_tree.rs | 43 +++++++------------ .../init/migrations/shardtree_support.rs | 11 +++-- 2 files changed, 23 insertions(+), 31 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/commitment_tree.rs b/zcash_client_sqlite/src/wallet/commitment_tree.rs index d28b5c701..885195767 100644 --- a/zcash_client_sqlite/src/wallet/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/commitment_tree.rs @@ -542,7 +542,6 @@ pub(crate) fn get_checkpoint( checkpoint_position .map(|pos_opt| { - let mut marks_removed = BTreeSet::new(); let mut stmt = conn .prepare_cached(&format!( "SELECT mark_removed_position @@ -551,17 +550,14 @@ pub(crate) fn get_checkpoint( table_prefix )) .map_err(Either::Right)?; - let mut mark_removed_rows = stmt + let mark_removed_rows = stmt .query([u32::from(checkpoint_id)]) .map_err(Either::Right)?; - while let Some(row) = mark_removed_rows.next().map_err(Either::Right)? { - marks_removed.insert( - row.get::<_, u64>(0) - .map(Position::from) - .map_err(Either::Right)?, - ); - } + let marks_removed = mark_removed_rows + .mapped(|row| row.get::<_, u64>(0).map(Position::from)) + .collect::, _>>() + .map_err(Either::Right)?; Ok(Checkpoint::from_parts( pos_opt.map_or(TreeState::Empty, TreeState::AtPosition), @@ -605,7 +601,6 @@ pub(crate) fn get_checkpoint_at_depth( checkpoint_parts .map(|(checkpoint_id, pos_opt)| { - let mut marks_removed = BTreeSet::new(); let mut stmt = conn .prepare_cached(&format!( "SELECT mark_removed_position @@ -614,17 +609,14 @@ pub(crate) fn get_checkpoint_at_depth( table_prefix )) .map_err(Either::Right)?; - let mut mark_removed_rows = stmt + let mark_removed_rows = stmt .query([u32::from(checkpoint_id)]) .map_err(Either::Right)?; - while let Some(row) = mark_removed_rows.next().map_err(Either::Right)? { - marks_removed.insert( - row.get::<_, u64>(0) - .map(Position::from) - .map_err(Either::Right)?, - ); - } + let marks_removed = mark_removed_rows + .mapped(|row| row.get::<_, u64>(0).map(Position::from)) + .collect::, _>>() + .map_err(Either::Right)?; Ok(( checkpoint_id, @@ -675,17 +667,14 @@ where .map(|opt| opt.map_or_else(|| TreeState::Empty, |p| TreeState::AtPosition(p.into()))) .map_err(Either::Right)?; - let mut mark_removed_rows = stmt_get_checkpoint_marks_removed + let mark_removed_rows = stmt_get_checkpoint_marks_removed .query(named_params![":checkpoint_id": checkpoint_id]) .map_err(Either::Right)?; - let mut marks_removed = BTreeSet::new(); - while let Some(mr_row) = mark_removed_rows.next().map_err(Either::Right)? { - let mark_removed_position = mr_row - .get::<_, u64>(0) - .map(Position::from) - .map_err(Either::Right)?; - marks_removed.insert(mark_removed_position); - } + + let marks_removed = mark_removed_rows + .mapped(|row| row.get::<_, u64>(0).map(Position::from)) + .collect::, _>>() + .map_err(Either::Right)?; callback( &BlockHeight::from(checkpoint_id), diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index f4bce16b8..08708d65c 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -18,10 +18,13 @@ use zcash_primitives::{ sapling, }; -use crate::{wallet::{ - commitment_tree::SqliteShardStore, - init::{migrations::received_notes_nullable_nf, WalletMigrationError}, -}, SAPLING_TABLES_PREFIX}; +use crate::{ + wallet::{ + commitment_tree::SqliteShardStore, + init::{migrations::received_notes_nullable_nf, WalletMigrationError}, + }, + SAPLING_TABLES_PREFIX, +}; pub(super) const MIGRATION_ID: Uuid = Uuid::from_fields( 0x7da6489d, From cd939f94c42a2726fe47ddccef8dea80b4ef1f84 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 30 Jun 2023 09:48:19 -0600 Subject: [PATCH 18/27] Ensure that checkpoints are ordered by position when querying for pruning. --- zcash_client_sqlite/src/wallet/commitment_tree.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/zcash_client_sqlite/src/wallet/commitment_tree.rs b/zcash_client_sqlite/src/wallet/commitment_tree.rs index 885195767..78548836a 100644 --- a/zcash_client_sqlite/src/wallet/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/commitment_tree.rs @@ -642,6 +642,7 @@ where .prepare_cached(&format!( "SELECT checkpoint_id, position FROM {}_tree_checkpoints + ORDER BY position LIMIT :limit", table_prefix )) From 70497a241cb0382758db4d9f3dfae209f393a999 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 30 Jun 2023 10:42:48 -0600 Subject: [PATCH 19/27] Only store z->t transaction data once, not once per Sapling output. --- zcash_client_sqlite/src/lib.rs | 83 +++++++++++++++++----------------- 1 file changed, 42 insertions(+), 41 deletions(-) diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 9c1ac261a..01c613c2d 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -447,50 +447,51 @@ impl WalletWrite for WalletDb d_tx: DecryptedTransaction, ) -> Result { self.transactionally(|wdb| { - let tx_ref = wallet::put_tx_data(wdb.conn.0, d_tx.tx, None, None)?; - - let mut spending_account_id: Option = None; - for output in d_tx.sapling_outputs { - match output.transfer_type { - TransferType::Outgoing | TransferType::WalletInternal => { - let recipient = if output.transfer_type == TransferType::Outgoing { - Recipient::Sapling(output.note.recipient()) - } else { - Recipient::InternalAccount(output.account, PoolType::Sapling) - }; - - wallet::put_sent_output( - wdb.conn.0, - &wdb.params, - output.account, - tx_ref, - output.index, - &recipient, - Amount::from_u64(output.note.value().inner()).map_err(|_| { - SqliteClientError::CorruptedData( - "Note value is not a valid Zcash amount.".to_string(), - ) - })?, - Some(&output.memo), - )?; + let tx_ref = wallet::put_tx_data(wdb.conn.0, d_tx.tx, None, None)?; + + let mut spending_account_id: Option = None; + for output in d_tx.sapling_outputs { + match output.transfer_type { + TransferType::Outgoing | TransferType::WalletInternal => { + let recipient = if output.transfer_type == TransferType::Outgoing { + Recipient::Sapling(output.note.recipient()) + } else { + Recipient::InternalAccount(output.account, PoolType::Sapling) + }; - if matches!(recipient, Recipient::InternalAccount(_, _)) { - wallet::sapling::put_received_note(wdb.conn.0, output, tx_ref)?; + wallet::put_sent_output( + wdb.conn.0, + &wdb.params, + output.account, + tx_ref, + output.index, + &recipient, + Amount::from_u64(output.note.value().inner()).map_err(|_| { + SqliteClientError::CorruptedData( + "Note value is not a valid Zcash amount.".to_string(), + ) + })?, + Some(&output.memo), + )?; + + if matches!(recipient, Recipient::InternalAccount(_, _)) { + wallet::sapling::put_received_note(wdb.conn.0, output, tx_ref)?; + } } - } - TransferType::Incoming => { - match spending_account_id { - Some(id) => { - if id != output.account { - panic!("Unable to determine a unique account identifier for z->t spend."); + TransferType::Incoming => { + match spending_account_id { + Some(id) => { + if id != output.account { + panic!("Unable to determine a unique account identifier for z->t spend."); + } + } + None => { + spending_account_id = Some(output.account); } } - None => { - spending_account_id = Some(output.account); - } - } - wallet::sapling::put_received_note(wdb.conn.0, output, tx_ref)?; + wallet::sapling::put_received_note(wdb.conn.0, output, tx_ref)?; + } } } @@ -527,8 +528,8 @@ impl WalletWrite for WalletDb } } } - } - Ok(tx_ref) + + Ok(tx_ref) }) } From 8625e9a7771752340a312d6365daae000c40dfbb Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 30 Jun 2023 12:05:15 -0600 Subject: [PATCH 20/27] Handle parsing of the not-present `CommitmentTree` sentinel. --- zcash_client_sqlite/src/wallet.rs | 42 ++++++++++++++++++------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index cbce42617..7847c880b 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -108,6 +108,8 @@ pub(crate) mod commitment_tree; pub mod init; pub(crate) mod sapling; +pub(crate) const BLOCK_SAPLING_FRONTIER_ABSENT: &[u8] = &[0x0]; + pub(crate) fn pool_code(pool_type: PoolType) -> i64 { // These constants are *incidentally* shared with the typecodes // for unified addresses, but this is exclusively an internal @@ -563,23 +565,29 @@ pub(crate) fn fully_scanned_height( .optional()?; res_opt - .map(|(max_height, sapling_tree_size, sapling_tree)| { - let commitment_tree_meta = - CommitmentTreeMeta::from_parts(if let Some(known_size) = sapling_tree_size { - known_size - } else { - // parse the legacy commitment tree data - read_commitment_tree::< - zcash_primitives::sapling::Node, - _, - { zcash_primitives::sapling::NOTE_COMMITMENT_TREE_DEPTH }, - >(Cursor::new(sapling_tree))? - .size() - .try_into() - .expect("usize values are convertible to u64 on all supported platforms.") - }); - - Ok((max_height, commitment_tree_meta)) + .and_then(|(max_height, sapling_tree_size, sapling_tree)| { + sapling_tree_size + .map(|s| Ok(CommitmentTreeMeta::from_parts(s))) + .or_else(|| { + if &sapling_tree == BLOCK_SAPLING_FRONTIER_ABSENT { + None + } else { + // parse the legacy commitment tree data + read_commitment_tree::< + zcash_primitives::sapling::Node, + _, + { zcash_primitives::sapling::NOTE_COMMITMENT_TREE_DEPTH }, + >(Cursor::new(sapling_tree)) + .map(|tree| { + Some(CommitmentTreeMeta::from_parts( + tree.size().try_into().unwrap(), + )) + }) + .map_err(SqliteClientError::from) + .transpose() + } + }) + .map(|meta_res| meta_res.map(|meta| (max_height, meta))) }) .transpose() } From e225a54d2e882b7a8ed54e894797ab786adab28f Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Fri, 30 Jun 2023 12:37:41 -0600 Subject: [PATCH 21/27] Use `NonZeroU32` for all `min_confirmations` values. --- zcash_client_backend/CHANGELOG.md | 6 ++ zcash_client_backend/src/data_api.rs | 5 +- zcash_client_backend/src/data_api/wallet.rs | 45 ++++---------- .../src/data_api/wallet/input_selection.rs | 9 +-- zcash_client_sqlite/src/chain.rs | 4 +- zcash_client_sqlite/src/wallet.rs | 11 +++- zcash_client_sqlite/src/wallet/sapling.rs | 59 ++++++++++++------- 7 files changed, 78 insertions(+), 61 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 0110fa88b..fa7d27cc9 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -30,6 +30,7 @@ and this library adheres to Rust's notion of wallet states where the full note plaintext is not available. - `WalletRead::get_nullifiers` has been renamed to `WalletRead::get_sapling_nullifiers` and its signature has changed; it now subsumes the removed `WalletRead::get_all_nullifiers`. + - `WalletRead::get_target_and_anchor_heights` now takes its argument as a `NonZeroU32` - `chain::scan_cached_blocks` now takes a `from_height` argument that permits the caller to control the starting position of the scan range. - A new `CommitmentTree` variant has been added to `data_api::error::Error` @@ -38,6 +39,11 @@ and this library adheres to Rust's notion of implemented for the type passed to them for the `wallet_db` parameter. - `data_api::wallet::create_proposed_transaction` now takes an additional `min_confirmations` argument. + - `data_api::wallet::{spend, create_spend_to_address, shield_transparent_funds, + propose_transfer, propose_shielding, create_proposed_transaction}` now take their + respective `min_confirmations` arguments as `NonZeroU32` + - `data_api::wallet::input_selection::InputSelector::{propose_transaction, propose_shielding}` + now take their respective `min_confirmations` arguments as `NonZeroU32` - A new `Sync` variant has been added to `data_api::chain::error::Error`. - A new `SyncRequired` variant has been added to `data_api::wallet::input_selection::InputSelectorError`. - `zcash_client_backend::wallet`: diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index d1031e570..5b199b422 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use std::fmt::Debug; +use std::num::NonZeroU32; use std::{cmp, ops::Range}; use incrementalmerkletree::Retention; @@ -97,7 +98,7 @@ pub trait WalletRead { /// This will return `Ok(None)` if no block data is present in the database. fn get_target_and_anchor_heights( &self, - min_confirmations: u32, + min_confirmations: NonZeroU32, ) -> Result, Self::Error> { self.block_height_extrema().map(|heights| { heights.map(|(min_height, max_height)| { @@ -106,7 +107,7 @@ pub trait WalletRead { // Select an anchor min_confirmations back from the target block, // unless that would be before the earliest block we have. let anchor_height = BlockHeight::from(cmp::max( - u32::from(target_height).saturating_sub(min_confirmations), + u32::from(target_height).saturating_sub(min_confirmations.into()), u32::from(min_height), )); diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index a47528920..c423c02ea 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -1,4 +1,4 @@ -use std::convert::Infallible; +use std::{convert::Infallible, num::NonZeroU32}; use std::fmt::Debug; use shardtree::{ShardStore, ShardTree, ShardTreeError}; @@ -122,10 +122,6 @@ where /// spent. A value of 10 confirmations is recommended and 0-conf transactions are /// not supported. /// -/// # Panics -/// -/// Panics if `min_confirmations == 0`; 0-conf transactions are not supported. -/// /// # Examples /// /// ``` @@ -203,7 +199,7 @@ pub fn create_spend_to_address( amount: Amount, memo: Option, ovk_policy: OvkPolicy, - min_confirmations: u32, + min_confirmations: NonZeroU32, ) -> Result< DbT::TxRef, Error< @@ -292,7 +288,8 @@ where /// can allow the sender to view the resulting notes on the blockchain. /// * `min_confirmations`: The minimum number of confirmations that a previously /// received note must have in the blockchain in order to be considered for being -/// spent. A value of 10 confirmations is recommended. +/// spent. A value of 10 confirmations is recommended and 0-conf transactions are +/// not supported. /// /// [`sapling::TxProver`]: zcash_primitives::sapling::prover::TxProver #[allow(clippy::too_many_arguments)] @@ -305,7 +302,7 @@ pub fn spend( usk: &UnifiedSpendingKey, request: zip321::TransactionRequest, ovk_policy: OvkPolicy, - min_confirmations: u32, + min_confirmations: NonZeroU32, ) -> Result< DbT::TxRef, Error< @@ -323,10 +320,6 @@ where ParamsT: consensus::Parameters + Clone, InputsT: InputSelector, { - assert!( - min_confirmations > 0, - "zero-conf transactions are not supported" - ); let account = wallet_db .get_account_for_ufvk(&usk.to_unified_full_viewing_key()) .map_err(Error::DataSource)? @@ -364,7 +357,7 @@ pub fn propose_transfer( spend_from_account: AccountId, input_selector: &InputsT, request: zip321::TransactionRequest, - min_confirmations: u32, + min_confirmations: NonZeroU32, ) -> Result< Proposal, Error< @@ -381,10 +374,6 @@ where ParamsT: consensus::Parameters + Clone, InputsT: InputSelector, { - assert!( - min_confirmations > 0, - "zero-conf transactions are not supported" - ); input_selector .propose_transaction( params, @@ -405,7 +394,7 @@ pub fn propose_shielding( input_selector: &InputsT, shielding_threshold: NonNegativeAmount, from_addrs: &[TransparentAddress], - min_confirmations: u32, + min_confirmations: NonZeroU32 ) -> Result< Proposal, Error< @@ -422,10 +411,6 @@ where DbT::NoteRef: Copy + Eq + Ord, InputsT: InputSelector, { - assert!( - min_confirmations > 0, - "zero-conf transactions are not supported" - ); input_selector .propose_shielding( params, @@ -451,7 +436,7 @@ pub fn create_proposed_transaction( usk: &UnifiedSpendingKey, ovk_policy: OvkPolicy, proposal: Proposal, - min_confirmations: u32, + min_confirmations: NonZeroU32, change_memo: Option, ) -> Result< DbT::TxRef, @@ -470,10 +455,6 @@ where ParamsT: consensus::Parameters + Clone, FeeRuleT: FeeRule, { - assert!( - min_confirmations > 0, - "zero-conf transactions are not supported" - ); let account = wallet_db .get_account_for_ufvk(&usk.to_unified_full_viewing_key()) .map_err(Error::DataSource)? @@ -516,8 +497,7 @@ where selected, usk.sapling(), &dfvk, - usize::try_from(min_confirmations - 1) - .expect("min_confirmations should never be anywhere close to usize::MAX"), + usize::try_from(u32::from(min_confirmations) - 1).unwrap() )? .ok_or(Error::NoteMismatch(selected.note_id))?; @@ -711,8 +691,9 @@ where /// to the wallet that the wallet can use to improve how it represents those /// shielding transactions to the user. /// * `min_confirmations`: The minimum number of confirmations that a previously -/// received UTXO must have in the blockchain in order to be considered for being -/// spent. +/// received note must have in the blockchain in order to be considered for being +/// spent. A value of 10 confirmations is recommended and 0-conf transactions are +/// not supported. /// /// [`sapling::TxProver`]: zcash_primitives::sapling::prover::TxProver #[cfg(feature = "transparent-inputs")] @@ -727,7 +708,7 @@ pub fn shield_transparent_funds( usk: &UnifiedSpendingKey, from_addrs: &[TransparentAddress], memo: &MemoBytes, - min_confirmations: u32, + min_confirmations: NonZeroU32, ) -> Result< DbT::TxRef, Error< diff --git a/zcash_client_backend/src/data_api/wallet/input_selection.rs b/zcash_client_backend/src/data_api/wallet/input_selection.rs index 90c701336..578082a33 100644 --- a/zcash_client_backend/src/data_api/wallet/input_selection.rs +++ b/zcash_client_backend/src/data_api/wallet/input_selection.rs @@ -2,6 +2,7 @@ use core::marker::PhantomData; use std::fmt; +use std::num::NonZeroU32; use std::{collections::BTreeSet, fmt::Debug}; use zcash_primitives::{ @@ -180,7 +181,7 @@ pub trait InputSelector { wallet_db: &Self::DataSource, account: AccountId, transaction_request: TransactionRequest, - min_confirmations: u32, + min_confirmations: NonZeroU32, ) -> Result< Proposal::DataSource as WalletRead>::NoteRef>, InputSelectorError<<::DataSource as WalletRead>::Error, Self::Error>, @@ -204,7 +205,7 @@ pub trait InputSelector { wallet_db: &Self::DataSource, shielding_threshold: NonNegativeAmount, source_addrs: &[TransparentAddress], - min_confirmations: u32, + min_confirmations: NonZeroU32, ) -> Result< Proposal::DataSource as WalletRead>::NoteRef>, InputSelectorError<<::DataSource as WalletRead>::Error, Self::Error>, @@ -324,7 +325,7 @@ where wallet_db: &Self::DataSource, account: AccountId, transaction_request: TransactionRequest, - min_confirmations: u32, + min_confirmations: NonZeroU32, ) -> Result, InputSelectorError> where ParamsT: consensus::Parameters, @@ -442,7 +443,7 @@ where wallet_db: &Self::DataSource, shielding_threshold: NonNegativeAmount, source_addrs: &[TransparentAddress], - min_confirmations: u32, + min_confirmations: NonZeroU32, ) -> Result, InputSelectorError> where ParamsT: consensus::Parameters, diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index f5300de90..191b2a6b0 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -265,6 +265,8 @@ where #[cfg(test)] #[allow(deprecated)] mod tests { + use std::num::NonZeroU32; + use secrecy::Secret; use tempfile::NamedTempFile; @@ -681,7 +683,7 @@ mod tests { &usk, req, OvkPolicy::Sender, - 1, + NonZeroU32::new(1).unwrap(), ), Ok(_) ); diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 7847c880b..722153dbf 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -544,6 +544,8 @@ pub(crate) fn block_height_extrema( pub(crate) fn fully_scanned_height( conn: &rusqlite::Connection, ) -> Result, SqliteClientError> { + // FIXME: this will need to be rewritten once out-of-order scan range suggestion + // is implemented. let res_opt = conn .query_row( "SELECT height, sapling_commitment_tree_size, sapling_tree @@ -1155,6 +1157,8 @@ pub(crate) fn put_sent_output( #[cfg(test)] mod tests { + use std::num::NonZeroU32; + use secrecy::Secret; use tempfile::NamedTempFile; @@ -1197,7 +1201,12 @@ mod tests { ); // We can't get an anchor height, as we have not scanned any blocks. - assert_eq!(db_data.get_target_and_anchor_heights(10).unwrap(), None); + assert_eq!( + db_data + .get_target_and_anchor_heights(NonZeroU32::new(10).unwrap()) + .unwrap(), + None + ); // An invalid account has zero balance assert_matches!( diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 6544b166e..85e101c73 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -367,6 +367,8 @@ pub(crate) fn put_received_note( #[cfg(test)] #[allow(deprecated)] pub(crate) mod tests { + use std::num::NonZeroU32; + use rusqlite::Connection; use secrecy::Secret; use tempfile::NamedTempFile; @@ -461,7 +463,7 @@ pub(crate) mod tests { Amount::from_u64(1).unwrap(), None, OvkPolicy::Sender, - 1, + NonZeroU32::new(1).unwrap(), ), Err(data_api::error::Error::KeyNotRecognized) ); @@ -490,7 +492,7 @@ pub(crate) mod tests { Amount::from_u64(1).unwrap(), None, OvkPolicy::Sender, - 1, + NonZeroU32::new(1).unwrap(), ), Err(data_api::error::Error::ScanRequired) ); @@ -533,7 +535,7 @@ pub(crate) mod tests { Amount::from_u64(1).unwrap(), None, OvkPolicy::Sender, - 1, + NonZeroU32::new(1).unwrap(), ), Err(data_api::error::Error::InsufficientFunds { available, @@ -572,7 +574,10 @@ pub(crate) mod tests { scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Verified balance matches total balance - let (_, anchor_height) = db_data.get_target_and_anchor_heights(10).unwrap().unwrap(); + let (_, anchor_height) = db_data + .get_target_and_anchor_heights(NonZeroU32::new(10).unwrap()) + .unwrap() + .unwrap(); assert_eq!( get_balance(&db_data.conn, AccountId::from(0)).unwrap(), value @@ -595,7 +600,10 @@ pub(crate) mod tests { scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Verified balance does not include the second note - let (_, anchor_height2) = db_data.get_target_and_anchor_heights(10).unwrap().unwrap(); + let (_, anchor_height2) = db_data + .get_target_and_anchor_heights(NonZeroU32::new(10).unwrap()) + .unwrap() + .unwrap(); assert_eq!( get_balance(&db_data.conn, AccountId::from(0)).unwrap(), (value + value).unwrap() @@ -618,7 +626,7 @@ pub(crate) mod tests { Amount::from_u64(70000).unwrap(), None, OvkPolicy::Sender, - 10, + NonZeroU32::new(10).unwrap(), ), Err(data_api::error::Error::InsufficientFunds { available, @@ -654,7 +662,7 @@ pub(crate) mod tests { Amount::from_u64(70000).unwrap(), None, OvkPolicy::Sender, - 10, + NonZeroU32::new(10).unwrap(), ), Err(data_api::error::Error::InsufficientFunds { available, @@ -687,7 +695,7 @@ pub(crate) mod tests { Amount::from_u64(70000).unwrap(), None, OvkPolicy::Sender, - 10, + NonZeroU32::new(10).unwrap(), ), Ok(_) ); @@ -738,7 +746,7 @@ pub(crate) mod tests { Amount::from_u64(15000).unwrap(), None, OvkPolicy::Sender, - 1, + NonZeroU32::new(1).unwrap(), ), Ok(_) ); @@ -754,7 +762,7 @@ pub(crate) mod tests { Amount::from_u64(2000).unwrap(), None, OvkPolicy::Sender, - 1, + NonZeroU32::new(1).unwrap(), ), Err(data_api::error::Error::InsufficientFunds { available, @@ -789,7 +797,7 @@ pub(crate) mod tests { Amount::from_u64(2000).unwrap(), None, OvkPolicy::Sender, - 1, + NonZeroU32::new(1).unwrap(), ), Err(data_api::error::Error::InsufficientFunds { available, @@ -820,7 +828,7 @@ pub(crate) mod tests { Amount::from_u64(2000).unwrap(), None, OvkPolicy::Sender, - 1, + NonZeroU32::new(1).unwrap(), ) .unwrap(); } @@ -872,7 +880,7 @@ pub(crate) mod tests { Amount::from_u64(15000).unwrap(), None, ovk_policy, - 1, + NonZeroU32::new(1).unwrap(), ) .unwrap(); @@ -960,7 +968,10 @@ pub(crate) mod tests { scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Verified balance matches total balance - let (_, anchor_height) = db_data.get_target_and_anchor_heights(1).unwrap().unwrap(); + let (_, anchor_height) = db_data + .get_target_and_anchor_heights(NonZeroU32::new(1).unwrap()) + .unwrap() + .unwrap(); assert_eq!( get_balance(&db_data.conn, AccountId::from(0)).unwrap(), value @@ -981,7 +992,7 @@ pub(crate) mod tests { Amount::from_u64(50000).unwrap(), None, OvkPolicy::Sender, - 1, + NonZeroU32::new(1).unwrap(), ), Ok(_) ); @@ -1016,7 +1027,10 @@ pub(crate) mod tests { scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); // Verified balance matches total balance - let (_, anchor_height) = db_data.get_target_and_anchor_heights(10).unwrap().unwrap(); + let (_, anchor_height) = db_data + .get_target_and_anchor_heights(NonZeroU32::new(10).unwrap()) + .unwrap() + .unwrap(); assert_eq!( get_balance(&db_data.conn, AccountId::from(0)).unwrap(), value @@ -1037,7 +1051,7 @@ pub(crate) mod tests { Amount::from_u64(50000).unwrap(), None, OvkPolicy::Sender, - 1, + NonZeroU32::new(1).unwrap(), ), Ok(_) ); @@ -1086,7 +1100,10 @@ pub(crate) mod tests { // Verified balance matches total balance let total = Amount::from_u64(60000).unwrap(); - let (_, anchor_height) = db_data.get_target_and_anchor_heights(1).unwrap().unwrap(); + let (_, anchor_height) = db_data + .get_target_and_anchor_heights(NonZeroU32::new(1).unwrap()) + .unwrap() + .unwrap(); assert_eq!( get_balance(&db_data.conn, AccountId::from(0)).unwrap(), total @@ -1121,7 +1138,7 @@ pub(crate) mod tests { &usk, req, OvkPolicy::Sender, - 1, + NonZeroU32::new(1).unwrap(), ), Err(Error::InsufficientFunds { available, required }) if available == Amount::from_u64(51000).unwrap() @@ -1149,7 +1166,7 @@ pub(crate) mod tests { &usk, req, OvkPolicy::Sender, - 1, + NonZeroU32::new(1).unwrap(), ), Ok(_) ); @@ -1213,7 +1230,7 @@ pub(crate) mod tests { &usk, &[*taddr], &MemoBytes::empty(), - 1 + NonZeroU32::new(1).unwrap() ), Ok(_) ); From 77b638012b3704082364377f00f87aea1e45c74d Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Sat, 1 Jul 2023 17:58:01 -0600 Subject: [PATCH 22/27] Remove `zcash_client_backend::data_api::chain::validate_chain` Local chain validation will be performed internal to `scan_cached_blocks`, and as handling of chain reorgs will need to change to support out-of-order scanning, the `validate_chain` method will be superfluous. It is removed in advance of other changes in order to avoid updating it to reflect the forthcoming changes. --- zcash_client_backend/CHANGELOG.md | 2 + zcash_client_backend/src/data_api/chain.rs | 101 +-------------------- zcash_client_sqlite/src/chain.rs | 40 ++------ 3 files changed, 16 insertions(+), 127 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index fa7d27cc9..8a6fc7f36 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -75,6 +75,8 @@ and this library adheres to Rust's notion of feature flag, has been modified by the addition of a `sapling_tree` property. - `wallet::input_selection`: - `Proposal::target_height` (use `Proposal::min_target_height` instead). +- `zcash_client_backend::data_api::chain::validate_chain` TODO: document how + to handle validation given out-of-order blocks. - `zcash_client_backend::wallet::WalletSaplingOutput::{witness, witness_mut}` have been removed as individual incremental witnesses are no longer tracked on a per-note basis. The global note commitment tree for the wallet should be used diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index b09432198..a55470307 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -36,43 +36,12 @@ //! let mut db_data = testing::MockWalletDb::new(Network::TestNetwork); //! //! // 1) Download new CompactBlocks into block_source. -//! -//! // 2) Run the chain validator on the received blocks. //! // -//! // Given that we assume the server always gives us correct-at-the-time blocks, any -//! // errors are in the blocks we have previously cached or scanned. -//! let max_height_hash = db_data.get_max_height_hash().map_err(Error::Wallet)?; -//! if let Err(e) = validate_chain(&block_source, max_height_hash, None) { -//! match e { -//! Error::Chain(e) => { -//! // a) Pick a height to rewind to. -//! // -//! // This might be informed by some external chain reorg information, or -//! // heuristics such as the platform, available bandwidth, size of recent -//! // CompactBlocks, etc. -//! let rewind_height = e.at_height() - 10; -//! -//! // b) Rewind scanned block information. -//! db_data.truncate_to_height(rewind_height); -//! -//! // c) Delete cached blocks from rewind_height onwards. -//! // -//! // This does imply that assumed-valid blocks will be re-downloaded, but it -//! // is also possible that in the intervening time, a chain reorg has -//! // occurred that orphaned some of those blocks. -//! -//! // d) If there is some separate thread or service downloading -//! // CompactBlocks, tell it to go back and download from rewind_height -//! // onwards. -//! }, -//! e => { -//! // handle or return other errors -//! -//! } -//! } -//! } -//! -//! // 3) Scan (any remaining) cached blocks. +//! // 2) FIXME: Obtain necessary block metadata for continuity checking? +//! // +//! // 3) Scan cached blocks. +//! // +//! // FIXME: update documentation on how to detect when a rewind is required. //! // //! // At this point, the cache and scanned data are locally consistent (though not //! // necessarily consistent with the latest chain tip - this would be discovered the @@ -82,10 +51,7 @@ //! # } //! ``` -use std::convert::Infallible; - use zcash_primitives::{ - block::BlockHash, consensus::{self, BlockHeight}, sapling::{self, note_encryption::PreparedIncomingViewingKey}, zip32::Scope, @@ -143,63 +109,6 @@ pub trait BlockSource { F: FnMut(CompactBlock) -> Result<(), error::Error>; } -/// Checks that the scanned blocks in the data database, when combined with the recent -/// `CompactBlock`s in the block_source database, form a valid chain. -/// -/// This function is built on the core assumption that the information provided in the -/// block source is more likely to be accurate than the previously-scanned information. -/// This follows from the design (and trust) assumption that the `lightwalletd` server -/// provides accurate block information as of the time it was requested. -/// -/// Arguments: -/// - `block_source` Source of compact blocks -/// - `validate_from` Height & hash of last validated block; -/// - `limit` specified number of blocks that will be valididated. Callers providing -/// a `limit` argument are responsible of making subsequent calls to `validate_chain()` -/// to complete validating the remaining blocks stored on the `block_source`. If `none` -/// is provided, there will be no limit set to the validation and upper bound of the -/// validation range will be the latest height present in the `block_source`. -/// -/// Returns: -/// - `Ok(())` if the combined chain is valid up to the given height -/// and block hash. -/// - `Err(Error::Chain(cause))` if the combined chain is invalid. -/// - `Err(e)` if there was an error during validation unrelated to chain validity. -pub fn validate_chain( - block_source: &BlockSourceT, - mut validate_from: Option<(BlockHeight, BlockHash)>, - limit: Option, -) -> Result<(), Error> -where - BlockSourceT: BlockSource, -{ - // The block source will contain blocks above the `validate_from` height. Validate from that - // maximum height up to the chain tip, returning the hash of the block found in the block - // source at the `validate_from` height, which can then be used to verify chain integrity by - // comparing against the `validate_from` hash. - - block_source.with_blocks::<_, Infallible, Infallible>( - validate_from.map(|(h, _)| h + 1), - limit, - move |block| { - if let Some((valid_height, valid_hash)) = validate_from { - if block.height() != valid_height + 1 { - return Err(ChainError::block_height_discontinuity( - valid_height + 1, - block.height(), - ) - .into()); - } else if block.prev_hash() != valid_hash { - return Err(ChainError::prev_hash_mismatch(block.height()).into()); - } - } - - validate_from = Some((block.height(), block.hash())); - Ok(()) - }, - ) -} - /// Scans at most `limit` new blocks added to the block source for any transactions received by the /// tracked accounts. /// diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index 191b2a6b0..4d968bb36 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -279,7 +279,7 @@ mod tests { use zcash_client_backend::{ address::RecipientAddress, data_api::{ - chain::{error::Error, scan_cached_blocks, validate_chain}, + chain::scan_cached_blocks, wallet::{input_selection::GreedyInputSelector, spend}, WalletRead, WalletWrite, }, @@ -329,21 +329,9 @@ mod tests { insert_into_cache(&db_cache, &cb); - // Cache-only chain should be valid - let validate_chain_result = validate_chain( - &db_cache, - Some((fake_block_height, fake_block_hash)), - Some(1), - ); - - assert_matches!(validate_chain_result, Ok(())); - // Scan the cache scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); - // Data-only chain should be valid - validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None).unwrap(); - // Create a second fake CompactBlock sending more value to the address let (cb2, _) = fake_compact_block( sapling_activation_height() + 1, @@ -355,14 +343,8 @@ mod tests { ); insert_into_cache(&db_cache, &cb2); - // Data+cache chain should be valid - validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None).unwrap(); - // Scan the cache again scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); - - // Data-only chain should be valid - validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None).unwrap(); } #[test] @@ -401,9 +383,6 @@ mod tests { // Scan the cache scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); - // Data-only chain should be valid - validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None).unwrap(); - // Create more fake CompactBlocks that don't connect to the scanned ones let (cb3, _) = fake_compact_block( sapling_activation_height() + 2, @@ -425,9 +404,10 @@ mod tests { insert_into_cache(&db_cache, &cb4); // Data+cache chain should be invalid at the data/cache boundary - let val_result = validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None); - - assert_matches!(val_result, Err(Error::Chain(e)) if e.at_height() == sapling_activation_height() + 2); + assert_matches!( + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None), + Err(_) // FIXME: check error result more closely + ); } #[test] @@ -466,9 +446,6 @@ mod tests { // Scan the cache scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); - // Data-only chain should be valid - validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None).unwrap(); - // Create more fake CompactBlocks that contain a reorg let (cb3, _) = fake_compact_block( sapling_activation_height() + 2, @@ -490,9 +467,10 @@ mod tests { insert_into_cache(&db_cache, &cb4); // Data+cache chain should be invalid inside the cache - let val_result = validate_chain(&db_cache, db_data.get_max_height_hash().unwrap(), None); - - assert_matches!(val_result, Err(Error::Chain(e)) if e.at_height() == sapling_activation_height() + 3); + assert_matches!( + scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None), + Err(_) // FIXME: check error result more closely + ); } #[test] From e3aafdad19d0097819f0c6f5925e4e063d3fd474 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Sat, 1 Jul 2023 18:16:23 -0600 Subject: [PATCH 23/27] Move chain continuity checks into `scan_block_with_runner` In preparation for out-of-order range-based scanning, it is necessary to ensure that the size of the Sapling note commitment tree is carried along through the scan process and that stored blocks are always persisted with the updated note commitment tree size. --- zcash_client_backend/CHANGELOG.md | 25 +-- zcash_client_backend/src/data_api.rs | 121 +++++++++++-- zcash_client_backend/src/data_api/chain.rs | 144 ++++++---------- .../src/data_api/chain/error.rs | 156 ++--------------- zcash_client_backend/src/data_api/wallet.rs | 6 +- zcash_client_backend/src/lib.rs | 2 +- zcash_client_backend/src/welding_rig.rs | 162 ++++++++++++------ zcash_client_sqlite/CHANGELOG.md | 6 + zcash_client_sqlite/src/chain.rs | 16 +- zcash_client_sqlite/src/error.rs | 13 +- zcash_client_sqlite/src/lib.rs | 99 +++++------ zcash_client_sqlite/src/wallet.rs | 135 +++++++++++---- zcash_client_sqlite/src/wallet/init.rs | 49 +++++- zcash_client_sqlite/src/wallet/sapling.rs | 45 +++-- zcash_primitives/src/block.rs | 18 +- 15 files changed, 544 insertions(+), 453 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 8a6fc7f36..5df41eb40 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -11,14 +11,16 @@ and this library adheres to Rust's notion of - `impl Eq for zcash_client_backend::zip321::{Payment, TransactionRequest}` - `impl Debug` for `zcash_client_backend::{data_api::wallet::input_selection::Proposal, wallet::ReceivedSaplingNote}` - `zcash_client_backend::data_api`: - - `WalletRead::{fully_scanned_height, suggest_scan_ranges}` + - `WalletRead::{block_metadata, block_fully_scanned, suggest_scan_ranges}` - `WalletWrite::put_block` - `WalletCommitmentTrees` - `testing::MockWalletDb::new` - `NullifierQuery` for use with `WalletRead::get_sapling_nullifiers` + - `BlockMetadata` + - `ScannedBlock` - `wallet::input_sellection::Proposal::{min_target_height, min_anchor_height}`: - `zcash_client_backend::wallet::WalletSaplingOutput::note_commitment_tree_position` -- `zcash_client_backend::welding_rig::SyncError` +- `zcash_client_backend::welding_rig::ScanError` ### Changed - MSRV is now 1.65.0. @@ -56,27 +58,28 @@ and this library adheres to Rust's notion of - `zcash_client_backend::welding_rig::ScanningKey::sapling_nf` has been changed to take a note position instead of an incremental witness for the note. - Arguments to `zcash_client_backend::welding_rig::scan_block` have changed. This - method now takes an optional `CommitmentTreeMeta` argument instead of a base commitment - tree and incremental witnesses for each previously-known note. + method now takes an optional `BlockMetadata` argument instead of a base commitment + tree and incremental witnesses for each previously-known note. In addition, the + return type has now been updated to return a `Result` + in the case of scan failure. ### Removed - `zcash_client_backend::data_api`: - `WalletRead::get_all_nullifiers` - - `WalletRead::{get_commitment_tree, get_witnesses}` (use - `WalletRead::fully_scanned_height` instead). + - `WalletRead::{get_commitment_tree, get_witnesses}` have been removed + without replacement. The utility of these methods is now subsumed + by those available from the `WalletCommitmentTrees` trait. - `WalletWrite::advance_by_block` (use `WalletWrite::put_block` instead). - - The `commitment_tree` and `transactions` properties of the `PrunedBlock` - struct are now owned values instead of references, making this a fully owned type. - It is now parameterized by the nullifier type instead of by a lifetime for the - fields that were previously references. In addition, two new properties, - `sapling_commitment_tree_size` and `sapling_commitments` have been added. + - `PrunedBlock` has been replaced by `ScannedBlock` - `testing::MockWalletDb`, which is available under the `test-dependencies` feature flag, has been modified by the addition of a `sapling_tree` property. - `wallet::input_selection`: - `Proposal::target_height` (use `Proposal::min_target_height` instead). - `zcash_client_backend::data_api::chain::validate_chain` TODO: document how to handle validation given out-of-order blocks. +- `zcash_client_backend::data_api::chain::error::{ChainError, Cause}` have been + replaced by `zcash_client_backend::welding_rig::ScanError` - `zcash_client_backend::wallet::WalletSaplingOutput::{witness, witness_mut}` have been removed as individual incremental witnesses are no longer tracked on a per-note basis. The global note commitment tree for the wallet should be used diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 5b199b422..6c5f8a115 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -66,15 +66,17 @@ pub trait WalletRead { /// This will return `Ok(None)` if no block data is present in the database. fn block_height_extrema(&self) -> Result, Self::Error>; - /// Returns the height to which the wallet has been fully scanned. + /// Returns the available block metadata for the block at the specified height, if any. + fn block_metadata(&self, height: BlockHeight) -> Result, Self::Error>; + + /// Returns the metadata for the block at the height to which the wallet has been fully + /// scanned. /// /// This is the height for which the wallet has fully trial-decrypted this and all preceding /// blocks above the wallet's birthday height. Along with this height, this method returns /// metadata describing the state of the wallet's note commitment trees as of the end of that /// block. - fn fully_scanned_height( - &self, - ) -> Result, Self::Error>; + fn block_fully_scanned(&self) -> Result, Self::Error>; /// Returns a vector of suggested scan ranges based upon the current wallet state. /// @@ -248,17 +250,99 @@ pub trait WalletRead { ) -> Result, Self::Error>; } +/// Metadata describing the sizes of the zcash note commitment trees as of a particular block. +#[derive(Debug, Clone, Copy)] +pub struct BlockMetadata { + block_height: BlockHeight, + block_hash: BlockHash, + sapling_tree_size: u32, + //TODO: orchard_tree_size: u32 +} + +impl BlockMetadata { + /// Constructs a new [`BlockMetadata`] value from its constituent parts. + pub fn from_parts( + block_height: BlockHeight, + block_hash: BlockHash, + sapling_tree_size: u32, + ) -> Self { + Self { + block_height, + block_hash, + sapling_tree_size, + } + } + + /// Returns the block height. + pub fn block_height(&self) -> BlockHeight { + self.block_height + } + + /// Returns the hash of the block + pub fn block_hash(&self) -> BlockHash { + self.block_hash + } + + /// Returns the size of the Sapling note commitment tree as of the block that this + /// [`BlockMetadata`] describes. + pub fn sapling_tree_size(&self) -> u32 { + self.sapling_tree_size + } +} + /// The subset of information that is relevant to this wallet that has been /// decrypted and extracted from a [`CompactBlock`]. /// /// [`CompactBlock`]: crate::proto::compact_formats::CompactBlock -pub struct PrunedBlock { - pub block_height: BlockHeight, - pub block_hash: BlockHash, - pub block_time: u32, - pub transactions: Vec>, - pub sapling_commitment_tree_size: Option, - pub sapling_commitments: Vec<(sapling::Node, Retention)>, +pub struct ScannedBlock { + metadata: BlockMetadata, + block_time: u32, + transactions: Vec>, + sapling_commitments: Vec<(sapling::Node, Retention)>, +} + +impl ScannedBlock { + pub fn from_parts( + metadata: BlockMetadata, + block_time: u32, + transactions: Vec>, + sapling_commitments: Vec<(sapling::Node, Retention)>, + ) -> Self { + Self { + metadata, + block_time, + transactions, + sapling_commitments, + } + } + + pub fn height(&self) -> BlockHeight { + self.metadata.block_height + } + + pub fn block_hash(&self) -> BlockHash { + self.metadata.block_hash + } + + pub fn block_time(&self) -> u32 { + self.block_time + } + + pub fn metadata(&self) -> &BlockMetadata { + &self.metadata + } + + pub fn transactions(&self) -> &[WalletTx] { + &self.transactions + } + + pub fn sapling_commitments(&self) -> &[(sapling::Node, Retention)] { + &self.sapling_commitments + } + + pub fn take_sapling_commitments(self) -> Vec<(sapling::Node, Retention)> { + self.sapling_commitments + } } /// A transaction that was detected during scanning of the blockchain, @@ -404,7 +488,7 @@ pub trait WalletWrite: WalletRead { #[allow(clippy::type_complexity)] fn put_block( &mut self, - block: PrunedBlock, + block: ScannedBlock, ) -> Result, Self::Error>; /// Caches a decrypted transaction in the persistent wallet store. @@ -489,7 +573,7 @@ pub mod testing { }; use super::{ - chain, DecryptedTransaction, NullifierQuery, PrunedBlock, SentTransaction, + BlockMetadata, DecryptedTransaction, NullifierQuery, ScannedBlock, SentTransaction, WalletCommitmentTrees, WalletRead, WalletWrite, SAPLING_SHARD_HEIGHT, }; @@ -520,9 +604,14 @@ pub mod testing { Ok(None) } - fn fully_scanned_height( + fn block_metadata( &self, - ) -> Result, Self::Error> { + _height: BlockHeight, + ) -> Result, Self::Error> { + Ok(None) + } + + fn block_fully_scanned(&self) -> Result, Self::Error> { Ok(None) } @@ -667,7 +756,7 @@ pub mod testing { #[allow(clippy::type_complexity)] fn put_block( &mut self, - _block: PrunedBlock, + _block: ScannedBlock, ) -> Result, Self::Error> { Ok(vec![]) } diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index a55470307..8e1ceb864 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -17,7 +17,6 @@ //! BlockSource, //! error::Error, //! scan_cached_blocks, -//! validate_chain, //! testing as chain_testing, //! }, //! testing, @@ -30,7 +29,7 @@ //! # test(); //! # } //! # -//! # fn test() -> Result<(), Error<(), Infallible, u32>> { +//! # fn test() -> Result<(), Error<(), Infallible>> { //! let network = Network::TestNetwork; //! let block_source = chain_testing::MockBlockSource; //! let mut db_data = testing::MockWalletDb::new(Network::TestNetwork); @@ -38,7 +37,7 @@ //! // 1) Download new CompactBlocks into block_source. //! // //! // 2) FIXME: Obtain necessary block metadata for continuity checking? -//! // +//! // //! // 3) Scan cached blocks. //! // //! // FIXME: update documentation on how to detect when a rewind is required. @@ -65,26 +64,7 @@ use crate::{ }; pub mod error; -use error::{ChainError, Error}; - -/// Metadata describing the sizes of the zcash note commitment trees as of a particular block. -pub struct CommitmentTreeMeta { - sapling_tree_size: u32, - //TODO: orchard_tree_size: u32 -} - -impl CommitmentTreeMeta { - /// Constructs a new [`CommitmentTreeMeta`] value from its constituent parts. - pub fn from_parts(sapling_tree_size: u32) -> Self { - Self { sapling_tree_size } - } - - /// Returns the size of the Sapling note commitment tree as of the block that this - /// [`CommitmentTreeMeta`] describes. - pub fn sapling_tree_size(&self) -> u32 { - self.sapling_tree_size - } -} +use error::Error; /// This trait provides sequential access to raw blockchain data via a callback-oriented /// API. @@ -99,14 +79,14 @@ pub trait BlockSource { /// as part of processing each row. /// * `NoteRefT`: the type of note identifiers in the wallet data store, for use in /// reporting errors related to specific notes. - fn with_blocks( + fn with_blocks( &self, from_height: Option, limit: Option, with_row: F, - ) -> Result<(), error::Error> + ) -> Result<(), error::Error> where - F: FnMut(CompactBlock) -> Result<(), error::Error>; + F: FnMut(CompactBlock) -> Result<(), error::Error>; } /// Scans at most `limit` new blocks added to the block source for any transactions received by the @@ -133,7 +113,7 @@ pub fn scan_cached_blocks( data_db: &mut DbT, from_height: Option, limit: Option, -) -> Result<(), Error> +) -> Result<(), Error> where ParamsT: consensus::Parameters + Send + 'static, BlockSourceT: BlockSource, @@ -169,74 +149,60 @@ where ); // Start at either the provided height, or where we synced up to previously. - let (scan_from, commitment_tree_meta) = from_height.map_or_else( - || { - data_db.fully_scanned_height().map_or_else( - |e| Err(Error::Wallet(e)), - |last_scanned| { - Ok(last_scanned.map_or_else(|| (None, None), |(h, m)| (Some(h + 1), Some(m)))) + let (scan_from, mut prior_block_metadata) = match from_height { + Some(h) => { + // if we are provided with a starting height, obtain the metadata for the previous + // block (if any is available) + ( + Some(h), + if h > BlockHeight::from(0) { + data_db.block_metadata(h - 1).map_err(Error::Wallet)? + } else { + None }, ) - }, - |h| Ok((Some(h), None)), - )?; + } + None => { + let last_scanned = data_db.block_fully_scanned().map_err(Error::Wallet)?; + last_scanned.map_or_else(|| (None, None), |m| (Some(m.block_height + 1), Some(m))) + } + }; - block_source.with_blocks::<_, DbT::Error, DbT::NoteRef>( - scan_from, - limit, - |block: CompactBlock| { - add_block_to_runner(params, block, &mut batch_runner); - Ok(()) - }, - )?; + block_source.with_blocks::<_, DbT::Error>(scan_from, limit, |block: CompactBlock| { + add_block_to_runner(params, block, &mut batch_runner); + Ok(()) + })?; batch_runner.flush(); - let mut last_scanned = None; - block_source.with_blocks::<_, DbT::Error, DbT::NoteRef>( - scan_from, - limit, - |block: CompactBlock| { - // block heights should be sequential within a single scan range - let block_height = block.height(); - if let Some(h) = last_scanned { - if block_height != h + 1 { - return Err(Error::Chain(ChainError::block_height_discontinuity( - block.height(), - h, - ))); - } - } - - let pruned_block = scan_block_with_runner( - params, - block, - &dfvks, - &sapling_nullifiers, - commitment_tree_meta.as_ref(), - Some(&mut batch_runner), - ) - .map_err(Error::Sync)?; + block_source.with_blocks::<_, DbT::Error>(scan_from, limit, |block: CompactBlock| { + let scanned_block = scan_block_with_runner( + params, + block, + &dfvks, + &sapling_nullifiers, + prior_block_metadata.as_ref(), + Some(&mut batch_runner), + ) + .map_err(Error::Scan)?; + + let spent_nf: Vec<&sapling::Nullifier> = scanned_block + .transactions + .iter() + .flat_map(|tx| tx.sapling_spends.iter().map(|spend| spend.nf())) + .collect(); - let spent_nf: Vec<&sapling::Nullifier> = pruned_block - .transactions + sapling_nullifiers.retain(|(_, nf)| !spent_nf.contains(&nf)); + sapling_nullifiers.extend(scanned_block.transactions.iter().flat_map(|tx| { + tx.sapling_outputs .iter() - .flat_map(|tx| tx.sapling_spends.iter().map(|spend| spend.nf())) - .collect(); - - sapling_nullifiers.retain(|(_, nf)| !spent_nf.contains(&nf)); - sapling_nullifiers.extend(pruned_block.transactions.iter().flat_map(|tx| { - tx.sapling_outputs - .iter() - .map(|out| (out.account(), *out.nf())) - })); + .map(|out| (out.account(), *out.nf())) + })); - data_db.put_block(pruned_block).map_err(Error::Wallet)?; - - last_scanned = Some(block_height); - Ok(()) - }, - )?; + prior_block_metadata = Some(*scanned_block.metadata()); + data_db.put_block(scanned_block).map_err(Error::Wallet)?; + Ok(()) + })?; Ok(()) } @@ -255,14 +221,14 @@ pub mod testing { impl BlockSource for MockBlockSource { type Error = Infallible; - fn with_blocks( + fn with_blocks( &self, _from_height: Option, _limit: Option, _with_row: F, - ) -> Result<(), Error> + ) -> Result<(), Error> where - F: FnMut(CompactBlock) -> Result<(), Error>, + F: FnMut(CompactBlock) -> Result<(), Error>, { Ok(()) } diff --git a/zcash_client_backend/src/data_api/chain/error.rs b/zcash_client_backend/src/data_api/chain/error.rs index b28380d13..c1c78cf61 100644 --- a/zcash_client_backend/src/data_api/chain/error.rs +++ b/zcash_client_backend/src/data_api/chain/error.rs @@ -3,136 +3,11 @@ use std::error; use std::fmt::{self, Debug, Display}; -use zcash_primitives::{consensus::BlockHeight, sapling, transaction::TxId}; - -use crate::welding_rig::SyncError; - -/// The underlying cause of a [`ChainError`]. -#[derive(Copy, Clone, Debug)] -pub enum Cause { - /// The hash of the parent block given by a proposed new chain tip does not match the hash of - /// the current chain tip. - PrevHashMismatch, - - /// The block height field of the proposed new chain tip is not equal to the height of the - /// previous chain tip + 1. This variant stores a copy of the incorrect height value for - /// reporting purposes. - BlockHeightDiscontinuity(BlockHeight), - - /// The root of an output's witness tree in a newly arrived transaction does not correspond to - /// root of the stored commitment tree at the recorded height. - /// - /// This error is currently only produced when performing the slow checks that are enabled by - /// compiling with `-C debug-assertions`. - InvalidNewWitnessAnchor { - /// The id of the transaction containing the mismatched witness. - txid: TxId, - /// The index of the shielded output within the transaction where the witness root does not - /// match. - index: usize, - /// The root of the witness that failed to match the root of the current note commitment - /// tree. - node: sapling::Node, - }, - - /// The root of an output's witness tree in a previously stored transaction does not correspond - /// to root of the current commitment tree. - /// - /// This error is currently only produced when performing the slow checks that are enabled by - /// compiling with `-C debug-assertions`. - InvalidWitnessAnchor(NoteRef), -} - -/// Errors that may occur in chain scanning or validation. -#[derive(Copy, Clone, Debug)] -pub struct ChainError { - at_height: BlockHeight, - cause: Cause, -} - -impl fmt::Display for ChainError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match &self.cause { - Cause::PrevHashMismatch => write!( - f, - "The parent hash of proposed block does not correspond to the block hash at height {}.", - self.at_height - ), - Cause::BlockHeightDiscontinuity(h) => { - write!(f, "Block height discontinuity at height {}; next height is : {}", self.at_height, h) - } - Cause::InvalidNewWitnessAnchor { txid, index, node } => write!( - f, - "New witness for output {} in tx {} at height {} has incorrect anchor: {:?}", - index, txid, self.at_height, node, - ), - Cause::InvalidWitnessAnchor(id_note) => { - write!(f, "Witness for note {} has incorrect anchor for height {}", id_note, self.at_height) - } - } - } -} - -impl ChainError { - /// Constructs an error that indicates block hashes failed to chain. - /// - /// * `at_height` the height of the block whose parent hash does not match the hash of the - /// previous block - pub fn prev_hash_mismatch(at_height: BlockHeight) -> Self { - ChainError { - at_height, - cause: Cause::PrevHashMismatch, - } - } - - /// Constructs an error that indicates a gap in block heights. - /// - /// * `at_height` the height of the block being added to the chain. - /// * `prev_chain_tip` the height of the previous chain tip. - pub fn block_height_discontinuity(at_height: BlockHeight, prev_chain_tip: BlockHeight) -> Self { - ChainError { - at_height, - cause: Cause::BlockHeightDiscontinuity(prev_chain_tip), - } - } - - /// Constructs an error that indicates a mismatch between an updated note's witness and the - /// root of the current note commitment tree. - pub fn invalid_witness_anchor(at_height: BlockHeight, note_ref: NoteRef) -> Self { - ChainError { - at_height, - cause: Cause::InvalidWitnessAnchor(note_ref), - } - } - - /// Constructs an error that indicates a mismatch between a new note's witness and the root of - /// the current note commitment tree. - pub fn invalid_new_witness_anchor( - at_height: BlockHeight, - txid: TxId, - index: usize, - node: sapling::Node, - ) -> Self { - ChainError { - at_height, - cause: Cause::InvalidNewWitnessAnchor { txid, index, node }, - } - } - - /// Returns the block height at which this error was discovered. - pub fn at_height(&self) -> BlockHeight { - self.at_height - } - - /// Returns the cause of this error. - pub fn cause(&self) -> &Cause { - &self.cause - } -} +use crate::welding_rig::ScanError; /// Errors related to chain validation and scanning. #[derive(Debug)] -pub enum Error { +pub enum Error { /// An error that was produced by wallet operations in the course of scanning the chain. Wallet(WalletError), @@ -143,13 +18,10 @@ pub enum Error { /// A block that was received violated rules related to chain continuity or contained note /// commitments that could not be reconciled with the note commitment tree(s) maintained by the /// wallet. - Chain(ChainError), - - /// An error occorred in block scanning. - Sync(SyncError), + Scan(ScanError), } -impl fmt::Display for Error { +impl fmt::Display for Error { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match &self { Error::Wallet(e) => { @@ -166,25 +38,17 @@ impl fmt::Display for Error { - write!(f, "{}", err) - } - Error::Sync(SyncError::SaplingTreeSizeUnknown(h)) => { - write!( - f, - "Sync failed due to missing Sapling note commitment tree size at height {}", - h - ) + Error::Scan(e) => { + write!(f, "Scanning produced the following error: {}", e) } } } } -impl error::Error for Error +impl error::Error for Error where WE: Debug + Display + error::Error + 'static, BE: Debug + Display + error::Error + 'static, - N: Debug + Display, { fn source(&self) -> Option<&(dyn error::Error + 'static)> { match &self { @@ -195,8 +59,8 @@ where } } -impl From> for Error { - fn from(e: ChainError) -> Self { - Error::Chain(e) +impl From for Error { + fn from(e: ScanError) -> Self { + Error::Scan(e) } } diff --git a/zcash_client_backend/src/data_api/wallet.rs b/zcash_client_backend/src/data_api/wallet.rs index c423c02ea..7f3d4ce6a 100644 --- a/zcash_client_backend/src/data_api/wallet.rs +++ b/zcash_client_backend/src/data_api/wallet.rs @@ -1,5 +1,5 @@ -use std::{convert::Infallible, num::NonZeroU32}; use std::fmt::Debug; +use std::{convert::Infallible, num::NonZeroU32}; use shardtree::{ShardStore, ShardTree, ShardTreeError}; use zcash_primitives::{ @@ -394,7 +394,7 @@ pub fn propose_shielding( input_selector: &InputsT, shielding_threshold: NonNegativeAmount, from_addrs: &[TransparentAddress], - min_confirmations: NonZeroU32 + min_confirmations: NonZeroU32, ) -> Result< Proposal, Error< @@ -497,7 +497,7 @@ where selected, usk.sapling(), &dfvk, - usize::try_from(u32::from(min_confirmations) - 1).unwrap() + usize::try_from(u32::from(min_confirmations) - 1).unwrap(), )? .ok_or(Error::NoteMismatch(selected.note_id))?; diff --git a/zcash_client_backend/src/lib.rs b/zcash_client_backend/src/lib.rs index f73711662..1cb87bc9f 100644 --- a/zcash_client_backend/src/lib.rs +++ b/zcash_client_backend/src/lib.rs @@ -16,8 +16,8 @@ pub mod fees; pub mod keys; pub mod proto; pub mod scan; +pub mod scanning; pub mod wallet; -pub mod welding_rig; pub mod zip321; pub use decrypt::{decrypt_transaction, DecryptedOutput, TransferType}; diff --git a/zcash_client_backend/src/welding_rig.rs b/zcash_client_backend/src/welding_rig.rs index 79e3a4c39..a5d8e9c5d 100644 --- a/zcash_client_backend/src/welding_rig.rs +++ b/zcash_client_backend/src/welding_rig.rs @@ -4,6 +4,7 @@ use std::collections::{HashMap, HashSet}; use std::convert::TryFrom; +use std::fmt::{self, Debug}; use incrementalmerkletree::{Position, Retention}; use subtle::{ConditionallySelectable, ConstantTimeEq, CtOption}; @@ -20,8 +21,7 @@ use zcash_primitives::{ zip32::{sapling::DiversifiableFullViewingKey, AccountId, Scope}, }; -use crate::data_api::chain::CommitmentTreeMeta; -use crate::data_api::PrunedBlock; +use crate::data_api::{BlockMetadata, ScannedBlock}; use crate::{ proto::compact_formats::CompactBlock, scan::{Batch, BatchRunner, Tasks}, @@ -109,12 +109,42 @@ impl ScanningKey for SaplingIvk { fn sapling_nf(_key: &Self::SaplingNk, _note: &sapling::Note, _position: Position) {} } -/// Errors that can occur in block scanning. -#[derive(Debug)] -pub enum SyncError { +/// Errors that may occur in chain scanning +#[derive(Copy, Clone, Debug)] +pub enum ScanError { + /// The hash of the parent block given by a proposed new chain tip does not match the hash of + /// the current chain tip. + PrevHashMismatch { at_height: BlockHeight }, + + /// The block height field of the proposed new chain tip is not equal to the height of the + /// previous chain tip + 1. This variant stores a copy of the incorrect height value for + /// reporting purposes. + BlockHeightDiscontinuity { + previous_tip: BlockHeight, + new_height: BlockHeight, + }, + /// The size of the Sapling note commitment tree was not provided as part of a [`CompactBlock`] /// being scanned, making it impossible to construct the nullifier for a detected note. - SaplingTreeSizeUnknown(BlockHeight), + SaplingTreeSizeUnknown { at_height: BlockHeight }, +} + +impl fmt::Display for ScanError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match &self { + ScanError::PrevHashMismatch { at_height } => write!( + f, + "The parent hash of proposed block does not correspond to the block hash at height {}.", + at_height + ), + ScanError::BlockHeightDiscontinuity { previous_tip, new_height } => { + write!(f, "Block height discontinuity at height {}; next height is : {}", previous_tip, new_height) + } + ScanError::SaplingTreeSizeUnknown { at_height } => { + write!(f, "Unable to determine Sapling note commitment tree size at height {}", at_height) + } + } + } } /// Scans a [`CompactBlock`] with a set of [`ScanningKey`]s. @@ -145,14 +175,14 @@ pub fn scan_block( block: CompactBlock, vks: &[(&AccountId, &K)], sapling_nullifiers: &[(AccountId, sapling::Nullifier)], - initial_commitment_tree_meta: Option<&CommitmentTreeMeta>, -) -> Result, SyncError> { + prior_block_metadata: Option<&BlockMetadata>, +) -> Result, ScanError> { scan_block_with_runner::<_, _, ()>( params, block, vks, sapling_nullifiers, - initial_commitment_tree_meta, + prior_block_metadata, None, ) } @@ -204,13 +234,28 @@ pub(crate) fn scan_block_with_runner< block: CompactBlock, vks: &[(&AccountId, &K)], nullifiers: &[(AccountId, sapling::Nullifier)], - initial_commitment_tree_meta: Option<&CommitmentTreeMeta>, + prior_block_metadata: Option<&BlockMetadata>, mut batch_runner: Option<&mut TaggedBatchRunner>, -) -> Result, SyncError> { +) -> Result, ScanError> { let mut wtxs: Vec> = vec![]; let mut sapling_note_commitments: Vec<(sapling::Node, Retention)> = vec![]; - let block_height = block.height(); - let block_hash = block.hash(); + let cur_height = block.height(); + let cur_hash = block.hash(); + + if let Some(prev) = prior_block_metadata { + if cur_height != prev.block_height() + 1 { + return Err(ScanError::BlockHeightDiscontinuity { + previous_tip: prev.block_height(), + new_height: cur_height, + }); + } + + if block.prev_hash() != prev.block_hash() { + return Err(ScanError::PrevHashMismatch { + at_height: cur_height, + }); + } + } // It's possible to make progress without a Sapling tree position if we don't have any Sapling // notes in the block, since we only use the position for constructing nullifiers for our own @@ -235,7 +280,10 @@ pub(crate) fn scan_block_with_runner< Some(m.sapling_commitment_tree_size - block_note_count) } }) - .or_else(|| initial_commitment_tree_meta.map(|m| m.sapling_tree_size())); + .or_else(|| prior_block_metadata.map(|m| m.sapling_tree_size())) + .ok_or(ScanError::SaplingTreeSizeUnknown { + at_height: cur_height, + })?; let block_tx_count = block.vtx.len(); for (tx_idx, tx) in block.vtx.into_iter().enumerate() { @@ -283,7 +331,7 @@ pub(crate) fn scan_block_with_runner< .into_iter() .map(|output| { ( - SaplingDomain::for_height(params.clone(), block_height), + SaplingDomain::for_height(params.clone(), cur_height), CompactOutputDescription::try_from(output) .expect("Invalid output found in compact block decoding."), ) @@ -300,7 +348,7 @@ pub(crate) fn scan_block_with_runner< }) .collect::>(); - let mut decrypted = runner.collect_results(block_hash, txid); + let mut decrypted = runner.collect_results(cur_hash, txid); (0..decoded.len()) .map(|i| { decrypted.remove(&(txid, i)).map(|d_note| { @@ -347,7 +395,7 @@ pub(crate) fn scan_block_with_runner< let is_checkpoint = output_idx + 1 == decoded.len() && tx_idx + 1 == block_tx_count; let retention = match (dec_output.is_some(), is_checkpoint) { (is_marked, true) => Retention::Checkpoint { - id: block_height, + id: cur_height, is_marked, }, (true, false) => Retention::Marked, @@ -362,9 +410,9 @@ pub(crate) fn scan_block_with_runner< // - Notes created by consolidation transactions. // - Notes sent from one account to itself. let is_change = spent_from_accounts.contains(&account); - let note_commitment_tree_position = sapling_commitment_tree_size - .map(|s| Position::from(u64::from(s + u32::try_from(output_idx).unwrap()))) - .ok_or(SyncError::SaplingTreeSizeUnknown(block_height))?; + let note_commitment_tree_position = Position::from(u64::from( + sapling_commitment_tree_size + u32::try_from(output_idx).unwrap(), + )); let nf = K::sapling_nf(&nk, ¬e, note_commitment_tree_position); shielded_outputs.push(WalletSaplingOutput::from_parts( @@ -392,17 +440,15 @@ pub(crate) fn scan_block_with_runner< }); } - sapling_commitment_tree_size = sapling_commitment_tree_size.map(|s| s + tx_outputs_len); + sapling_commitment_tree_size += tx_outputs_len; } - Ok(PrunedBlock { - block_height, - block_hash, - block_time: block.time, - transactions: wtxs, - sapling_commitment_tree_size, - sapling_commitments: sapling_note_commitments, - }) + Ok(ScannedBlock::from_parts( + BlockMetadata::from_parts(cur_height, cur_hash, sapling_commitment_tree_size), + block.time, + wtxs, + sapling_note_commitments, + )) } #[cfg(test)] @@ -415,6 +461,7 @@ mod tests { use rand_core::{OsRng, RngCore}; use zcash_note_encryption::Domain; use zcash_primitives::{ + block::BlockHash, consensus::{BlockHeight, Network}, constants::SPENDING_KEY_GENERATOR, memo::MemoBytes, @@ -430,9 +477,9 @@ mod tests { }; use crate::{ - data_api::chain::CommitmentTreeMeta, + data_api::BlockMetadata, proto::compact_formats::{ - BlockMetadata, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, + self as compact, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, }, scan::BatchRunner, }; @@ -479,6 +526,7 @@ mod tests { /// from a `lightwalletd` that is not currently tracking note commitment tree sizes. fn fake_compact_block( height: BlockHeight, + prev_hash: BlockHash, nf: Nullifier, dfvk: &DiversifiableFullViewingKey, value: Amount, @@ -510,6 +558,7 @@ mod tests { rng.fill_bytes(&mut hash); hash }, + prev_hash: prev_hash.0.to_vec(), height: height.into(), ..Default::default() }; @@ -543,7 +592,7 @@ mod tests { cb.vtx.push(tx); } - cb.block_metadata = initial_sapling_tree_size.map(|s| BlockMetadata { + cb.block_metadata = initial_sapling_tree_size.map(|s| compact::BlockMetadata { sapling_commitment_tree_size: s + cb .vtx .iter() @@ -564,6 +613,7 @@ mod tests { let cb = fake_compact_block( 1u32.into(), + BlockHash([0; 32]), Nullifier([0; 32]), &dfvk, Amount::from_u64(5).unwrap(), @@ -589,16 +639,20 @@ mod tests { None }; - let pruned_block = scan_block_with_runner( + let scanned_block = scan_block_with_runner( &Network::TestNetwork, cb, &[(&account, &dfvk)], &[], - Some(&CommitmentTreeMeta::from_parts(0)), + Some(&BlockMetadata::from_parts( + BlockHeight::from(0), + BlockHash([0u8; 32]), + 0, + )), batch_runner.as_mut(), ) .unwrap(); - let txs = pruned_block.transactions; + let txs = scanned_block.transactions(); assert_eq!(txs.len(), 1); let tx = &txs[0]; @@ -613,17 +667,17 @@ mod tests { Position::from(1) ); - assert_eq!(pruned_block.sapling_commitment_tree_size, Some(2)); + assert_eq!(scanned_block.metadata().sapling_tree_size(), 2); assert_eq!( - pruned_block - .sapling_commitments + scanned_block + .sapling_commitments() .iter() .map(|(_, retention)| *retention) .collect::>(), vec![ Retention::Ephemeral, Retention::Checkpoint { - id: pruned_block.block_height, + id: scanned_block.height(), is_marked: true } ] @@ -643,6 +697,7 @@ mod tests { let cb = fake_compact_block( 1u32.into(), + BlockHash([0; 32]), Nullifier([0; 32]), &dfvk, Amount::from_u64(5).unwrap(), @@ -668,7 +723,7 @@ mod tests { None }; - let pruned_block = scan_block_with_runner( + let scanned_block = scan_block_with_runner( &Network::TestNetwork, cb, &[(&AccountId::from(0), &dfvk)], @@ -677,7 +732,7 @@ mod tests { batch_runner.as_mut(), ) .unwrap(); - let txs = pruned_block.transactions; + let txs = scanned_block.transactions(); assert_eq!(txs.len(), 1); let tx = &txs[0]; @@ -689,8 +744,8 @@ mod tests { assert_eq!(tx.sapling_outputs[0].note().value().inner(), 5); assert_eq!( - pruned_block - .sapling_commitments + scanned_block + .sapling_commitments() .iter() .map(|(_, retention)| *retention) .collect::>(), @@ -698,7 +753,7 @@ mod tests { Retention::Ephemeral, Retention::Marked, Retention::Checkpoint { - id: pruned_block.block_height, + id: scanned_block.height(), is_marked: false } ] @@ -718,6 +773,7 @@ mod tests { let cb = fake_compact_block( 1u32.into(), + BlockHash([0; 32]), nf, &dfvk, Amount::from_u64(5).unwrap(), @@ -727,15 +783,9 @@ mod tests { assert_eq!(cb.vtx.len(), 2); let vks: Vec<(&AccountId, &SaplingIvk)> = vec![]; - let pruned_block = scan_block( - &Network::TestNetwork, - cb, - &vks[..], - &[(account, nf)], - Some(&CommitmentTreeMeta::from_parts(0)), - ) - .unwrap(); - let txs = pruned_block.transactions; + let scanned_block = + scan_block(&Network::TestNetwork, cb, &vks[..], &[(account, nf)], None).unwrap(); + let txs = scanned_block.transactions(); assert_eq!(txs.len(), 1); let tx = &txs[0]; @@ -747,15 +797,15 @@ mod tests { assert_eq!(tx.sapling_spends[0].account(), account); assert_eq!( - pruned_block - .sapling_commitments + scanned_block + .sapling_commitments() .iter() .map(|(_, retention)| *retention) .collect::>(), vec![ Retention::Ephemeral, Retention::Checkpoint { - id: pruned_block.block_height, + id: scanned_block.height(), is_marked: false } ] diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index b1e332913..68ffd2ff3 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -19,10 +19,16 @@ and this library adheres to Rust's notion of a note could be spent with fewer than `min_confirmations` confirmations if the wallet did not contain enough observed blocks to satisfy the `min_confirmations` value specified; this situation is now treated as an error. +- A `BlockConflict` variant has been added to `zcash_client_sqlite::error::SqliteClientError` ### Removed - The empty `wallet::transact` module has been removed. +### Fixed +- Fixed an off-by-one error in the `BlockSource` implementation for the SQLite-backed + `BlockDb` block database which could result in blocks being skipped at the start of + scan ranges. + ## [0.7.1] - 2023-05-17 ### Fixed diff --git a/zcash_client_sqlite/src/chain.rs b/zcash_client_sqlite/src/chain.rs index 4d968bb36..478a3bf45 100644 --- a/zcash_client_sqlite/src/chain.rs +++ b/zcash_client_sqlite/src/chain.rs @@ -26,16 +26,16 @@ pub mod migrations; /// Starting at `from_height`, the `with_row` callback is invoked with each block retrieved from /// the backing store. If the `limit` value provided is `None`, all blocks are traversed up to the /// maximum height. -pub(crate) fn blockdb_with_blocks( +pub(crate) fn blockdb_with_blocks( block_source: &BlockDb, from_height: Option, limit: Option, mut with_row: F, -) -> Result<(), Error> +) -> Result<(), Error> where - F: FnMut(CompactBlock) -> Result<(), Error>, + F: FnMut(CompactBlock) -> Result<(), Error>, { - fn to_chain_error, N>(err: E) -> Error { + fn to_chain_error>(err: E) -> Error { Error::BlockSource(err.into()) } @@ -195,16 +195,16 @@ pub(crate) fn blockmetadb_find_block( /// the backing store. If the `limit` value provided is `None`, all blocks are traversed up to the /// maximum height for which metadata is available. #[cfg(feature = "unstable")] -pub(crate) fn fsblockdb_with_blocks( +pub(crate) fn fsblockdb_with_blocks( cache: &FsBlockDb, from_height: Option, limit: Option, mut with_block: F, -) -> Result<(), Error> +) -> Result<(), Error> where - F: FnMut(CompactBlock) -> Result<(), Error>, + F: FnMut(CompactBlock) -> Result<(), Error>, { - fn to_chain_error, N>(err: E) -> Error { + fn to_chain_error>(err: E) -> Error { Error::BlockSource(err.into()) } diff --git a/zcash_client_sqlite/src/error.rs b/zcash_client_sqlite/src/error.rs index 6eb0939f2..db122f1a2 100644 --- a/zcash_client_sqlite/src/error.rs +++ b/zcash_client_sqlite/src/error.rs @@ -53,13 +53,15 @@ pub enum SqliteClientError { /// A received memo cannot be interpreted as a UTF-8 string. InvalidMemo(zcash_primitives::memo::Error), - /// A requested rewind would violate invariants of the - /// storage layer. The payload returned with this error is - /// (safe rewind height, requested height). + /// An attempt to update block data would overwrite the current hash for a block with a + /// different hash. This indicates that a required rewind was not performed. + BlockConflict(BlockHeight), + + /// A requested rewind would violate invariants of the storage layer. The payload returned with + /// this error is (safe rewind height, requested height). RequestedRewindInvalid(BlockHeight, BlockHeight), - /// The space of allocatable diversifier indices has been exhausted for - /// the given account. + /// The space of allocatable diversifier indices has been exhausted for the given account. DiversifierIndexOutOfRange, /// An error occurred deriving a spending key from a seed and an account @@ -115,6 +117,7 @@ impl fmt::Display for SqliteClientError { SqliteClientError::DbError(e) => write!(f, "{}", e), SqliteClientError::Io(e) => write!(f, "{}", e), SqliteClientError::InvalidMemo(e) => write!(f, "{}", e), + SqliteClientError::BlockConflict(h) => write!(f, "A block hash conflict occurred at height {}; rewind required.", u32::from(*h)), SqliteClientError::DiversifierIndexOutOfRange => write!(f, "The space of available diversifier indices is exhausted"), SqliteClientError::KeyDerivationError(acct_id) => write!(f, "Key derivation failed for account {:?}", acct_id), SqliteClientError::AccountIdDiscontinuity => write!(f, "Wallet account identifiers must be sequential."), diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 01c613c2d..c94cf9129 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -55,10 +55,9 @@ use zcash_primitives::{ use zcash_client_backend::{ address::{AddressMetadata, UnifiedAddress}, data_api::{ - self, - chain::{BlockSource, CommitmentTreeMeta}, - DecryptedTransaction, NullifierQuery, PoolType, PrunedBlock, Recipient, SentTransaction, - WalletCommitmentTrees, WalletRead, WalletWrite, SAPLING_SHARD_HEIGHT, + self, chain::BlockSource, BlockMetadata, DecryptedTransaction, NullifierQuery, PoolType, + Recipient, ScannedBlock, SentTransaction, WalletCommitmentTrees, WalletRead, WalletWrite, + SAPLING_SHARD_HEIGHT, }, keys::{UnifiedFullViewingKey, UnifiedSpendingKey}, proto::compact_formats::CompactBlock, @@ -85,7 +84,7 @@ pub mod wallet; /// this delta from the chain tip to be pruned. pub(crate) const PRUNING_HEIGHT: u32 = 100; -pub(crate) const SAPLING_TABLES_PREFIX: &'static str = "sapling"; +pub(crate) const SAPLING_TABLES_PREFIX: &str = "sapling"; /// A newtype wrapper for sqlite primary key values for the notes /// table. @@ -157,10 +156,12 @@ impl, P: consensus::Parameters> WalletRead for W wallet::block_height_extrema(self.conn.borrow()).map_err(SqliteClientError::from) } - fn fully_scanned_height( - &self, - ) -> Result, Self::Error> { - wallet::fully_scanned_height(self.conn.borrow()) + fn block_metadata(&self, height: BlockHeight) -> Result, Self::Error> { + wallet::block_metadata(self.conn.borrow(), height) + } + + fn block_fully_scanned(&self) -> Result, Self::Error> { + wallet::block_fully_scanned(self.conn.borrow()) } fn suggest_scan_ranges( @@ -183,6 +184,14 @@ impl, P: consensus::Parameters> WalletRead for W wallet::get_tx_height(self.conn.borrow(), txid).map_err(SqliteClientError::from) } + fn get_current_address( + &self, + account: AccountId, + ) -> Result, Self::Error> { + wallet::get_current_address(self.conn.borrow(), &self.params, account) + .map(|res| res.map(|(addr, _)| addr)) + } + fn get_unified_full_viewing_keys( &self, ) -> Result, Self::Error> { @@ -196,14 +205,6 @@ impl, P: consensus::Parameters> WalletRead for W wallet::get_account_for_ufvk(self.conn.borrow(), &self.params, ufvk) } - fn get_current_address( - &self, - account: AccountId, - ) -> Result, Self::Error> { - wallet::get_current_address(self.conn.borrow(), &self.params, account) - .map(|res| res.map(|(addr, _)| addr)) - } - fn is_valid_account_extfvk( &self, account: AccountId, @@ -220,10 +221,6 @@ impl, P: consensus::Parameters> WalletRead for W wallet::get_balance_at(self.conn.borrow(), account, anchor_height) } - fn get_transaction(&self, id_tx: i64) -> Result { - wallet::get_transaction(self.conn.borrow(), &self.params, id_tx) - } - fn get_memo(&self, id_note: Self::NoteRef) -> Result, Self::Error> { match id_note { NoteId::SentNoteId(id_note) => wallet::get_sent_memo(self.conn.borrow(), id_note), @@ -233,6 +230,10 @@ impl, P: consensus::Parameters> WalletRead for W } } + fn get_transaction(&self, id_tx: i64) -> Result { + wallet::get_transaction(self.conn.borrow(), &self.params, id_tx) + } + fn get_sapling_nullifiers( &self, query: NullifierQuery, @@ -390,25 +391,25 @@ impl WalletWrite for WalletDb ) } - #[tracing::instrument(skip_all, fields(height = u32::from(block.block_height)))] + #[tracing::instrument(skip_all, fields(height = u32::from(block.height())))] #[allow(clippy::type_complexity)] fn put_block( &mut self, - block: PrunedBlock, + block: ScannedBlock, ) -> Result, Self::Error> { self.transactionally(|wdb| { // Insert the block into the database. wallet::put_block( wdb.conn.0, - block.block_height, - block.block_hash, - block.block_time, - block.sapling_commitment_tree_size.map(|s| s.into()), + block.height(), + block.block_hash(), + block.block_time(), + block.metadata().sapling_tree_size(), )?; let mut wallet_note_ids = vec![]; - for tx in &block.transactions { - let tx_row = wallet::put_tx_meta(wdb.conn.0, tx, block.block_height)?; + for tx in block.transactions() { + let tx_row = wallet::put_tx_meta(wdb.conn.0, tx, block.height())?; // Mark notes as spent and remove them from the scanning cache for spend in &tx.sapling_spends { @@ -424,19 +425,19 @@ impl WalletWrite for WalletDb } } - let sapling_commitments_len = block.sapling_commitments.len(); - let mut sapling_commitments = block.sapling_commitments.into_iter(); + let block_height = block.height(); + let sapling_tree_size = block.metadata().sapling_tree_size(); + let sapling_commitments_len = block.sapling_commitments().len(); + let mut sapling_commitments = block.take_sapling_commitments().into_iter(); wdb.with_sapling_tree_mut::<_, _, SqliteClientError>(move |sapling_tree| { - if let Some(sapling_tree_size) = block.sapling_commitment_tree_size { - let start_position = Position::from(u64::from(sapling_tree_size)) - - u64::try_from(sapling_commitments_len).unwrap(); - sapling_tree.batch_insert(start_position, &mut sapling_commitments)?; - } + let start_position = Position::from(u64::from(sapling_tree_size)) + - u64::try_from(sapling_commitments_len).unwrap(); + sapling_tree.batch_insert(start_position, &mut sapling_commitments)?; Ok(()) })?; // Update now-expired transactions that didn't get mined. - wallet::update_expired_notes(wdb.conn.0, block.block_height)?; + wallet::update_expired_notes(wdb.conn.0, block_height)?; Ok(wallet_note_ids) }) @@ -688,17 +689,14 @@ impl BlockDb { impl BlockSource for BlockDb { type Error = SqliteClientError; - fn with_blocks( + fn with_blocks( &self, from_height: Option, limit: Option, with_row: F, - ) -> Result<(), data_api::chain::error::Error> + ) -> Result<(), data_api::chain::error::Error> where - F: FnMut( - CompactBlock, - ) - -> Result<(), data_api::chain::error::Error>, + F: FnMut(CompactBlock) -> Result<(), data_api::chain::error::Error>, { chain::blockdb_with_blocks(self, from_height, limit, with_row) } @@ -869,17 +867,14 @@ impl FsBlockDb { impl BlockSource for FsBlockDb { type Error = FsBlockDbError; - fn with_blocks( + fn with_blocks( &self, from_height: Option, limit: Option, with_row: F, - ) -> Result<(), data_api::chain::error::Error> + ) -> Result<(), data_api::chain::error::Error> where - F: FnMut( - CompactBlock, - ) - -> Result<(), data_api::chain::error::Error>, + F: FnMut(CompactBlock) -> Result<(), data_api::chain::error::Error>, { fsblockdb_with_blocks(self, from_height, limit, with_row) } @@ -967,7 +962,7 @@ mod tests { data_api::{WalletRead, WalletWrite}, keys::{sapling, UnifiedFullViewingKey}, proto::compact_formats::{ - BlockMetadata, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, + self as compact, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, }, }; @@ -1112,7 +1107,7 @@ mod tests { }; cb.prev_hash.extend_from_slice(&prev_hash.0); cb.vtx.push(ctx); - cb.block_metadata = Some(BlockMetadata { + cb.block_metadata = Some(compact::BlockMetadata { sapling_commitment_tree_size: initial_sapling_tree_size + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(), ..Default::default() @@ -1203,7 +1198,7 @@ mod tests { }; cb.prev_hash.extend_from_slice(&prev_hash.0); cb.vtx.push(ctx); - cb.block_metadata = Some(BlockMetadata { + cb.block_metadata = Some(compact::BlockMetadata { sapling_commitment_tree_size: initial_sapling_tree_size + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(), ..Default::default() diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 722153dbf..13d7ab8c0 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -65,9 +65,9 @@ //! - `memo` the shielded memo associated with the output, if any. use rusqlite::{self, named_params, OptionalExtension, ToSql}; -use std::collections::HashMap; use std::convert::TryFrom; use std::io::Cursor; +use std::{collections::HashMap, io}; use zcash_primitives::{ block::BlockHash, @@ -83,7 +83,7 @@ use zcash_primitives::{ use zcash_client_backend::{ address::{RecipientAddress, UnifiedAddress}, - data_api::{chain::CommitmentTreeMeta, PoolType, Recipient, SentTransactionOutput}, + data_api::{BlockMetadata, PoolType, Recipient, SentTransactionOutput}, encoding::AddressCodec, keys::UnifiedFullViewingKey, wallet::WalletTx, @@ -541,24 +541,88 @@ pub(crate) fn block_height_extrema( }) } -pub(crate) fn fully_scanned_height( +fn parse_block_metadata( + row: (BlockHeight, Vec, Option, Vec), +) -> Option> { + let (block_height, hash_data, sapling_tree_size_opt, sapling_tree) = row; + let sapling_tree_size = sapling_tree_size_opt.map(Ok).or_else(|| { + if sapling_tree == BLOCK_SAPLING_FRONTIER_ABSENT { + None + } else { + // parse the legacy commitment tree data + read_commitment_tree::< + zcash_primitives::sapling::Node, + _, + { zcash_primitives::sapling::NOTE_COMMITMENT_TREE_DEPTH }, + >(Cursor::new(sapling_tree)) + .map(|tree| Some(tree.size().try_into().unwrap())) + .map_err(SqliteClientError::from) + .transpose() + } + })?; + + let block_hash = BlockHash::try_from_slice(&hash_data).ok_or_else(|| { + SqliteClientError::from(io::Error::new( + io::ErrorKind::InvalidData, + format!("Invalid block hash length: {}", hash_data.len()), + )) + }); + + Some(sapling_tree_size.and_then(|sapling_tree_size| { + block_hash.map(|block_hash| { + BlockMetadata::from_parts(block_height, block_hash, sapling_tree_size) + }) + })) +} + +pub(crate) fn block_metadata( + conn: &rusqlite::Connection, + block_height: BlockHeight, +) -> Result, SqliteClientError> { + let res_opt = conn + .query_row( + "SELECT height, hash, sapling_commitment_tree_size, sapling_tree + FROM blocks + WHERE height = :block_height", + named_params![":block_height": u32::from(block_height)], + |row| { + let height: u32 = row.get(0)?; + let block_hash: Vec = row.get(1)?; + let sapling_tree_size: Option = row.get(2)?; + let sapling_tree: Vec = row.get(3)?; + Ok(( + BlockHeight::from(height), + block_hash, + sapling_tree_size, + sapling_tree, + )) + }, + ) + .optional()?; + + res_opt.and_then(parse_block_metadata).transpose() +} + +pub(crate) fn block_fully_scanned( conn: &rusqlite::Connection, -) -> Result, SqliteClientError> { +) -> Result, SqliteClientError> { // FIXME: this will need to be rewritten once out-of-order scan range suggestion // is implemented. let res_opt = conn .query_row( - "SELECT height, sapling_commitment_tree_size, sapling_tree + "SELECT height, hash, sapling_commitment_tree_size, sapling_tree FROM blocks ORDER BY height DESC LIMIT 1", [], |row| { - let max_height: u32 = row.get(0)?; - let sapling_tree_size: Option = row.get(1)?; - let sapling_tree: Vec = row.get(2)?; + let height: u32 = row.get(0)?; + let block_hash: Vec = row.get(1)?; + let sapling_tree_size: Option = row.get(2)?; + let sapling_tree: Vec = row.get(3)?; Ok(( - BlockHeight::from(max_height), + BlockHeight::from(height), + block_hash, sapling_tree_size, sapling_tree, )) @@ -566,32 +630,7 @@ pub(crate) fn fully_scanned_height( ) .optional()?; - res_opt - .and_then(|(max_height, sapling_tree_size, sapling_tree)| { - sapling_tree_size - .map(|s| Ok(CommitmentTreeMeta::from_parts(s))) - .or_else(|| { - if &sapling_tree == BLOCK_SAPLING_FRONTIER_ABSENT { - None - } else { - // parse the legacy commitment tree data - read_commitment_tree::< - zcash_primitives::sapling::Node, - _, - { zcash_primitives::sapling::NOTE_COMMITMENT_TREE_DEPTH }, - >(Cursor::new(sapling_tree)) - .map(|tree| { - Some(CommitmentTreeMeta::from_parts( - tree.size().try_into().unwrap(), - )) - }) - .map_err(SqliteClientError::from) - .transpose() - } - }) - .map(|meta_res| meta_res.map(|meta| (max_height, meta))) - }) - .transpose() + res_opt.and_then(parse_block_metadata).transpose() } /// Returns the block height at which the specified transaction was mined, @@ -834,12 +873,34 @@ pub(crate) fn get_transparent_balances( /// Inserts information about a scanned block into the database. pub(crate) fn put_block( - conn: &rusqlite::Connection, + conn: &rusqlite::Transaction<'_>, block_height: BlockHeight, block_hash: BlockHash, block_time: u32, - sapling_commitment_tree_size: Option, + sapling_commitment_tree_size: u32, ) -> Result<(), SqliteClientError> { + let block_hash_data = conn + .query_row( + "SELECT hash FROM blocks WHERE height = ?", + [u32::from(block_height)], + |row| row.get::<_, Vec>(0), + ) + .optional()?; + + // Ensure that in the case of an upsert, we don't overwrite block data + // with information for a block with a different hash. + if let Some(bytes) = block_hash_data { + let expected_hash = BlockHash::try_from_slice(&bytes).ok_or_else(|| { + SqliteClientError::CorruptedData(format!( + "Invalid block hash at height {}", + u32::from(block_height) + )) + })?; + if expected_hash != block_hash { + return Err(SqliteClientError::BlockConflict(block_height)); + } + } + let mut stmt_upsert_block = conn.prepare_cached( "INSERT INTO blocks ( height, diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index c730aea73..05ccb72a4 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -1,24 +1,29 @@ //! Functions for initializing the various databases. use either::Either; +use incrementalmerkletree::Retention; use std::{collections::HashMap, fmt, io}; use rusqlite::{self, types::ToSql}; use schemer::{Migrator, MigratorError}; use schemer_rusqlite::RusqliteAdapter; use secrecy::SecretVec; -use shardtree::ShardTreeError; +use shardtree::{ShardTree, ShardTreeError}; use uuid::Uuid; use zcash_primitives::{ block::BlockHash, consensus::{self, BlockHeight}, + merkle_tree::read_commitment_tree, + sapling, transaction::components::amount::BalanceError, zip32::AccountId, }; -use zcash_client_backend::keys::UnifiedFullViewingKey; +use zcash_client_backend::{data_api::SAPLING_SHARD_HEIGHT, keys::UnifiedFullViewingKey}; -use crate::{error::SqliteClientError, wallet, WalletDb}; +use crate::{error::SqliteClientError, wallet, WalletDb, SAPLING_TABLES_PREFIX}; + +use super::commitment_tree::SqliteShardStore; mod migrations; @@ -289,9 +294,21 @@ pub fn init_blocks_table( return Err(SqliteClientError::TableNotEmpty); } + let block_end_tree = + read_commitment_tree::( + sapling_tree, + ) + .map_err(|e| { + rusqlite::Error::FromSqlConversionFailure( + sapling_tree.len(), + rusqlite::types::Type::Blob, + Box::new(e), + ) + })?; + wdb.conn.0.execute( "INSERT INTO blocks (height, hash, time, sapling_tree) - VALUES (?, ?, ?, ?)", + VALUES (?, ?, ?, ?)", [ u32::from(height).to_sql()?, hash.0.to_sql()?, @@ -300,6 +317,26 @@ pub fn init_blocks_table( ], )?; + if let Some(nonempty_frontier) = block_end_tree.to_frontier().value() { + let shard_store = + SqliteShardStore::<_, sapling::Node, SAPLING_SHARD_HEIGHT>::from_connection( + wdb.conn.0, + SAPLING_TABLES_PREFIX, + )?; + let mut shard_tree: ShardTree< + _, + { sapling::NOTE_COMMITMENT_TREE_DEPTH }, + SAPLING_SHARD_HEIGHT, + > = ShardTree::new(shard_store, 100); + shard_tree.insert_frontier_nodes( + nonempty_frontier.clone(), + Retention::Checkpoint { + id: height, + is_marked: false, + }, + )?; + } + Ok(()) }) } @@ -1154,7 +1191,7 @@ mod tests { BlockHeight::from(1u32), BlockHash([1; 32]), 1, - &[], + &[0x0, 0x0, 0x0], ) .unwrap(); @@ -1164,7 +1201,7 @@ mod tests { BlockHeight::from(2u32), BlockHash([2; 32]), 2, - &[], + &[0x0, 0x0, 0x0], ) .unwrap_err(); } diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index 85e101c73..a686ec617 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -508,7 +508,7 @@ pub(crate) mod tests { BlockHeight::from(1u32), BlockHash([1; 32]), 1, - &[], + &[0x0, 0x0, 0x0], ) .unwrap(); @@ -562,7 +562,7 @@ pub(crate) mod tests { // Add funds to the wallet in a single note let value = Amount::from_u64(50000).unwrap(); - let (cb, _) = fake_compact_block( + let (mut cb, _) = fake_compact_block( sapling_activation_height(), BlockHash([0; 32]), &dfvk, @@ -588,14 +588,15 @@ pub(crate) mod tests { ); // Add more funds to the wallet in a second note - let (cb, _) = fake_compact_block( + cb = fake_compact_block( sapling_activation_height() + 1, cb.hash(), &dfvk, AddressType::DefaultExternal, value, 1, - ); + ) + .0; insert_into_cache(&db_cache, &cb); scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); @@ -639,14 +640,15 @@ pub(crate) mod tests { // Mine blocks SAPLING_ACTIVATION_HEIGHT + 2 to 9 until just before the second // note is verified for i in 2..10 { - let (cb, _) = fake_compact_block( + cb = fake_compact_block( sapling_activation_height() + i, cb.hash(), &dfvk, AddressType::DefaultExternal, value, i, - ); + ) + .0; insert_into_cache(&db_cache, &cb); } scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); @@ -673,14 +675,15 @@ pub(crate) mod tests { ); // Mine block 11 so that the second note becomes verified - let (cb, _) = fake_compact_block( + cb = fake_compact_block( sapling_activation_height() + 10, cb.hash(), &dfvk, AddressType::DefaultExternal, value, 11, - ); + ) + .0; insert_into_cache(&db_cache, &cb); scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); @@ -718,7 +721,7 @@ pub(crate) mod tests { // Add funds to the wallet in a single note let value = Amount::from_u64(50000).unwrap(); - let (cb, _) = fake_compact_block( + let (mut cb, _) = fake_compact_block( sapling_activation_height(), BlockHash([0; 32]), &dfvk, @@ -774,14 +777,15 @@ pub(crate) mod tests { // Mine blocks SAPLING_ACTIVATION_HEIGHT + 1 to 41 (that don't send us funds) // until just before the first transaction expires for i in 1..42 { - let (cb, _) = fake_compact_block( + cb = fake_compact_block( sapling_activation_height() + i, cb.hash(), &ExtendedSpendingKey::master(&[i as u8]).to_diversifiable_full_viewing_key(), AddressType::DefaultExternal, value, i, - ); + ) + .0; insert_into_cache(&db_cache, &cb); } scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); @@ -807,14 +811,15 @@ pub(crate) mod tests { ); // Mine block SAPLING_ACTIVATION_HEIGHT + 42 so that the first transaction expires - let (cb, _) = fake_compact_block( + cb = fake_compact_block( sapling_activation_height() + 42, cb.hash(), &ExtendedSpendingKey::master(&[42]).to_diversifiable_full_viewing_key(), AddressType::DefaultExternal, value, 42, - ); + ) + .0; insert_into_cache(&db_cache, &cb); scan_cached_blocks(&tests::network(), &db_cache, &mut db_data, None, None).unwrap(); @@ -851,7 +856,7 @@ pub(crate) mod tests { // Add funds to the wallet in a single note let value = Amount::from_u64(50000).unwrap(); - let (cb, _) = fake_compact_block( + let (mut cb, _) = fake_compact_block( sapling_activation_height(), BlockHash([0; 32]), &dfvk, @@ -922,14 +927,15 @@ pub(crate) mod tests { // Mine blocks SAPLING_ACTIVATION_HEIGHT + 1 to 42 (that don't send us funds) // so that the first transaction expires for i in 1..=42 { - let (cb, _) = fake_compact_block( + cb = fake_compact_block( sapling_activation_height() + i, cb.hash(), &ExtendedSpendingKey::master(&[i as u8]).to_diversifiable_full_viewing_key(), AddressType::DefaultExternal, value, i, - ); + ) + .0; insert_into_cache(&db_cache, &cb); } scan_cached_blocks(&network, &db_cache, &mut db_data, None, None).unwrap(); @@ -1073,7 +1079,7 @@ pub(crate) mod tests { let dfvk = usk.sapling().to_diversifiable_full_viewing_key(); // Add funds to the wallet - let (cb, _) = fake_compact_block( + let (mut cb, _) = fake_compact_block( sapling_activation_height(), BlockHash([0; 32]), &dfvk, @@ -1085,14 +1091,15 @@ pub(crate) mod tests { // Add 10 dust notes to the wallet for i in 1..=10 { - let (cb, _) = fake_compact_block( + cb = fake_compact_block( sapling_activation_height() + i, cb.hash(), &dfvk, AddressType::DefaultExternal, Amount::from_u64(1000).unwrap(), i, - ); + ) + .0; insert_into_cache(&db_cache, &cb); } diff --git a/zcash_primitives/src/block.rs b/zcash_primitives/src/block.rs index 6271e0535..b9ef9a5ca 100644 --- a/zcash_primitives/src/block.rs +++ b/zcash_primitives/src/block.rs @@ -39,10 +39,20 @@ impl BlockHash { /// /// This function will panic if the slice is not exactly 32 bytes. pub fn from_slice(bytes: &[u8]) -> Self { - assert_eq!(bytes.len(), 32); - let mut hash = [0; 32]; - hash.copy_from_slice(bytes); - BlockHash(hash) + Self::try_from_slice(bytes).unwrap() + } + + /// Constructs a [`BlockHash`] from the given slice. + /// + /// Returns `None` if `bytes` has any length other than 32 + pub fn try_from_slice(bytes: &[u8]) -> Option { + if bytes.len() == 32 { + let mut hash = [0; 32]; + hash.copy_from_slice(bytes); + Some(BlockHash(hash)) + } else { + None + } } } From 09a0096c748820b85d2997c0444b9ee6ac65b29a Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Sat, 1 Jul 2023 19:49:35 -0600 Subject: [PATCH 24/27] Use valid serialized CommitmentTree values for migration tests. --- zcash_client_sqlite/src/wallet/init.rs | 4 ++-- .../src/wallet/init/migrations/shardtree_support.rs | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index 05ccb72a4..a5fc9f1d0 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -916,7 +916,7 @@ mod tests { // add a sapling sent note wdb.conn.execute( - "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, x'00')", + "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, x'000000')", [], )?; @@ -1080,7 +1080,7 @@ mod tests { RecipientAddress::Transparent(*ufvk.default_address().0.transparent().unwrap()) .encode(&tests::network()); wdb.conn.execute( - "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, x'00')", + "INSERT INTO blocks (height, hash, time, sapling_tree) VALUES (0, 0, 0, x'000000')", [], )?; wdb.conn.execute( diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index 08708d65c..f9f13771f 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -114,9 +114,6 @@ impl RusqliteMigration for Migration { while let Some(row) = block_rows.next()? { let block_height: u32 = row.get(0)?; let sapling_tree_data: Vec = row.get(1)?; - if sapling_tree_data == vec![0x00] { - continue; - } let block_end_tree = read_commitment_tree::< sapling::Node, From 42ed6ba2a1c02eadb86f6ede4fa692c1ba6ae175 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Sat, 1 Jul 2023 20:45:24 -0600 Subject: [PATCH 25/27] Rename `zcash_client_backend::welding_rig` to `zcash_client_backend::scanning` --- zcash_client_backend/CHANGELOG.md | 9 +++++---- zcash_client_backend/src/data_api/chain.rs | 2 +- zcash_client_backend/src/data_api/chain/error.rs | 2 +- zcash_client_backend/src/{welding_rig.rs => scanning.rs} | 4 ++-- 4 files changed, 9 insertions(+), 8 deletions(-) rename zcash_client_backend/src/{welding_rig.rs => scanning.rs} (99%) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 5df41eb40..06cf34db9 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -20,7 +20,7 @@ and this library adheres to Rust's notion of - `ScannedBlock` - `wallet::input_sellection::Proposal::{min_target_height, min_anchor_height}`: - `zcash_client_backend::wallet::WalletSaplingOutput::note_commitment_tree_position` -- `zcash_client_backend::welding_rig::ScanError` +- `zcash_client_backend::scanning::ScanError` ### Changed - MSRV is now 1.65.0. @@ -55,9 +55,10 @@ and this library adheres to Rust's notion of - Arguments to `{propose_transaction, propose_shielding}` have changed. - `zcash_client_backend::wallet::ReceivedSaplingNote::note_commitment_tree_position` has replaced the `witness` field in the same struct. -- `zcash_client_backend::welding_rig::ScanningKey::sapling_nf` has been changed to +- `zcash_client_backend::welding_rig` has been renamed to `zcash_client_backend::scanning` +- `zcash_client_backend::scanning::ScanningKey::sapling_nf` has been changed to take a note position instead of an incremental witness for the note. -- Arguments to `zcash_client_backend::welding_rig::scan_block` have changed. This +- Arguments to `zcash_client_backend::scanning::scan_block` have changed. This method now takes an optional `BlockMetadata` argument instead of a base commitment tree and incremental witnesses for each previously-known note. In addition, the return type has now been updated to return a `Result` @@ -79,7 +80,7 @@ and this library adheres to Rust's notion of - `zcash_client_backend::data_api::chain::validate_chain` TODO: document how to handle validation given out-of-order blocks. - `zcash_client_backend::data_api::chain::error::{ChainError, Cause}` have been - replaced by `zcash_client_backend::welding_rig::ScanError` + replaced by `zcash_client_backend::scanning::ScanError` - `zcash_client_backend::wallet::WalletSaplingOutput::{witness, witness_mut}` have been removed as individual incremental witnesses are no longer tracked on a per-note basis. The global note commitment tree for the wallet should be used diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index 8e1ceb864..3546058e4 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -60,7 +60,7 @@ use crate::{ data_api::{NullifierQuery, WalletWrite}, proto::compact_formats::CompactBlock, scan::BatchRunner, - welding_rig::{add_block_to_runner, scan_block_with_runner}, + scanning::{add_block_to_runner, scan_block_with_runner}, }; pub mod error; diff --git a/zcash_client_backend/src/data_api/chain/error.rs b/zcash_client_backend/src/data_api/chain/error.rs index c1c78cf61..3a21884bc 100644 --- a/zcash_client_backend/src/data_api/chain/error.rs +++ b/zcash_client_backend/src/data_api/chain/error.rs @@ -3,7 +3,7 @@ use std::error; use std::fmt::{self, Debug, Display}; -use crate::welding_rig::ScanError; +use crate::scanning::ScanError; /// Errors related to chain validation and scanning. #[derive(Debug)] diff --git a/zcash_client_backend/src/welding_rig.rs b/zcash_client_backend/src/scanning.rs similarity index 99% rename from zcash_client_backend/src/welding_rig.rs rename to zcash_client_backend/src/scanning.rs index a5d8e9c5d..0229f1a98 100644 --- a/zcash_client_backend/src/welding_rig.rs +++ b/zcash_client_backend/src/scanning.rs @@ -40,7 +40,7 @@ use crate::{ /// nullifier for the note can also be obtained. /// /// [`CompactSaplingOutput`]: crate::proto::compact_formats::CompactSaplingOutput -/// [`scan_block`]: crate::welding_rig::scan_block +/// [`scan_block`]: crate::scanning::scan_block pub trait ScanningKey { /// The type representing the scope of the scanning key. type Scope: Clone + Eq + std::hash::Hash + Send + 'static; @@ -165,7 +165,7 @@ impl fmt::Display for ScanError { /// [`ExtendedFullViewingKey`]: zcash_primitives::zip32::ExtendedFullViewingKey /// [`SaplingIvk`]: zcash_primitives::sapling::SaplingIvk /// [`CompactBlock`]: crate::proto::compact_formats::CompactBlock -/// [`ScanningKey`]: crate::welding_rig::ScanningKey +/// [`ScanningKey`]: crate::scanning::ScanningKey /// [`CommitmentTree`]: zcash_primitives::sapling::CommitmentTree /// [`IncrementalWitness`]: zcash_primitives::sapling::IncrementalWitness /// [`WalletSaplingOutput`]: crate::wallet::WalletSaplingOutput From c363e71fa9d9595a00e25d26384afd16a12884d2 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 3 Jul 2023 16:19:13 -0600 Subject: [PATCH 26/27] Rename proto::compact::{BlockMetadata => ChainMetadata} --- zcash_client_backend/proto/compact_formats.proto | 4 ++-- zcash_client_backend/src/proto/compact_formats.rs | 6 +++--- zcash_client_backend/src/proto/service.rs | 2 +- zcash_client_backend/src/scanning.rs | 4 ++-- zcash_client_sqlite/src/lib.rs | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/zcash_client_backend/proto/compact_formats.proto b/zcash_client_backend/proto/compact_formats.proto index 740d7f7f3..49a5c6f3f 100644 --- a/zcash_client_backend/proto/compact_formats.proto +++ b/zcash_client_backend/proto/compact_formats.proto @@ -13,7 +13,7 @@ option swift_prefix = ""; // BlockMetadata represents information about a block that may not be // represented directly in the block data, but is instead derived from chain // data or other external sources. -message BlockMetadata { +message ChainMetadata { uint32 saplingCommitmentTreeSize = 1; // the size of the Sapling note commitment tree as of the end of this block uint32 orchardCommitmentTreeSize = 2; // the size of the Orchard note commitment tree as of the end of this block } @@ -30,7 +30,7 @@ message CompactBlock { uint32 time = 5; // Unix epoch time when the block was mined bytes header = 6; // (hash, prevHash, and time) OR (full header) repeated CompactTx vtx = 7; // zero or more compact transactions from this block - BlockMetadata blockMetadata = 8; // information about this block derived from the chain or other sources + ChainMetadata chainMetadata = 8; // information about the state of the chain as of this block } // CompactTx contains the minimum information for a wallet to know if this transaction diff --git a/zcash_client_backend/src/proto/compact_formats.rs b/zcash_client_backend/src/proto/compact_formats.rs index bf023eacf..b03018059 100644 --- a/zcash_client_backend/src/proto/compact_formats.rs +++ b/zcash_client_backend/src/proto/compact_formats.rs @@ -3,7 +3,7 @@ /// data or other external sources. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] -pub struct BlockMetadata { +pub struct ChainMetadata { /// the size of the Sapling note commitment tree as of the end of this block #[prost(uint32, tag = "1")] pub sapling_commitment_tree_size: u32, @@ -39,9 +39,9 @@ pub struct CompactBlock { /// zero or more compact transactions from this block #[prost(message, repeated, tag = "7")] pub vtx: ::prost::alloc::vec::Vec, - /// information about this block derived from the chain or other sources + /// information about the state of the chain as of this block #[prost(message, optional, tag = "8")] - pub block_metadata: ::core::option::Option, + pub chain_metadata: ::core::option::Option, } /// CompactTx contains the minimum information for a wallet to know if this transaction /// is relevant to it (either pays to it or spends from it) via shielded elements diff --git a/zcash_client_backend/src/proto/service.rs b/zcash_client_backend/src/proto/service.rs index 581762bb3..17b375e3e 100644 --- a/zcash_client_backend/src/proto/service.rs +++ b/zcash_client_backend/src/proto/service.rs @@ -3,7 +3,7 @@ /// data or other external sources. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] -pub struct BlockMetadata { +pub struct ChainMetadata { /// the size of the Sapling note commitment tree as of the end of this block #[prost(uint32, tag = "1")] pub sapling_commitment_tree_size: u32, diff --git a/zcash_client_backend/src/scanning.rs b/zcash_client_backend/src/scanning.rs index 0229f1a98..194c3cc74 100644 --- a/zcash_client_backend/src/scanning.rs +++ b/zcash_client_backend/src/scanning.rs @@ -264,7 +264,7 @@ pub(crate) fn scan_block_with_runner< // the block, and we can't have a note of ours in a block with no outputs so treating the zero // default value from the protobuf as `None` is always correct. let mut sapling_commitment_tree_size = block - .block_metadata + .chain_metadata .as_ref() .and_then(|m| { if m.sapling_commitment_tree_size == 0 { @@ -592,7 +592,7 @@ mod tests { cb.vtx.push(tx); } - cb.block_metadata = initial_sapling_tree_size.map(|s| compact::BlockMetadata { + cb.chain_metadata = initial_sapling_tree_size.map(|s| compact::ChainMetadata { sapling_commitment_tree_size: s + cb .vtx .iter() diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index c94cf9129..00411d1a6 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -1107,7 +1107,7 @@ mod tests { }; cb.prev_hash.extend_from_slice(&prev_hash.0); cb.vtx.push(ctx); - cb.block_metadata = Some(compact::BlockMetadata { + cb.chain_metadata = Some(compact::ChainMetadata { sapling_commitment_tree_size: initial_sapling_tree_size + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(), ..Default::default() @@ -1198,7 +1198,7 @@ mod tests { }; cb.prev_hash.extend_from_slice(&prev_hash.0); cb.vtx.push(ctx); - cb.block_metadata = Some(compact::BlockMetadata { + cb.chain_metadata = Some(compact::ChainMetadata { sapling_commitment_tree_size: initial_sapling_tree_size + cb.vtx.iter().map(|tx| tx.outputs.len() as u32).sum::(), ..Default::default() From c13c8c667896e8089ad49e88d5c033f9e844ffc4 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Mon, 3 Jul 2023 17:06:43 -0600 Subject: [PATCH 27/27] Address comments from code review. --- zcash_client_backend/CHANGELOG.md | 5 +- zcash_client_backend/build.rs | 4 + .../proto/compact_formats.proto | 4 +- zcash_client_backend/src/data_api.rs | 2 +- zcash_client_backend/src/data_api/chain.rs | 9 +- .../src/proto/compact_formats.rs | 4 +- zcash_client_backend/src/proto/service.rs | 13 --- zcash_client_backend/src/scanning.rs | 5 +- zcash_client_sqlite/src/error.rs | 4 +- zcash_client_sqlite/src/lib.rs | 8 +- zcash_client_sqlite/src/wallet.rs | 105 +++++++++--------- zcash_client_sqlite/src/wallet/init.rs | 4 +- .../init/migrations/shardtree_support.rs | 4 +- 13 files changed, 78 insertions(+), 93 deletions(-) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 06cf34db9..1cb28eaef 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -46,7 +46,7 @@ and this library adheres to Rust's notion of respective `min_confirmations` arguments as `NonZeroU32` - `data_api::wallet::input_selection::InputSelector::{propose_transaction, propose_shielding}` now take their respective `min_confirmations` arguments as `NonZeroU32` - - A new `Sync` variant has been added to `data_api::chain::error::Error`. + - A new `Scan` variant has been added to `data_api::chain::error::Error`. - A new `SyncRequired` variant has been added to `data_api::wallet::input_selection::InputSelectorError`. - `zcash_client_backend::wallet`: - `SpendableNote` has been renamed to `ReceivedSaplingNote`. @@ -61,8 +61,7 @@ and this library adheres to Rust's notion of - Arguments to `zcash_client_backend::scanning::scan_block` have changed. This method now takes an optional `BlockMetadata` argument instead of a base commitment tree and incremental witnesses for each previously-known note. In addition, the - return type has now been updated to return a `Result` - in the case of scan failure. + return type has now been updated to return a `Result`. ### Removed diff --git a/zcash_client_backend/build.rs b/zcash_client_backend/build.rs index 271b0f781..fdc201f57 100644 --- a/zcash_client_backend/build.rs +++ b/zcash_client_backend/build.rs @@ -45,6 +45,10 @@ fn build() -> io::Result<()> { // Build the gRPC types and client. tonic_build::configure() .build_server(false) + .extern_path( + ".cash.z.wallet.sdk.rpc.ChainMetadata", + "crate::proto::compact_formats::ChainMetadata", + ) .extern_path( ".cash.z.wallet.sdk.rpc.CompactBlock", "crate::proto::compact_formats::CompactBlock", diff --git a/zcash_client_backend/proto/compact_formats.proto b/zcash_client_backend/proto/compact_formats.proto index 49a5c6f3f..1db1ecf69 100644 --- a/zcash_client_backend/proto/compact_formats.proto +++ b/zcash_client_backend/proto/compact_formats.proto @@ -10,9 +10,7 @@ option swift_prefix = ""; // Remember that proto3 fields are all optional. A field that is not present will be set to its zero value. // bytes fields of hashes are in canonical little-endian format. -// BlockMetadata represents information about a block that may not be -// represented directly in the block data, but is instead derived from chain -// data or other external sources. +// ChainMetadata represents information about the state of the chain as of a given block. message ChainMetadata { uint32 saplingCommitmentTreeSize = 1; // the size of the Sapling note commitment tree as of the end of this block uint32 orchardCommitmentTreeSize = 2; // the size of the Orchard note commitment tree as of the end of this block diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 6c5f8a115..bd281e282 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -340,7 +340,7 @@ impl ScannedBlock { &self.sapling_commitments } - pub fn take_sapling_commitments(self) -> Vec<(sapling::Node, Retention)> { + pub fn into_sapling_commitments(self) -> Vec<(sapling::Node, Retention)> { self.sapling_commitments } } diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index 3546058e4..fab86eaf6 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -92,10 +92,11 @@ pub trait BlockSource { /// Scans at most `limit` new blocks added to the block source for any transactions received by the /// tracked accounts. /// -/// If the `from_height` argument is not `None`, then the block source will begin requesting blocks -/// from the provided block source at the specified height; if `from_height` is `None then this -/// will begin scanning at first block after the position to which the wallet has previously -/// fully scanned the chain, thereby beginning or continuing a linear scan over all blocks. +/// If the `from_height` argument is not `None`, then this method block source will begin +/// requesting blocks from the provided block source at the specified height; if `from_height` is +/// `None then this will begin scanning at first block after the position to which the wallet has +/// previously fully scanned the chain, thereby beginning or continuing a linear scan over all +/// blocks. /// /// This function will return without error after scanning at most `limit` new blocks, to enable /// the caller to update their UI with scanning progress. Repeatedly calling this function with diff --git a/zcash_client_backend/src/proto/compact_formats.rs b/zcash_client_backend/src/proto/compact_formats.rs index b03018059..2e8a435db 100644 --- a/zcash_client_backend/src/proto/compact_formats.rs +++ b/zcash_client_backend/src/proto/compact_formats.rs @@ -1,6 +1,4 @@ -/// BlockMetadata represents information about a block that may not be -/// represented directly in the block data, but is instead derived from chain -/// data or other external sources. +/// ChainMetadata represents information about the state of the chain as of a given block. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ChainMetadata { diff --git a/zcash_client_backend/src/proto/service.rs b/zcash_client_backend/src/proto/service.rs index 17b375e3e..38b15abdb 100644 --- a/zcash_client_backend/src/proto/service.rs +++ b/zcash_client_backend/src/proto/service.rs @@ -1,16 +1,3 @@ -/// BlockMetadata represents information about a block that may not be -/// represented directly in the block data, but is instead derived from chain -/// data or other external sources. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ChainMetadata { - /// the size of the Sapling note commitment tree as of the end of this block - #[prost(uint32, tag = "1")] - pub sapling_commitment_tree_size: u32, - /// the size of the Orchard note commitment tree as of the end of this block - #[prost(uint32, tag = "2")] - pub orchard_commitment_tree_size: u32, -} /// A BlockID message contains identifiers to select a block: a height or a /// hash. Specification by hash is not implemented, but may be in the future. #[allow(clippy::derive_partial_eq_without_eq)] diff --git a/zcash_client_backend/src/scanning.rs b/zcash_client_backend/src/scanning.rs index 194c3cc74..8eeb1b57e 100644 --- a/zcash_client_backend/src/scanning.rs +++ b/zcash_client_backend/src/scanning.rs @@ -285,7 +285,7 @@ pub(crate) fn scan_block_with_runner< at_height: cur_height, })?; - let block_tx_count = block.vtx.len(); + let compact_block_tx_count = block.vtx.len(); for (tx_idx, tx) in block.vtx.into_iter().enumerate() { let txid = tx.txid(); @@ -392,7 +392,8 @@ pub(crate) fn scan_block_with_runner< { // Collect block note commitments let node = sapling::Node::from_cmu(&output.cmu); - let is_checkpoint = output_idx + 1 == decoded.len() && tx_idx + 1 == block_tx_count; + let is_checkpoint = + output_idx + 1 == decoded.len() && tx_idx + 1 == compact_block_tx_count; let retention = match (dec_output.is_some(), is_checkpoint) { (is_marked, true) => Retention::Checkpoint { id: cur_height, diff --git a/zcash_client_sqlite/src/error.rs b/zcash_client_sqlite/src/error.rs index db122f1a2..1dd14f1c2 100644 --- a/zcash_client_sqlite/src/error.rs +++ b/zcash_client_sqlite/src/error.rs @@ -9,7 +9,7 @@ use shardtree::ShardTreeError; use zcash_client_backend::encoding::{Bech32DecodeError, TransparentCodecError}; use zcash_primitives::{consensus::BlockHeight, zip32::AccountId}; -use crate::PRUNING_HEIGHT; +use crate::PRUNING_DEPTH; #[cfg(feature = "transparent-inputs")] use zcash_primitives::legacy::TransparentAddress; @@ -108,7 +108,7 @@ impl fmt::Display for SqliteClientError { SqliteClientError::InvalidNoteId => write!(f, "The note ID associated with an inserted witness must correspond to a received note."), SqliteClientError::RequestedRewindInvalid(h, r) => - write!(f, "A rewind must be either of less than {} blocks, or at least back to block {} for your wallet; the requested height was {}.", PRUNING_HEIGHT, h, r), + write!(f, "A rewind must be either of less than {} blocks, or at least back to block {} for your wallet; the requested height was {}.", PRUNING_DEPTH, h, r), SqliteClientError::Bech32DecodeError(e) => write!(f, "{}", e), #[cfg(feature = "transparent-inputs")] SqliteClientError::HdwalletError(e) => write!(f, "{:?}", e), diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 00411d1a6..b7351edf8 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -82,7 +82,7 @@ pub mod wallet; /// The maximum number of blocks the wallet is allowed to rewind. This is /// consistent with the bound in zcashd, and allows block data deeper than /// this delta from the chain tip to be pruned. -pub(crate) const PRUNING_HEIGHT: u32 = 100; +pub(crate) const PRUNING_DEPTH: u32 = 100; pub(crate) const SAPLING_TABLES_PREFIX: &str = "sapling"; @@ -428,7 +428,7 @@ impl WalletWrite for WalletDb let block_height = block.height(); let sapling_tree_size = block.metadata().sapling_tree_size(); let sapling_commitments_len = block.sapling_commitments().len(); - let mut sapling_commitments = block.take_sapling_commitments().into_iter(); + let mut sapling_commitments = block.into_sapling_commitments().into_iter(); wdb.with_sapling_tree_mut::<_, _, SqliteClientError>(move |sapling_tree| { let start_position = Position::from(u64::from(sapling_tree_size)) - u64::try_from(sapling_commitments_len).unwrap(); @@ -640,7 +640,7 @@ impl WalletCommitmentTrees for WalletDb WalletCommitmentTrees for WalletDb, Option, Vec), -) -> Option> { +) -> Result { let (block_height, hash_data, sapling_tree_size_opt, sapling_tree) = row; - let sapling_tree_size = sapling_tree_size_opt.map(Ok).or_else(|| { + let sapling_tree_size = sapling_tree_size_opt.map_or_else(|| { if sapling_tree == BLOCK_SAPLING_FRONTIER_ABSENT { - None + Err(SqliteClientError::CorruptedData("One of either the Sapling tree size or the legacy Sapling commitment tree must be present.".to_owned())) } else { // parse the legacy commitment tree data read_commitment_tree::< @@ -555,52 +555,50 @@ fn parse_block_metadata( _, { zcash_primitives::sapling::NOTE_COMMITMENT_TREE_DEPTH }, >(Cursor::new(sapling_tree)) - .map(|tree| Some(tree.size().try_into().unwrap())) + .map(|tree| tree.size().try_into().unwrap()) .map_err(SqliteClientError::from) - .transpose() } - })?; + }, Ok)?; let block_hash = BlockHash::try_from_slice(&hash_data).ok_or_else(|| { SqliteClientError::from(io::Error::new( io::ErrorKind::InvalidData, format!("Invalid block hash length: {}", hash_data.len()), )) - }); + })?; - Some(sapling_tree_size.and_then(|sapling_tree_size| { - block_hash.map(|block_hash| { - BlockMetadata::from_parts(block_height, block_hash, sapling_tree_size) - }) - })) + Ok(BlockMetadata::from_parts( + block_height, + block_hash, + sapling_tree_size, + )) } pub(crate) fn block_metadata( conn: &rusqlite::Connection, block_height: BlockHeight, ) -> Result, SqliteClientError> { - let res_opt = conn - .query_row( - "SELECT height, hash, sapling_commitment_tree_size, sapling_tree + conn.query_row( + "SELECT height, hash, sapling_commitment_tree_size, sapling_tree FROM blocks WHERE height = :block_height", - named_params![":block_height": u32::from(block_height)], - |row| { - let height: u32 = row.get(0)?; - let block_hash: Vec = row.get(1)?; - let sapling_tree_size: Option = row.get(2)?; - let sapling_tree: Vec = row.get(3)?; - Ok(( - BlockHeight::from(height), - block_hash, - sapling_tree_size, - sapling_tree, - )) - }, - ) - .optional()?; - - res_opt.and_then(parse_block_metadata).transpose() + named_params![":block_height": u32::from(block_height)], + |row| { + let height: u32 = row.get(0)?; + let block_hash: Vec = row.get(1)?; + let sapling_tree_size: Option = row.get(2)?; + let sapling_tree: Vec = row.get(3)?; + Ok(( + BlockHeight::from(height), + block_hash, + sapling_tree_size, + sapling_tree, + )) + }, + ) + .optional() + .map_err(SqliteClientError::from) + .and_then(|meta_row| meta_row.map(parse_block_metadata).transpose()) } pub(crate) fn block_fully_scanned( @@ -608,29 +606,28 @@ pub(crate) fn block_fully_scanned( ) -> Result, SqliteClientError> { // FIXME: this will need to be rewritten once out-of-order scan range suggestion // is implemented. - let res_opt = conn - .query_row( - "SELECT height, hash, sapling_commitment_tree_size, sapling_tree + conn.query_row( + "SELECT height, hash, sapling_commitment_tree_size, sapling_tree FROM blocks ORDER BY height DESC LIMIT 1", - [], - |row| { - let height: u32 = row.get(0)?; - let block_hash: Vec = row.get(1)?; - let sapling_tree_size: Option = row.get(2)?; - let sapling_tree: Vec = row.get(3)?; - Ok(( - BlockHeight::from(height), - block_hash, - sapling_tree_size, - sapling_tree, - )) - }, - ) - .optional()?; - - res_opt.and_then(parse_block_metadata).transpose() + [], + |row| { + let height: u32 = row.get(0)?; + let block_hash: Vec = row.get(1)?; + let sapling_tree_size: Option = row.get(2)?; + let sapling_tree: Vec = row.get(3)?; + Ok(( + BlockHeight::from(height), + block_hash, + sapling_tree_size, + sapling_tree, + )) + }, + ) + .optional() + .map_err(SqliteClientError::from) + .and_then(|meta_row| meta_row.map(parse_block_metadata).transpose()) } /// Returns the block height at which the specified transaction was mined, @@ -704,7 +701,7 @@ pub(crate) fn truncate_to_height( .map(|opt| opt.map_or_else(|| sapling_activation_height - 1, BlockHeight::from)) })?; - if block_height < last_scanned_height - PRUNING_HEIGHT { + if block_height < last_scanned_height - PRUNING_DEPTH { if let Some(h) = get_min_unspent_height(conn)? { if block_height > h { return Err(SqliteClientError::RequestedRewindInvalid(h, block_height)); diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index a5fc9f1d0..66efda12e 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -21,7 +21,7 @@ use zcash_primitives::{ use zcash_client_backend::{data_api::SAPLING_SHARD_HEIGHT, keys::UnifiedFullViewingKey}; -use crate::{error::SqliteClientError, wallet, WalletDb, SAPLING_TABLES_PREFIX}; +use crate::{error::SqliteClientError, wallet, WalletDb, PRUNING_DEPTH, SAPLING_TABLES_PREFIX}; use super::commitment_tree::SqliteShardStore; @@ -327,7 +327,7 @@ pub fn init_blocks_table( _, { sapling::NOTE_COMMITMENT_TREE_DEPTH }, SAPLING_SHARD_HEIGHT, - > = ShardTree::new(shard_store, 100); + > = ShardTree::new(shard_store, PRUNING_DEPTH.try_into().unwrap()); shard_tree.insert_frontier_nodes( nonempty_frontier.clone(), Retention::Checkpoint { diff --git a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs index f9f13771f..8a238d00e 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/shardtree_support.rs @@ -23,7 +23,7 @@ use crate::{ commitment_tree::SqliteShardStore, init::{migrations::received_notes_nullable_nf, WalletMigrationError}, }, - SAPLING_TABLES_PREFIX, + PRUNING_DEPTH, SAPLING_TABLES_PREFIX, }; pub(super) const MIGRATION_ID: Uuid = Uuid::from_fields( @@ -103,7 +103,7 @@ impl RusqliteMigration for Migration { _, { sapling::NOTE_COMMITMENT_TREE_DEPTH }, SAPLING_SHARD_HEIGHT, - > = ShardTree::new(shard_store, 100); + > = ShardTree::new(shard_store, PRUNING_DEPTH.try_into().unwrap()); // Insert all the tree information that we can get from block-end commitment trees { let mut stmt_blocks = transaction.prepare("SELECT height, sapling_tree FROM blocks")?;