Skip to content

Commit

Permalink
Initial pull method (#1046)
Browse files Browse the repository at this point in the history 
* new method

Signed-off-by: SupremoUGH <[email protected]>

* checkpoint

Signed-off-by: SupremoUGH <[email protected]>

* Add self-bond filter condition when computing new set of collators (#1043)

Signed-off-by: SupremoUGH <[email protected]>

* everything compiles

Signed-off-by: SupremoUGH <[email protected]>

* typo

Signed-off-by: SupremoUGH <[email protected]>

* chore: add dense_initial_pull

Signed-off-by: SupremoUGH <[email protected]>

* bring back initial_pull

Signed-off-by: SupremoUGH <[email protected]>

* upgrade to 0.5.13

Signed-off-by: SupremoUGH <[email protected]>

* everything compiles

Signed-off-by: SupremoUGH <[email protected]>

* fix cargo lock

Signed-off-by: SupremoUGH <[email protected]>

---------

Signed-off-by: SupremoUGH <[email protected]>
Co-authored-by: Adam Reif <[email protected]>
Co-authored-by: Daniel <[email protected]>
  • Loading branch information
@SupremoUGH @DanielZhangReal
3 people authored Apr 4, 2023
1 parent bfbe464 commit c9bf02f
Show file tree
Hide file tree
Showing 10 changed files with 192 additions and 37 deletions.
20 changes: 10 additions & 10 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

12 changes: 6 additions & 6 deletions pallets/manta-pay/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,17 +107,17 @@ sp-std = { git = 'https://github.com/paritytech/substrate.git', branch = "polkad
jsonrpsee = { version = "0.15.0", features = ["server", "macros"], optional = true }

# manta dependencies
manta-accounting = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", default-features = false }
manta-crypto = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", default-features = false }
manta-pay = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", default-features = false, features = ["groth16", "parameters", "scale"] }
manta-accounting = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", default-features = false }
manta-crypto = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", default-features = false }
manta-pay = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", default-features = false, features = ["groth16", "parameters", "scale"] }
manta-primitives = { path = "../../primitives/manta", default-features = false }
manta-support = { package = "pallet-manta-support", path = "../manta-support", default-features = false }
manta-util = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", default-features = false }
manta-util = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", default-features = false }

[dev-dependencies]
lazy_static = "1.4.0"
manta-crypto = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", features = ["getrandom"] }
manta-pay = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", features = ["groth16", "parameters", "scale", "download", "test"] }
manta-crypto = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", features = ["getrandom"] }
manta-pay = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", features = ["groth16", "parameters", "scale", "download", "test"] }
pallet-asset-manager = { path = "../asset-manager" }
pallet-assets = { git = 'https://github.com/paritytech/substrate.git', branch = "polkadot-v0.9.28" }
pallet-balances = { git = 'https://github.com/paritytech/substrate.git', branch = "polkadot-v0.9.28" }
Expand Down
27 changes: 23 additions & 4 deletions pallets/manta-pay/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,13 +83,13 @@ use manta_pay::{
use manta_primitives::assets::{self, AssetConfig, FungibleLedger as _};
use manta_support::manta_pay::{
asset_value_decode, asset_value_encode, fp_decode, fp_encode, id_from_field, AccountId, Asset,
AssetValue, FullIncomingNote, MTParametersError, NullifierCommitment, OutgoingNote,
ReceiverChunk, SenderChunk, StandardAssetId, TransferPost, Utxo, UtxoAccumulatorOutput,
UtxoItemHashError, UtxoMerkleTreePath, VerifyingContextError, Wrap, WrapPair,
AssetValue, Checkpoint, FullIncomingNote, InitialSyncResponse, MTParametersError,
NullifierCommitment, OutgoingNote, PullResponse, ReceiverChunk, SenderChunk, StandardAssetId,
TransferPost, Utxo, UtxoAccumulatorOutput, UtxoItemHashError, UtxoMerkleTreePath,
VerifyingContextError, Wrap, WrapPair,
};
use manta_util::codec::Encode;

pub use manta_support::manta_pay::{Checkpoint, PullResponse, RawCheckpoint};
pub use pallet::*;
pub use weights::WeightInfo;

Expand Down Expand Up @@ -559,6 +559,25 @@ pub mod pallet {
Shards::<T>::contains_key(shard_index, max_receiver_index)
}

/// Returns the diff of ledger state since the given `checkpoint` and `max_receivers` to
/// perform the initial synchronization.
#[inline]
pub fn initial_pull(checkpoint: Checkpoint, max_receivers: u64) -> InitialSyncResponse {
let (should_continue, receivers) =
Self::pull_receivers(*checkpoint.receiver_index, max_receivers);
let utxo_data = receivers.into_iter().map(|receiver| receiver.0).collect();
let membership_proof_data = (0..=255)
.map(|i| ShardTrees::<T>::get(i).current_path)
.collect();
let nullifier_count = NullifierSetSize::<T>::get() as u128;
InitialSyncResponse {
should_continue,
utxo_data,
membership_proof_data,
nullifier_count,
}
}

/// Pulls sender data from the ledger starting at the `sender_index`.
#[inline]
fn pull_senders(sender_index: usize, max_update_request: u64) -> (bool, SenderChunk) {
Expand Down
60 changes: 58 additions & 2 deletions pallets/manta-pay/src/rpc.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,17 @@

//! MantaPay RPC Interfaces

use crate::{runtime::PullLedgerDiffApi, Checkpoint, PullResponse};
use crate::runtime::PullLedgerDiffApi;
use alloc::sync::Arc;
use core::marker::PhantomData;
use jsonrpsee::{
core::{async_trait, RpcResult},
proc_macros::rpc,
types::error::{CallError, ErrorObject},
};
use manta_support::manta_pay::DensePullResponse;
use manta_support::manta_pay::{
Checkpoint, DenseInitialSyncResponse, DensePullResponse, InitialSyncResponse, PullResponse,
};
use sp_api::ProvideRuntimeApi;
use sp_blockchain::HeaderBackend;
use sp_runtime::{generic::BlockId, traits::Block};
Expand Down Expand Up @@ -52,6 +54,21 @@ pub trait PullApi {
max_receivers: u64,
max_senders: u64,
) -> RpcResult<DensePullResponse>;

/// Returns the update required for the initial synchronization with the ledger.
#[method(name = "mantaPay_initial_pull", blocking)]
fn initial_pull(
&self,
checkpoint: Checkpoint,
max_receivers: u64,
) -> RpcResult<InitialSyncResponse>;

#[method(name = "mantaPay_dense_initial_pull", blocking)]
fn dense_initial_pull(
&self,
checkpoint: Checkpoint,
max_receivers: u64,
) -> RpcResult<DenseInitialSyncResponse>;
}

/// Pull RPC API Implementation
Expand Down Expand Up @@ -121,4 +138,43 @@ where
.into()
})
}

#[inline]
fn initial_pull(
&self,
checkpoint: Checkpoint,
max_receivers: u64,
) -> RpcResult<InitialSyncResponse> {
let api = self.client.runtime_api();
let at = BlockId::hash(self.client.info().finalized_hash);
api.initial_pull(&at, checkpoint.into(), max_receivers)
.map_err(|err| {
CallError::Custom(ErrorObject::owned(
PULL_LEDGER_DIFF_ERROR,
"Unable to compute state diff for initial pull",
Some(format!("{err:?}")),
))
.into()
})
}

#[inline]
fn dense_initial_pull(
&self,
checkpoint: Checkpoint,
max_receivers: u64,
) -> RpcResult<DenseInitialSyncResponse> {
let api = self.client.runtime_api();
let at = BlockId::hash(self.client.info().finalized_hash);
api.initial_pull(&at, checkpoint.into(), max_receivers)
.map(Into::into)
.map_err(|err| {
CallError::Custom(ErrorObject::owned(
PULL_LEDGER_DIFF_ERROR,
"Unable to compute state diff for initial pull",
Some(format!("{err:?}")),
))
.into()
})
}
}
3 changes: 2 additions & 1 deletion pallets/manta-pay/src/runtime.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,11 @@

//! MantaPay Runtime APIs

use crate::{PullResponse, RawCheckpoint};
use manta_support::manta_pay::{InitialSyncResponse, PullResponse, RawCheckpoint};

sp_api::decl_runtime_apis! {
pub trait PullLedgerDiffApi {
fn pull_ledger_diff(checkpoint: RawCheckpoint, max_receivers: u64, max_senders: u64) -> PullResponse;
fn initial_pull(checkpoint: RawCheckpoint, max_receivers: u64) -> InitialSyncResponse;
}
}
12 changes: 6 additions & 6 deletions pallets/manta-sbt/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,17 +101,17 @@ sp-std = { git = 'https://github.com/paritytech/substrate.git', branch = "polkad
jsonrpsee = { version = "0.15.0", features = ["server", "macros"], optional = true }

# manta dependencies
manta-accounting = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", default-features = false }
manta-crypto = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", default-features = false }
manta-pay = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", default-features = false, features = ["groth16", "parameters", "scale"] }
manta-accounting = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", default-features = false }
manta-crypto = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", default-features = false }
manta-pay = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", default-features = false, features = ["groth16", "parameters", "scale"] }
manta-primitives = { path = "../../primitives/manta", default-features = false }
manta-support = { package = "pallet-manta-support", path = "../manta-support", default-features = false }
manta-util = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", default-features = false }
manta-util = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", default-features = false }

[dev-dependencies]
lazy_static = "1.4.0"
manta-crypto = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", features = ["getrandom"] }
manta-pay = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", features = ["groth16", "parameters", "scale", "download", "test"] }
manta-crypto = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", features = ["getrandom"] }
manta-pay = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", features = ["groth16", "parameters", "scale", "download", "test"] }
pallet-asset-manager = { path = "../asset-manager" }
pallet-assets = { git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.28" }
pallet-balances = { git = 'https://github.com/paritytech/substrate.git', branch = "polkadot-v0.9.28" }
Expand Down
8 changes: 4 additions & 4 deletions pallets/manta-support/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,11 @@ sp-runtime = { git = "https://github.com/paritytech/substrate.git", branch = "po
sp-std = { git = 'https://github.com/paritytech/substrate.git', branch = "polkadot-v0.9.28", default-features = false }

# manta dependencies
manta-accounting = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", default-features = false }
manta-crypto = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", default-features = false }
manta-pay = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", default-features = false, features = ["groth16", "parameters", "scale"] }
manta-accounting = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", default-features = false }
manta-crypto = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", default-features = false }
manta-pay = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", default-features = false, features = ["groth16", "parameters", "scale"] }
manta-primitives = { path = "../../primitives/manta", default-features = false }
manta-util = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", default-features = false }
manta-util = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", default-features = false }

[features]
default = ["std"]
Expand Down
77 changes: 75 additions & 2 deletions pallets/manta-support/src/manta_pay.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
//! Type Definitions for Manta Pay

use alloc::{boxed::Box, vec::Vec};
use manta_crypto::merkle_tree;
use manta_pay::{
config::{
self,
Expand All @@ -26,6 +25,7 @@ use manta_pay::{
crypto::poseidon::encryption::{self, BlockArray, CiphertextBlock},
manta_crypto::{
encryption::{hybrid, EmptyHeader},
merkle_tree,
permutation::duplex,
signature::schnorr,
},
Expand Down Expand Up @@ -825,7 +825,12 @@ pub type LeafDigest = [u8; 32];
pub type InnerDigest = [u8; 32];

/// Merkle Tree Current Path
#[derive(Clone, Debug, Decode, Default, Encode, Eq, PartialEq, TypeInfo)]
#[cfg_attr(
feature = "serde",
derive(Deserialize, Serialize),
serde(crate = "manta_util::serde", deny_unknown_fields)
)]
#[derive(Clone, Debug, Decode, Default, Encode, Eq, Hash, PartialEq, TypeInfo)]
pub struct CurrentPath {
/// Sibling Digest
pub sibling_digest: LeafDigest,
Expand Down Expand Up @@ -899,9 +904,77 @@ pub struct UtxoMerkleTreePath {
/// Receiver Chunk Data Type
pub type ReceiverChunk = Vec<(Utxo, FullIncomingNote)>;

/// Utxo Chunk Data Type
pub type UtxoChunk = Vec<Utxo>;

/// Merkle Tree [`CurrentPath`] Chunk Data Type
pub type CurrentPathChunk = Vec<CurrentPath>;

/// Sender Chunk Data Type
pub type SenderChunk = Vec<(NullifierCommitment, OutgoingNote)>;

/// Initial Sync Response
#[cfg_attr(
feature = "serde",
derive(Deserialize, Serialize),
serde(crate = "manta_util::serde", deny_unknown_fields)
)]
#[derive(Clone, Debug, Decode, Default, Encode, Eq, Hash, PartialEq, TypeInfo)]
pub struct InitialSyncResponse {
/// Initial Sync Continuation Flag
///
/// The `should_continue` flag is set to `true` if the client should request more data from the
/// ledger to finish the pull.
pub should_continue: bool,

/// Ledger Utxo Chunk
pub utxo_data: UtxoChunk,

/// Ledger [`CurrentPath`] Chunk
pub membership_proof_data: CurrentPathChunk,

/// Nullifier Count
pub nullifier_count: u128,
}

/// Ledger Source Dense Pull Response
#[cfg_attr(
feature = "serde",
derive(Deserialize, Serialize),
serde(crate = "manta_util::serde", deny_unknown_fields)
)]
#[derive(Clone, Debug, Encode, Default, Eq, Hash, Decode, PartialEq, TypeInfo)]
pub struct DenseInitialSyncResponse {
/// Pull Continuation Flag
///
/// The `should_continue` flag is set to `true` if the client should request more data from the
/// ledger to finish the pull.
pub should_continue: bool,

/// Ledger Utxo Chunk
#[codec(skip)]
pub utxo_data: alloc::string::String,

/// Ledger [`CurrentPath`] Chunk
#[codec(skip)]
pub membership_proof_data: alloc::string::String,

/// Nullifier Count
pub nullifier_count: u128,
}

impl From<InitialSyncResponse> for DenseInitialSyncResponse {
#[inline]
fn from(resp: InitialSyncResponse) -> DenseInitialSyncResponse {
Self {
should_continue: resp.should_continue,
utxo_data: base64::encode(resp.utxo_data.encode()),
membership_proof_data: base64::encode(resp.membership_proof_data.encode()),
nullifier_count: resp.nullifier_count,
}
}
}

/// Ledger Source Pull Response
#[cfg_attr(
feature = "serde",
Expand Down
5 changes: 4 additions & 1 deletion runtime/calamari/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ use manta_primitives::{
constants::{time::*, RocksDbWeight, STAKING_PALLET_ID, TREASURY_PALLET_ID, WEIGHT_PER_SECOND},
types::{AccountId, Balance, BlockNumber, Hash, Header, Index, Signature},
};
use manta_support::manta_pay::{PullResponse, RawCheckpoint};
use manta_support::manta_pay::{InitialSyncResponse, PullResponse, RawCheckpoint};
pub use pallet_parachain_staking::{InflationInfo, Range};
use pallet_session::ShouldEndSession;
use runtime_common::{
Expand Down Expand Up @@ -1070,6 +1070,9 @@ impl_runtime_apis! {
) -> PullResponse {
MantaPay::pull_ledger_diff(checkpoint.into(), max_receiver, max_sender)
}
fn initial_pull(checkpoint: RawCheckpoint, max_receiver: u64) -> InitialSyncResponse {
MantaPay::initial_pull(checkpoint.into(), max_receiver)
}
}

impl pallet_manta_sbt::runtime::SBTPullLedgerDiffApi<Block> for Runtime {
Expand Down
Loading

0 comments on commit c9bf02f

Please sign in to comment.